From 1d3090326210c6e6f7ec5432799ded25b75bba46 Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Thu, 28 May 2009 11:23:00 +0200 Subject: beta 2009.05.28 11:23 --- scripts/context/lua/luatools.lua | 8503 ++++++++------- scripts/context/lua/mtx-babel.lua | 18 +- scripts/context/lua/mtx-cache.lua | 4 +- scripts/context/lua/mtx-chars.lua | 306 +- scripts/context/lua/mtx-check.lua | 43 +- scripts/context/lua/mtx-context.lua | 979 +- scripts/context/lua/mtx-convert.lua | 8 +- scripts/context/lua/mtx-fonts.lua | 117 +- scripts/context/lua/mtx-grep.lua | 106 +- scripts/context/lua/mtx-interface.lua | 64 +- scripts/context/lua/mtx-metatex.lua | 69 + scripts/context/lua/mtx-mptopdf.lua | 24 +- scripts/context/lua/mtx-package.lua | 68 + scripts/context/lua/mtx-patterns.lua | 101 +- scripts/context/lua/mtx-profile.lua | 164 + scripts/context/lua/mtx-server-ctx-fonttest.lua | 681 ++ scripts/context/lua/mtx-server-ctx-help.lua | 648 ++ scripts/context/lua/mtx-server-ctx-startup.lua | 53 + scripts/context/lua/mtx-server.lua | 132 +- scripts/context/lua/mtx-timing.lua | 201 + scripts/context/lua/mtx-unzip.lua | 101 + scripts/context/lua/mtx-update.lua | 419 +- scripts/context/lua/mtx-watch.lua | 24 +- scripts/context/lua/mtxrun.lua | 12057 ++++++++++++---------- scripts/context/lua/x-ldx.lua | 4 +- scripts/context/ruby/base/exa.rb | 9 +- scripts/context/ruby/base/file.rb | 7 +- scripts/context/ruby/base/kpse.rb | 8 +- scripts/context/ruby/base/state.rb | 4 +- scripts/context/ruby/base/tex.rb | 176 +- scripts/context/ruby/base/texutil.rb | 33 +- scripts/context/ruby/base/tool.rb | 15 +- scripts/context/ruby/ctxtools.rb | 4 +- scripts/context/ruby/fcd_start.rb | 19 +- scripts/context/ruby/graphics/gs.rb | 20 +- scripts/context/ruby/pdftools.rb | 3 +- scripts/context/ruby/rlxtools.rb | 3 +- scripts/context/ruby/rsfiltool.rb | 3 +- scripts/context/ruby/runtools.rb | 3 +- scripts/context/ruby/texexec.rb | 8 +- scripts/context/ruby/texmfstart.rb | 1261 +-- scripts/context/ruby/textools.rb | 3 +- scripts/context/ruby/www/admin.rb | 215 - scripts/context/ruby/www/common.rb | 80 - scripts/context/ruby/www/dir.rb | 155 - scripts/context/ruby/www/exa.rb | 387 - scripts/context/ruby/www/lib.rb | 1405 --- scripts/context/ruby/www/login.rb | 13 - scripts/context/ruby/wwwclient.rb | 677 -- scripts/context/ruby/wwwserver.rb | 293 - scripts/context/ruby/wwwwatch.rb | 497 - scripts/context/stubs/mswin/ctxtools.bat | 5 +- scripts/context/stubs/mswin/exatools.bat | 2 - scripts/context/stubs/mswin/luatools.lua | 6977 +++++++++++++ scripts/context/stubs/mswin/makempy.bat | 5 +- scripts/context/stubs/mswin/metatex.cmd | 5 + scripts/context/stubs/mswin/mpstools.bat | 5 +- scripts/context/stubs/mswin/mptopdf.bat | 5 +- scripts/context/stubs/mswin/mtxrun.lua | 10190 ++++++++++++++++++ scripts/context/stubs/mswin/mtxtools.bat | 5 + scripts/context/stubs/mswin/pdftools.bat | 5 +- scripts/context/stubs/mswin/pdftrimwhite.bat | 2 - scripts/context/stubs/mswin/pstopdf.bat | 5 +- scripts/context/stubs/mswin/rlxtools.bat | 5 +- scripts/context/stubs/mswin/runtools.bat | 5 +- scripts/context/stubs/mswin/texexec.bat | 5 +- scripts/context/stubs/mswin/texexec.cmd | 5 + scripts/context/stubs/mswin/texfind.bat | 2 - scripts/context/stubs/mswin/texfont.bat | 5 +- scripts/context/stubs/mswin/texmfstart.cmd | 5 + scripts/context/stubs/mswin/texshow.bat | 2 - scripts/context/stubs/mswin/textools.bat | 5 +- scripts/context/stubs/mswin/texutil.bat | 5 +- scripts/context/stubs/mswin/tmftools.bat | 5 +- scripts/context/stubs/mswin/xmltools.bat | 5 +- scripts/context/stubs/unix/ctxtools | 2 +- scripts/context/stubs/unix/exatools | 2 - scripts/context/stubs/unix/luatools | 6977 +++++++++++++ scripts/context/stubs/unix/makempy | 2 +- scripts/context/stubs/unix/metatex | 2 + scripts/context/stubs/unix/mpstools | 2 +- scripts/context/stubs/unix/mptopdf | 2 +- scripts/context/stubs/unix/mtxrun | 10190 ++++++++++++++++++ scripts/context/stubs/unix/mtxtools | 2 + scripts/context/stubs/unix/pdftools | 2 +- scripts/context/stubs/unix/pdftrimwhite | 2 - scripts/context/stubs/unix/pstopdf | 2 +- scripts/context/stubs/unix/rlxtools | 2 +- scripts/context/stubs/unix/runtools | 2 +- scripts/context/stubs/unix/texexec | 2 +- scripts/context/stubs/unix/texfind | 2 - scripts/context/stubs/unix/texfont | 2 +- scripts/context/stubs/unix/texmfstart | 2 + scripts/context/stubs/unix/texshow | 2 - scripts/context/stubs/unix/textools | 2 +- scripts/context/stubs/unix/texutil | 2 +- scripts/context/stubs/unix/tmftools | 2 +- scripts/context/stubs/unix/xmltools | 2 +- 98 files changed, 49078 insertions(+), 15591 deletions(-) create mode 100644 scripts/context/lua/mtx-metatex.lua create mode 100644 scripts/context/lua/mtx-package.lua create mode 100644 scripts/context/lua/mtx-profile.lua create mode 100644 scripts/context/lua/mtx-server-ctx-fonttest.lua create mode 100644 scripts/context/lua/mtx-server-ctx-help.lua create mode 100644 scripts/context/lua/mtx-server-ctx-startup.lua create mode 100644 scripts/context/lua/mtx-timing.lua create mode 100644 scripts/context/lua/mtx-unzip.lua delete mode 100644 scripts/context/ruby/www/admin.rb delete mode 100644 scripts/context/ruby/www/common.rb delete mode 100644 scripts/context/ruby/www/dir.rb delete mode 100644 scripts/context/ruby/www/exa.rb delete mode 100644 scripts/context/ruby/www/lib.rb delete mode 100644 scripts/context/ruby/www/login.rb delete mode 100644 scripts/context/ruby/wwwclient.rb delete mode 100644 scripts/context/ruby/wwwserver.rb delete mode 100644 scripts/context/ruby/wwwwatch.rb delete mode 100755 scripts/context/stubs/mswin/exatools.bat create mode 100644 scripts/context/stubs/mswin/luatools.lua create mode 100644 scripts/context/stubs/mswin/metatex.cmd create mode 100644 scripts/context/stubs/mswin/mtxrun.lua create mode 100755 scripts/context/stubs/mswin/mtxtools.bat delete mode 100755 scripts/context/stubs/mswin/pdftrimwhite.bat create mode 100644 scripts/context/stubs/mswin/texexec.cmd delete mode 100755 scripts/context/stubs/mswin/texfind.bat create mode 100644 scripts/context/stubs/mswin/texmfstart.cmd delete mode 100755 scripts/context/stubs/mswin/texshow.bat delete mode 100755 scripts/context/stubs/unix/exatools create mode 100755 scripts/context/stubs/unix/luatools create mode 100755 scripts/context/stubs/unix/metatex create mode 100755 scripts/context/stubs/unix/mtxrun create mode 100755 scripts/context/stubs/unix/mtxtools delete mode 100755 scripts/context/stubs/unix/pdftrimwhite delete mode 100755 scripts/context/stubs/unix/texfind create mode 100755 scripts/context/stubs/unix/texmfstart delete mode 100755 scripts/context/stubs/unix/texshow (limited to 'scripts') diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua index 1e38edeab..aacdbd16d 100644 --- a/scripts/context/lua/luatools.lua +++ b/scripts/context/lua/luatools.lua @@ -1,25 +1,26 @@ #!/usr/bin/env texlua +if not modules then modules = { } end modules ['luatools'] = { + version = 1.001, + comment = "companion to context.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format = string.format + -- one can make a stub: -- -- #!/bin/sh -- env LUATEXDIR=/....../texmf/scripts/context/lua texlua luatools.lua "$@" --- filename : luatools.lua --- comment : companion to context.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -- Although this script is part of the ConTeXt distribution it is -- relatively indepent of ConTeXt. The same is true for some of -- the luat files. We may may make them even less dependent in -- the future. As long as Luatex is under development the -- interfaces and names of functions may change. -banner = "version 1.2.2 - 2006+ - PRAGMA ADE / CONTEXT" -texlua = true - -- For the sake of independence we optionally can merge the library -- code here. It's too much code, but that does not harm. Much of the -- library code is used elsewhere. We don't want dependencies on @@ -28,141 +29,42 @@ texlua = true -- needed when texmfstart is used, or when the proper stub is used or -- when (windows) suffix binding is active. +texlua = true + -- begin library merge --- filename : l-string.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-string'] = 1.001 - ---~ function string.split(str, pat) -- taken from the lua wiki ---~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ table.insert(t,cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end<=string.len(str) then ---~ table.insert(t,(string.sub(str,last_end))) ---~ end ---~ return t ---~ end ---~ function string:split(pat) -- taken from the lua wiki but adapted ---~ local t = { } -- self and colon usage (faster) ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = self:find(fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ t[#t+1] = cap ---~ end ---~ last_end = e+1 ---~ s, e, cap = self:find(fpat, last_end) ---~ end ---~ if last_end <= #self then ---~ t[#t+1] = self:sub(last_end) ---~ end ---~ return t ---~ end ---~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed ---~ ---~ function string:splitter(pat) ---~ local st, g = 1, self:gmatch("()"..pat.."()") ---~ local function splitter(self) ---~ if st then ---~ local s, f = g() ---~ local rv = self:sub(st, (s or 0)-1) ---~ st = f ---~ return rv ---~ end ---~ end ---~ return splitter, self ---~ end -function string:splitter(pat) - -- by Rici Lake (posted on lua list) -- only names changed - -- p 79 ref man: () returns position of match - local st, g = 1, self:gmatch("()("..pat..")") - local function strgetter(self, segs, seps, sep, cap1, ...) - st = sep and seps + #sep - return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... - end - local function strsplitter(self) - if st then return strgetter(self, st, g()) end - end - return strsplitter, self -end +do -- create closure to overcome 200 locals limit -function string:split(separator) - local t = {} - for k in self:splitter(separator) do t[#t+1] = k end - return t -end +if not modules then modules = { } end modules ['l-string'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --- faster than a string:split: +local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep -function string:splitchr(chr) - if #self > 0 then - local t = { } - for s in (self..chr):gmatch("(.-)"..chr) do - t[#t+1] = s +if not string.split then + + -- this will be overloaded by a faster lpeg variant + + function string:split(pattern) + if #self > 0 then + local t = { } + for s in gmatch(self..pattern,"(.-)"..pattern) do + t[#t+1] = s + end + return t + else + return { } end - return t - else - return { } end -end -function string.piecewise(str, pat, fnc) -- variant of split - for k in string.splitter(str,pat) do fnc(k) end end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ for k in str:splitter(pat) do fnc(k) end ---~ end - ---~ do if lpeg then - ---~ -- this alternative is 30% faster esp when we cache them ---~ -- problem: no expressions - ---~ splitters = { } - ---~ function string:split(separator) ---~ if #self > 0 then ---~ local split = splitters[separator] ---~ if not split then ---~ -- based on code by Roberto ---~ local p = lpeg.P(separator) ---~ local c = lpeg.C((1-p)^0) ---~ split = lpeg.Ct(c*(p*c)^0) ---~ splitters[separator] = split ---~ end ---~ return split:match(self) ---~ else ---~ return { } ---~ end ---~ end - ---~ string.splitchr = string.split - ---~ function string:piecewise(separator,fnc) ---~ for _,v in pairs(self:split(separator)) do ---~ fnc(v) ---~ end ---~ end - ---~ end end - local chr_to_esc = { ["%"] = "%%", ["."] = "%.", @@ -176,20 +78,20 @@ local chr_to_esc = { string.chr_to_esc = chr_to_esc function string:esc() -- variant 2 - return (self:gsub("(.)",chr_to_esc)) + return (gsub(self,"(.)",chr_to_esc)) end function string:unquote() - return (self:gsub("^([\"\'])(.*)%1$","%2")) + return (gsub(self,"^([\"\'])(.*)%1$","%2")) end -function string:quote() +function string:quote() -- we could use format("%q") return '"' .. self:unquote() .. '"' end function string:count(pattern) -- variant 3 local n = 0 - for _ in self:gmatch(pattern) do + for _ in gmatch(self,pattern) do n = n + 1 end return n @@ -198,29 +100,25 @@ end function string:limit(n,sentinel) if #self > n then sentinel = sentinel or " ..." - return self:sub(1,(n-#sentinel)) .. sentinel + return sub(self,1,(n-#sentinel)) .. sentinel else return self end end function string:strip() - return (self:gsub("^%s*(.-)%s*$", "%1")) + return (gsub(self,"^%s*(.-)%s*$", "%1")) end ---~ function string.strip(str) -- slightly different ---~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) ---~ end - function string:is_empty() - return not self:find("%S") + return not find(find,"%S") end function string:enhance(pattern,action) local ok, n = true, 0 while ok do ok = false - self = self:gsub(pattern, function(...) + self = gsub(self,pattern, function(...) ok, n = true, n + 1 return action(...) end) @@ -228,59 +126,19 @@ function string:enhance(pattern,action) return self, n end ---~ function string:enhance(pattern,action) ---~ local ok, n = 0, 0 ---~ repeat ---~ self, ok = self:gsub(pattern, function(...) ---~ n = n + 1 ---~ return action(...) ---~ end) ---~ until ok == 0 ---~ return self, n ---~ end - ---~ function string:to_hex() ---~ if self then ---~ return (self:gsub("(.)",function(c) ---~ return string.format("%02X",c:byte()) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - ---~ function string:from_hex() ---~ if self then ---~ return (self:gsub("(..)",function(c) ---~ return string.char(tonumber(c,16)) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - -string.chr_to_hex = { } -string.hex_to_chr = { } +local chr_to_hex, hex_to_chr = { }, { } for i=0,255 do - local c, h = string.char(i), string.format("%02X",i) - string.chr_to_hex[c], string.hex_to_chr[h] = h, c + local c, h = char(i), format("%02X",i) + chr_to_hex[c], hex_to_chr[h] = h, c end ---~ function string:to_hex() ---~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end ---~ end - ---~ function string:from_hex() ---~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end ---~ end - function string:to_hex() - return ((self or ""):gsub("(.)",string.chr_to_hex)) + return (gsub(self or "","(.)",chr_to_hex)) end function string:from_hex() - return ((self or ""):gsub("(..)",string.hex_to_chr)) + return (gsub(self or "","(..)",hex_to_chr)) end if not string.characters then @@ -294,7 +152,7 @@ if not string.characters then end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(str:sub(index,index)) end function string:bytes() return nextbyte, self, 0 @@ -302,9 +160,7 @@ if not string.characters then end ---~ function string:padd(n,chr) ---~ return self .. self.rep(chr or " ",n-#self) ---~ end +-- we can use format for this (neg n) function string:rpadd(n,chr) local m = n-#self @@ -326,8 +182,8 @@ end string.padd = string.rpadd -function is_number(str) - return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +function is_number(str) -- tonumber + return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 end --~ print(is_number("1")) @@ -339,9 +195,9 @@ end --~ print(is_number("+.1")) function string:split_settings() -- no {} handling, see l-aux for lpeg variant - if self:find("=") then + if find(self,"=") then local t = { } - for k,v in self:gmatch("(%a+)=([^%,]*)") do + for k,v in gmatch(self,"(%a+)=([^%,]*)") do t[k] = v end return t @@ -363,24 +219,67 @@ local patterns_escapes = { } function string:pattesc() - return (self:gsub(".",patterns_escapes)) + return (gsub(self,".",patterns_escapes)) end function string:tohash() local t = { } - for s in self:gmatch("([^, ]+)") do -- lpeg + for s in gmatch(self,"([^, ]+)") do -- lpeg t[s] = true end return t end +local pattern = lpeg.Ct(lpeg.C(1)^0) + +function string:totable() + return pattern:match(self) +end + +--~ for _, str in ipairs { +--~ "1234567123456712345671234567", +--~ "a\tb\tc", +--~ "aa\tbb\tcc", +--~ "aaa\tbbb\tccc", +--~ "aaaa\tbbbb\tcccc", +--~ "aaaaa\tbbbbb\tccccc", +--~ "aaaaaa\tbbbbbb\tcccccc", +--~ } do print(string.tabtospace(str)) end + +function string.tabtospace(str,tab) + -- we don't handle embedded newlines + while true do + local s = find(str,"\t") + if s then + if not tab then tab = 7 end -- only when found + local d = tab-(s-1)%tab + if d > 0 then + str = gsub(str,"\t",rep(" ",d),1) + else + str = gsub(str,"\t","",1) + end + else + break + end + end + return str +end + + + +end -- of closure --- filename : l-lpeg.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +do -- create closure to overcome 200 locals limit -if not versions then versions = { } end versions['l-lpeg'] = 1.001 +if not modules then modules = { } end modules ['l-lpeg'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc --~ l-lpeg.lua : @@ -404,36 +303,40 @@ if not versions then versions = { } end versions['l-lpeg'] = 1.001 local hash = { } function lpeg.anywhere(pattern) --slightly adapted from website - return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } + return P { P(pattern) + 1 * lpeg.V(1) } end function lpeg.startswith(pattern) --slightly adapted - return lpeg.P(pattern) + return P(pattern) end ---~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 - function lpeg.splitter(pattern, action) - return (((1-lpeg.P(pattern))^1)/action+1)^0 + return (((1-P(pattern))^1)/action+1)^0 end -local crlf = lpeg.P("\r\n") -local cr = lpeg.P("\r") -local lf = lpeg.P("\n") -local space = lpeg.S(" \t\f\v") +-- variant: + +--~ local parser = lpeg.Ct(lpeg.splitat(newline)) + +local crlf = P("\r\n") +local cr = P("\r") +local lf = P("\n") +local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) local newline = crlf + cr + lf local spacing = space^0 * newline -local empty = spacing * lpeg.Cc("") -local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local empty = spacing * Cc("") +local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 -local capture = lpeg.Ct(content^0) +local capture = Ct(content^0) function string:splitlines() return capture:match(self) end +lpeg.linebyline = content -- better make a sublibrary + --~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more --~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more --~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps @@ -441,16 +344,16 @@ end local splitters_s, splitters_m = { }, { } -function lpeg.splitat(separator,single) +local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] if not splitter then - separator = lpeg.P(separator) + separator = P(separator) if single then - local other, any = lpeg.C((1 - separator)^0), lpeg.P(1) - splitter = other * (separator * lpeg.C(any^0) + "") + local other, any = C((1 - separator)^0), P(1) + splitter = other * (separator * C(any^0) + "") splitters_s[separator] = splitter else - local other = lpeg.C((1 - separator)^0) + local other = C((1 - separator)^0) splitter = other * (separator * other)^0 splitters_m[separator] = splitter end @@ -458,26 +361,43 @@ function lpeg.splitat(separator,single) return splitter end +lpeg.splitat = splitat + +local cache = { } + +function string:split(separator) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return c:match(self) +end + + +end -- of closure --- filename : l-table.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +do -- create closure to overcome 200 locals limit -if not versions then versions = { } end versions['l-table'] = 1.001 +if not modules then modules = { } end modules ['l-table'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format = string.format +local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump local getmetatable, setmetatable = getmetatable, setmetatable -local pairs, ipairs, type, next, tostring = pairs, ipairs, type, next, tostring +local type, next, tostring, ipairs = type, next, tostring, ipairs function table.strip(tab) local lst = { } for i=1,#tab do - local s = tab[i]:gsub("^%s*(.-)%s*$","%1") + local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -489,7 +409,7 @@ end local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key if kind == 3 then -- no further check @@ -516,7 +436,7 @@ end local function sortedhashkeys(tab) -- fast one local srt = { } - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key end sort(srt) @@ -526,14 +446,25 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys +function table.sortedpairs(t) + local s = sortedhashkeys(t) -- maybe just sortedkeys + local n = 0 + local function kv(s) + n = n + 1 + local k = s[n] + return k, t[k] + end + return kv, s +end + function table.append(t, list) - for _,v in pairs(list) do + for _,v in next, list do insert(t,v) end end function table.prepend(t, list) - for k,v in pairs(list) do + for k,v in next, list do insert(t,k,v) end end @@ -542,7 +473,7 @@ function table.merge(t, ...) -- first one is target t = t or {} local lst = {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do t[k] = v end end @@ -552,7 +483,7 @@ end function table.merged(...) local tmp, lst = { }, {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do tmp[k] = v end end @@ -584,13 +515,14 @@ end local function fastcopy(old) -- fast one if old then local new = { } - for k,v in pairs(old) do + for k,v in next, old do if type(v) == "table" then new[k] = fastcopy(v) -- was just table.copy else new[k] = v end end + -- optional second arg local mt = getmetatable(old) if mt then setmetatable(new,mt) @@ -607,7 +539,7 @@ local function copy(t, tables) -- taken from lua wiki, slightly adapted if not tables[t] then tables[t] = tcopy end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed + for i,v in next, t do -- brrr, what happens with sparse indexed if type(i) == "table" then if tables[i] then i = tables[i] @@ -640,7 +572,7 @@ function table.sub(t,i,j) end function table.replace(a,b) - for k,v in pairs(b) do + for k,v in next, b do a[k] = v end end @@ -662,16 +594,18 @@ end function table.tohash(t,value) local h = { } - if value == nil then value = true end - for _, v in pairs(t) do -- no ipairs here - h[v] = value + if t then + if value == nil then value = true end + for _, v in next, t do -- no ipairs here + h[v] = value + end end return h end function table.fromhash(t) local h = { } - for k, v in pairs(t) do -- no ipairs here + for k, v in next, t do -- no ipairs here if v then h[#h+1] = k end end return h @@ -695,24 +629,10 @@ local reserved = table.tohash { -- intercept a language flaw, no reserved words 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', } -local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - if hexify then - return ("[0x%04X]"):format(k) - else - return "["..k.."]" - end - elseif noquotes and not reserved[k] and k:find("^%a[%a%d%_]*$") then - return k - else - return '["'..k..'"]' - end -end - local function simple_table(t) if #t > 0 then local n = 0 - for _,v in pairs(t) do + for _,v in next, t do n = n + 1 end if n == #t then @@ -722,14 +642,14 @@ local function simple_table(t) local tv = type(v) if tv == "number" then if hexify then - tt[#tt+1] = ("0x%04X"):format(v) + tt[#tt+1] = format("0x%04X",v) else - tt[#tt+1] = tostring(v) + tt[#tt+1] = tostring(v) -- tostring not needed end elseif tv == "boolean" then tt[#tt+1] = tostring(v) elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) + tt[#tt+1] = format("%q",v) else tt = nil break @@ -741,51 +661,72 @@ local function simple_table(t) return nil end +-- Because this is a core function of mkiv I moved some function calls +-- inline. +-- +-- twice as fast in a test: +-- +-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " if indexed then - handle(("%s{"):format(depth)) + handle(format("%s{",depth)) elseif name then - handle(("%s%s={"):format(depth,key(name))) + --~ handle(format("%s%s={",depth,key(name))) + if type(name) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end else - handle(("%s{"):format(depth)) + handle(format("%s{",depth)) end end if root and next(root) then local first, last = nil, 0 -- #root cannot be trusted here if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (we need to quit at nil) + -- NOT: for k=1,#root do (we need to quit at nil) + for k,v in ipairs(root) do -- can we use next? if not first then first = k end last = last + 1 end end - --~ for _,k in pairs(sortedkeys(root)) do -- 1% faster: local sk = sortedkeys(root) for i=1,#sk do local k = sk[i] local v = root[k] + --~ if v == root then + -- circular + --~ else local t = type(v) if compact and first and type(k) == "number" and k >= first and k <= last then if t == "number" then if hexify then - handle(("%s 0x%04X,"):format(depth,v)) + handle(format("%s 0x%04X,",depth,v)) else - handle(("%s %s,"):format(depth,v)) + handle(format("%s %s,",depth,v)) end elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) + if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(format("%s %s,",depth,v)) else - handle(("%s %q,"):format(depth,v)) + handle(format("%s %q,",depth,v)) end elseif t == "table" then if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then + handle(format("%s {},",depth)) + elseif inline then -- and #t > 0 local st = simple_table(v) if st then - handle(("%s { %s },"):format(depth,concat(st,", "))) + handle(format("%s { %s },",depth,concat(st,", "))) else do_serialize(v,k,depth,level+1,true) end @@ -793,39 +734,102 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) + handle(format("%s %s,",depth,tostring(v))) elseif t == "function" then if functions then - handle(('%s loadstring(%q),'):format(depth,v:dump())) + handle(format('%s loadstring(%q),',depth,dump(v))) else - handle(('%s "function",'):format(depth)) + handle(format('%s "function",',depth)) end else - handle(("%s %q,"):format(depth,tostring(v))) + handle(format("%s %q,",depth,tostring(v))) end elseif k == "__p__" then -- parent if false then - handle(("%s __p__=nil,"):format(depth)) + handle(format("%s __p__=nil,",depth)) end elseif t == "number" then - if hexify then - handle(("%s %s=0x%04X,"):format(depth,key(k),v)) + --~ if hexify then + --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) + --~ else + --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ end + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end else - handle(("%s %s=%s,"):format(depth,key(k),v)) + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end end elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + --~ handle(format("%s %s=%s,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end else - handle(("%s %s=%q,"):format(depth,key(k),v)) + --~ handle(format("%s %s=%q,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) + else + handle(format("%s [%s]=%q,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end end elseif t == "table" then if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) + --~ handle(format("%s %s={},",depth,key(k))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end elseif inline then local st = simple_table(v) if st then - handle(("%s %s={ %s },"):format(depth,key(k),concat(st,", "))) + --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) + end else do_serialize(v,k,depth,level+1) end @@ -833,24 +837,58 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1) end elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%s,",depth,k,tostring(v))) + end elseif t == "function" then if functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),v:dump())) - else - handle(('%s %s="function",'):format(depth,key(k))) + --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) + else + handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) + else + handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) + end end else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%q,",depth,k,tostring(v))) + end end + --~ end end end if level > 0 then - handle(("%s},"):format(depth)) + handle(format("%s},",depth)) end end +-- replacing handle by a direct t[#t+1] = ... (plus test) is not much +-- faster (0.03 on 1.00 for zapfino.tma) + local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) noquotes = _noquotes hexify = _hexify @@ -868,7 +906,7 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) end elseif tname == "number" then if hexify then - handle(("[0x%04X]={"):format(name)) + handle(format("[0x%04X]={",name)) else handle("[" .. name .. "]={") end @@ -1019,14 +1057,18 @@ function table.insert_after_value(t,value,str) end end -function table.are_equal(a,b,n,m) +local function are_equal(a,b,n,m) -- indexed if #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] - if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then - -- continue + if ai==bi then + -- same + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end else return false end @@ -1037,9 +1079,30 @@ function table.are_equal(a,b,n,m) end end +local function identical(a,b) -- assumes same structure + for ka, va in next, a do + local vb = b[k] + if va == vb then + -- same + elseif type(va) == "table" and type(vb) == "table" then + if not identical(va,vb) then + return false + end + else + return false + end + end + return true +end + +table.are_equal = are_equal +table.identical = identical + +-- maybe also make a combined one + function table.compact(t) if t then - for k,v in pairs(t) do + for k,v in next, t do if not next(v) then t[k] = nil end @@ -1068,7 +1131,7 @@ end function table.swapped(t) local s = { } - for k, v in pairs(t) do + for k, v in next, t do s[v] = k end return s @@ -1090,14 +1153,14 @@ end function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = ("0x%04X"):format(t[i]) end + for i=1,#t do tt[i] = format("0x%04X",t[i]) end return concat(tt,seperator or " ") end function table.reverse_hash(h) local r = { } - for k,v in pairs(h) do - r[v] = (k:gsub(" ","")):lower() + for k,v in next, h do + r[v] = lower(gsub(k," ","")) end return r end @@ -1112,14 +1175,36 @@ function table.reverse(t) return tt end +--~ function table.keys(t) +--~ local k = { } +--~ for k,_ in next, t do +--~ k[#k+1] = k +--~ end +--~ return k +--~ end + +--~ function table.keys_as_string(t) +--~ local k = { } +--~ for k,_ in next, t do +--~ k[#k+1] = k +--~ end +--~ return concat(k,"") +--~ end + + +end -- of closure --- filename : l-io.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-io'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-io'] = 1.001 +local byte = string.byte if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1127,8 +1212,8 @@ else io.fileseparator, io.pathseparator = "/" , ":" end -function io.loaddata(filename) - local f = io.open(filename,'rb') +function io.loaddata(filename,textmode) + local f = io.open(filename,(textmode and 'r') or 'rb') if f then local data = f:read('*all') -- garbagecollector.check(data) @@ -1186,146 +1271,83 @@ function io.noflines(f) return n end -do - - local sb = string.byte - - local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end - } - - function io.characters(f,n) - if f then - return nextchar[n or 1], f - else - return nil, nil - end +local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a end +} +function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end end -do - - local sb = string.byte - ---~ local nextbyte = { ---~ [4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(a), sb(b), sb(c), sb(d) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end, ---~ [2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(a), sb(b) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [1] = function (f) ---~ local a = f:read(1) ---~ if a then ---~ return sb(a) ---~ else ---~ return nil ---~ end ---~ end, ---~ [-2] = function (f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(b), sb(a) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(d), sb(c), sb(b), sb(a) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end ---~ } - - local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(a), sb(b), sb(c), sb(d) - else - return nil, nil, nil, nil - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return sb(a), sb(b) - else - return nil, nil - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return sb(a) - else - return nil - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return sb(b), sb(a) - else - return nil, nil - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(d), sb(c), sb(b), sb(a) - else - return nil, nil, nil, nil - end +local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(a), byte(b), byte(c), byte(d) + else + return nil, nil, nil, nil end - } - - function io.bytes(f,n) - if f then - return nextbyte[n or 1], f + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return byte(a), byte(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return byte(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return byte(b), byte(a) else return nil, nil end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(d), byte(c), byte(b), byte(a) + else + return nil, nil, nil, nil + end end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end end function io.ask(question,default,options) @@ -1361,24 +1383,28 @@ function io.ask(question,default,options) end --- filename : l-number.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-number'] = 1.001 +do -- create closure to overcome 200 locals limit -if not number then number = { } end - --- a,b,c,d,e,f = number.toset(100101) - -function number.toset(n) - return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") -end +if not modules then modules = { } end modules ['l-number'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} local format = string.format +number = number or { } + +-- a,b,c,d,e,f = number.toset(100101) + +function number.toset(n) + return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") +end + function number.toevenhex(n) local s = format("%X",n) if #s % 2 == 0 then @@ -1399,72 +1425,72 @@ end -- -- of course dedicated "(.)(.)(.)(.)" matches are even faster -do - local one = lpeg.C(1-lpeg.S(''))^1 +local one = lpeg.C(1-lpeg.S(''))^1 - function number.toset(n) - return one:match(tostring(n)) - end +function number.toset(n) + return one:match(tostring(n)) end --- filename : l-set.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-set'] = 1.001 +end -- of closure + +do -- create closure to overcome 200 locals limit -if not set then set = { } end +if not modules then modules = { } end modules ['l-set'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -do +set = set or { } - local nums = { } - local tabs = { } - local concat = table.concat +local nums = { } +local tabs = { } +local concat = table.concat - set.create = table.tohash +set.create = table.tohash - function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in pairs(t) do - if v then - -- why bother about the leading space - s = s .. " " .. k - end +function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k end - if not nums[s] then - tabs[#tabs+1] = t - nums[s] = #tabs - end - return nums[s] - else - return 0 end - end - - function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs end + return nums[s] + else + return 0 end +end - function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] - end +function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } end +end +function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end end --~ local c = set.create{'aap','noot','mies'} @@ -1481,16 +1507,19 @@ end --- filename : l-os.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure +do -- create closure to overcome 200 locals limit ---~ print(table.serialize(os.uname())) +if not modules then modules = { } end modules ['l-os'] = { + version = 1.001, + comment = "companion to luat-lub.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-os'] = 1.001 +local find = string.find function os.resultof(command) return io.popen(command,"r"):read("*all") @@ -1503,7 +1532,7 @@ if not os.spawn then os.spawn = os.execute end --~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then + if find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" else io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" @@ -1541,11 +1570,10 @@ end os.gettimeofday = os.gettimeofday or os.clock -do - local startuptime = os.gettimeofday() - function os.runtime() - return os.gettimeofday() - startuptime - end +local startuptime = os.gettimeofday() + +function os.runtime() + return os.gettimeofday() - startuptime end --~ print(os.gettimeofday()-os.time()) @@ -1554,47 +1582,92 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) +os.arch = os.arch or function() + local a = os.resultof("uname -m") or "linux" + os.arch = function() + return a + end + return a +end --- filename : l-md5.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-md5'] = 1.001 - -if md5 then do +local platform - local function convert(str,fmt) - return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) +function os.currentplatform(name,default) + if not platform then + local name = os.name or os.platform or name -- os.name is built in, os.platform is mine + if not name then + platform = default or "linux" + elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then + if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then + platform = "mswin-64" + else + platform = "mswin" + end + else + local architecture = os.arch() + if name == "linux" then + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + elseif name == "macosx" then + if find(architecture,"i386") then + platform = "osx-intel" + else + platform = "osx-ppc" + end + elseif name == "sunos" then + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + elseif name == "freebsd" then + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + else + platform = default or name + end + end + function os.currentplatform() + return platform + end end + return platform +end - if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end - if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end - if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end -end end +end -- of closure +do -- create closure to overcome 200 locals limit --- filename : l-file.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +if not modules then modules = { } end modules ['l-file'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-file'] = 1.001 +-- needs a cleanup -if not file then file = { } end +file = file or { } local concat = table.concat +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub function file.removesuffix(filename) - return (filename:gsub("%.[%a%d]+$","")) + return (gsub(filename,"%.[%a%d]+$","")) end -file.stripsuffix = file.removesuffix - function file.addsuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then + if not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1602,23 +1675,23 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - return (filename:gsub("%.[%a%d]+$","")) .. "." .. suffix + return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end function file.dirname(name) - return name:match("^(.+)[/\\].-$") or "" + return match(name,"^(.+)[/\\].-$") or "" end function file.basename(name) - return name:match("^.+[/\\](.-)$") or name + return match(name,"^.+[/\\](.-)$") or name end function file.nameonly(name) - return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) + return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end function file.extname(name) - return name:match("^.+%.([^/\\]-)$") or "" + return match(name,"^.+%.([^/\\]-)$") or "" end file.suffix = file.extname @@ -1631,66 +1704,47 @@ file.suffix = file.extname function file.join(...) local pth = concat({...},"/") - pth = pth:gsub("\\","/") - local a, b = pth:match("^(.*://)(.*)$") + pth = gsub(pth,"\\","/") + local a, b = match(pth,"^(.*://)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - a, b = pth:match("^(//)(.*)$") + a, b = match(pth,"^(//)(.*)$") if a and b then - return a .. b:gsub("//+","/") - end - return (pth:gsub("//+","/")) -end - -function file.is_writable(name) - local f = io.open(name, 'w') - if f then - f:close() - return true - else - return false + return a .. gsub(b,"//+","/") end + return (gsub(pth,"//+","/")) end -function file.is_readable(name) - local f = io.open(name,'r') - if f then - f:close() +function file.iswritable(name) + local a = lfs.attributes(name) + if a and a.permissions:sub(2,2) == "w" then return true else - return false + name = file.dirname(name) or "." + if name == "" then name = "." end + a = lfs.attributes(name) + return a and a.permissions:sub(2,2) == "w" end end -function file.iswritable(name) - local a = lfs.attributes(name) - return a and a.permissions:sub(2,2) == "w" -end - function file.isreadable(name) local a = lfs.attributes(name) return a and a.permissions:sub(1,1) == "r" end ---~ function file.split_path(str) ---~ if str:find(';') then ---~ return str:splitchr(";") ---~ else ---~ return str:splitchr(io.pathseparator) ---~ end ---~ end +file.is_readable = file.isreadable +file.is_writable = file.iswritable -- todo: lpeg function file.split_path(str) local t = { } - str = str:gsub("\\", "/") - str = str:gsub("(%a):([;/])", "%1\001%2") - for name in str:gmatch("([^;:]+)") do + str = gsub(str,"\\", "/") + str = gsub(str,"(%a):([;/])", "%1\001%2") + for name in gmatch(str,"([^;:]+)") do if name ~= "" then - name = name:gsub("\001",":") - t[#t+1] = name + t[#t+1] = gsub(name,"\001",":") end end return t @@ -1701,15 +1755,15 @@ function file.join_path(tab) end function file.collapse_path(str) - str = str:gsub("/%./","/") + str = gsub(str,"/%./","/") local n, m = 1, 1 while n > 0 or m > 0 do - str, n = str:gsub("[^/%.]+/%.%.$","") - str, m = str:gsub("[^/%.]+/%.%./","") + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") end - str = str:gsub("([^/])/$","%1") - str = str:gsub("^%./","") - str = str:gsub("/%.$","") + str = gsub(str,"([^/])/$","%1") + str = gsub(str,"^%./","") + str = gsub(str,"/%.$","") if str == "" then str = "." end return str end @@ -1722,7 +1776,7 @@ end --~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) - return (str:gsub("[^%a%d%/%-%.\\]+","-")) + return (gsub(str,"[^%a%d%/%-%.\\]+","-")) end file.readdata = io.loaddata @@ -1752,8 +1806,6 @@ end --~ return pattern:match(name) --~ end ---~ file.stripsuffix = file.removesuffix - --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) @@ -1807,7 +1859,6 @@ end --~ end --~ local test = file.extname ---~ local test = file.stripsuffix --~ local test = file.basename --~ local test = file.dirname --~ local test = file.addsuffix @@ -1824,14 +1875,117 @@ end --~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) +-- also rewrite previous + +local letter = lpeg.R("az","AZ") + lpeg.S("_-+") +local separator = lpeg.P("://") + +local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") +local rootbased = lpeg.P("/") + letter*lpeg.P(":") + +-- ./name ../name /name c: :// name/name + +function file.is_qualified_path(filename) + return qualified:match(filename) +end + +function file.is_rootbased_path(filename) + return rootbased:match(filename) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-md5'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This also provides file checksums and checkers. + +local gsub, format, byte = string.gsub, string.format, string.byte + +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end + +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +--~ if not md5.HEX then +--~ local function remap(chr) return format("%02X",byte(chr)) end +--~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.hex then +--~ local function remap(chr) return format("%02x",byte(chr)) end +--~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.dec then +--~ local function remap(chr) return format("%03i",byte(chr)) end +--~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end +--~ end + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.checksum(name) + if md5 then + local data = io.loaddata(name) + if data then + return md5.HEXsum(data) + end + end + return nil +end + +function file.loadchecksum(name) + if md5 then + local data = io.loaddata(name .. ".md5") + return data and data:gsub("%s","") + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.checksum(name) end + if checksum then + io.savedata(name .. ".md5",checksum) + return checksum + end + return nil +end + + +end -- of closure --- filename : l-url.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-url'] = 1.001 -if not url then url = { } end +local char, gmatch = string.char, string.gmatch +local tonumber, type = tonumber, type -- from the spec (on the web): -- @@ -1843,29 +1997,28 @@ if not url then url = { } end -- / \ / \ -- urn:example:animal:ferret:nose -do - - local function tochar(s) - return string.char(tonumber(s,16)) - end +url = url or { } - local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) +local function tochar(s) + return char(tonumber(s,16)) +end - local hexdigit = lpeg.R("09","AF","af") - local escaped = percent * lpeg.C(hexdigit * hexdigit) / tochar +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) - local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") - local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") - local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") - local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") - local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) - local parser = lpeg.Ct(scheme * authority * path * query * fragment) +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") - function url.split(str) - return (type(str) == "string" and parser:match(str)) or str - end +local parser = lpeg.Ct(scheme * authority * path * query * fragment) +function url.split(str) + return (type(str) == "string" and parser:match(str)) or str end function url.hashed(str) @@ -1888,7 +2041,7 @@ end function url.query(str) if type(str) == "string" then local t = { } - for k, v in str:gmatch("([^&=]*)=([^&=]*)") do + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do t[k] = v end return t @@ -1903,12 +2056,12 @@ end --~ print(url.filename("file:///etc/test.txt")) --~ print(url.filename("/oeps.txt")) --- from the spec on the web (sort of): +--~ from the spec on the web (sort of): --~ --~ function test(str) --~ print(table.serialize(url.hashed(str))) --~ end ----~ +--~ --~ test("%56pass%20words") --~ test("file:///c:/oeps.txt") --~ test("file:///c|/oeps.txt") @@ -1930,205 +2083,210 @@ end --~ test("zip:///oeps/oeps.zip?bla/bla.tex") --- filename : l-dir.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-dir'] = 1.001 +do -- create closure to overcome 200 locals limit -dir = { } +if not modules then modules = { } end modules ['l-dir'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --- optimizing for no string.find (*) does not save time +local type = type +local find, gmatch = string.find, string.gmatch -if lfs then do +dir = dir or { } - local attributes = lfs.attributes - local walkdir = lfs.dir +-- optimizing for no string.find (*) does not save time - local function glob_pattern(path,patt,recurse,action) - local ok, scanner - if path == "/" then - ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe - end - if ok and type(scanner) == "function" then - if not path:find("/$") then path = path .. '/' end - for name in scanner do - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if full:find(patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - glob_pattern(full,patt,recurse,action) +local attributes = lfs.attributes +local walkdir = lfs.dir + +local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if find(full,patt) then + action(full) end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) end end end +end - dir.glob_pattern = glob_pattern +dir.glob_pattern = glob_pattern - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V +local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) - } +local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) +} - local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) - )^0 ) - - local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for _, s in ipairs(str) do - glob(s,t) - end - return t - elseif lfs.isfile(str) then +local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) +)^0 ) + +local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t + else + local split = pattern:match(str) + if split then local t = t or { } - t[#t+1] = str + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) return t else - local split = pattern:match(str) - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = base:find("%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) - return t - else - return { } - end + return { } end end +end - dir.glob = glob +dir.glob = glob - --~ list = dir.glob("**/*.tif") - --~ list = dir.glob("/**/*.tif") - --~ list = dir.glob("./**/*.tif") - --~ list = dir.glob("oeps/**/*.tif") - --~ list = dir.glob("/oeps/**/*.tif") +--~ list = dir.glob("**/*.tif") +--~ list = dir.glob("/**/*.tif") +--~ list = dir.glob("./**/*.tif") +--~ list = dir.glob("oeps/**/*.tif") +--~ list = dir.glob("/oeps/**/*.tif") - local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func -- alas, we need this indirect way - func = function(name) return name:find(s) end - end - files = files or { } - for name in walkdir(path) do - if name:find("^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if func then - if func(name) then - files[#files+1] = path .. "/" .. name - end - else +local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return find(name,s) end + end + files = files or { } + for name in walkdir(path) do + if find(name,"^%.") then + --- skip + else + local mode = attributes(name,'mode') + if mode == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif mode == "file" then + if func then + if func(name) then files[#files+1] = path .. "/" .. name end + else + files[#files+1] = path .. "/" .. name end end end - return files end + return files +end - dir.globfiles = globfiles +dir.globfiles = globfiles - -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") - -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") - -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") - -- t = dir.glob("f:/minimal/tex/**/*") - -- print(dir.ls("f:/minimal/tex/**/*")) - -- print(dir.ls("*.tex")) +-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") +-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") +-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") +-- t = dir.glob("f:/minimal/tex/**/*") +-- print(dir.ls("f:/minimal/tex/**/*")) +-- print(dir.ls("*.tex")) - function dir.ls(pattern) - return table.concat(glob(pattern),"\n") - end +function dir.ls(pattern) + return table.concat(glob(pattern),"\n") +end - --~ mkdirs("temp") - --~ mkdirs("a/b/c") - --~ mkdirs(".","/a/b/c") - --~ mkdirs("a","b","c") +--~ mkdirs("temp") +--~ mkdirs("a/b/c") +--~ mkdirs(".","/a/b/c") +--~ mkdirs("a","b","c") - local make_indeed = true -- false +local make_indeed = true -- false - if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - local first, middle, last - local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") if first then - -- empty network path == local path + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end else - first, last = str:match("^(//)/*(.-)$") + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end + pth, drive = first .. middle, true else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = str:match("^(/*)(.-)$") - if not middle then - last = str - end + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str end end end - for s in last:gmatch("[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + for s in gmatch(last,"[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2142,79 +2300,79 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") - if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") - end - if not first then - first, last = str:match("^(//)/*(.*)$") - end - if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") - if first and not last:find("^/") then - local d = lfs.currentdir() - if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") - end - lfs.chdir(d) + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") end + lfs.chdir(d) end - if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") - end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") - if last == "" then - return first - else - return first .. "/" .. last - end end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end - else +else - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - str = str:gsub("/+","/") - if str:find("^/") then - pth = "/" - for s in str:gmatch("[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not lfs.isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in str:gmatch("[^/]+") do + end + str = str:gsub("/+","/") + if find(str,"^/") then + pth = "/" + for s in gmatch(str,"[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else pth = pth .. "/" .. s - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in gmatch(str,"[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2224,30 +2382,35 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - if not str:find("^/") then - str = lfs.currentdir() .. "/" .. str - end - str = str:gsub("//","/") - str = str:gsub("/%./","/") - return str + function dir.expand_name(str) + if not find(str,"^/") then + str = lfs.currentdir() .. "/" .. str end - + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str end - dir.makedirs = dir.mkdirs +end + +dir.makedirs = dir.mkdirs + -end end +end -- of closure +do -- create closure to overcome 200 locals limit --- filename : l-boolean.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +if not modules then modules = { } end modules ['l-boolean'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-boolean'] = 1.001 -if not boolean then boolean = { } end +boolean = boolean or { } + +local type, tonumber = type, tonumber function boolean.tonumber(b) if b then return 1 else return 0 end @@ -2294,24 +2457,24 @@ function boolean.falsetrue() end --- filename : l-unicode.lua --- comment : split off from luat-inp --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit -if not versions then versions = { } end versions['l-unicode'] = 1.001 -if not unicode then unicode = { } end +if not modules then modules = { } end modules ['l-unicode'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -local concat, utfchar, utfgsub = table.concat, unicode.utf8.char, unicode.utf8.gsub -local char, byte = string.char, string.byte +utf = utf or unicode.utf8 -if not garbagecollector then - garbagecollector = { - push = function() collectgarbage("stop") end, - pop = function() collectgarbage("restart") end, - } -end +local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub +local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs + +unicode = unicode or { } -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian @@ -2332,17 +2495,17 @@ function unicode.utftype(f) -- \000 fails ! if not str then f:seek('set') return 0 - elseif str:find("^%z%z\254\255") then + elseif find(str,"^%z%z\254\255") then return 4 - elseif str:find("^\255\254%z%z") then + elseif find(str,"^\255\254%z%z") then return 3 - elseif str:find("^\254\255") then + elseif find(str,"^\254\255") then f:seek('set',2) return 2 - elseif str:find("^\255\254") then + elseif find(str,"^\255\254") then f:seek('set',2) return 1 - elseif str:find("^\239\187\191") then + elseif find(str,"^\239\187\191") then f:seek('set',3) return 0 else @@ -2352,18 +2515,17 @@ function unicode.utftype(f) -- \000 fails ! end function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg ---~ garbagecollector.push() local result, tmp, n, m, p = { }, { }, 0, 0, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = n else @@ -2371,7 +2533,7 @@ function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg p = 0 end end - for l,r in str:bytepairs() do + for l,r in bytepairs(str) do if r then if endian then n = l*256 + r @@ -2390,26 +2552,24 @@ function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg end end if #tmp > 0 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) end ---~ garbagecollector.pop() return result end function unicode.utf32_to_utf8(str, endian) ---~ garbagecollector.push() local result = { } local tmp, n, m, p = { }, 0, -1, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = n else @@ -2417,7 +2577,7 @@ function unicode.utf32_to_utf8(str, endian) p = 0 end end - for a,b in str:bytepairs() do + for a,b in bytepairs(str) do if a and b then if m < 0 then if endian then @@ -2439,48 +2599,55 @@ function unicode.utf32_to_utf8(str, endian) end end if #tmp > 0 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) end ---~ garbagecollector.pop() return result end +local function little(c) + local b = byte(c) -- b = c:byte() + if b < 0x10000 then + return char(b%256,b/256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end + +local function big(c) + local b = byte(c) + if b < 0x10000 then + return char(b/256,b%256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end + function unicode.utf8_to_utf16(str,littleendian) if littleendian then - return char(255,254) .. utfgsub(str,".",function(c) - local b = byte(c) - if b < 0x10000 then - return char(b%256,b/256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1%256,b1/256,b2%256,b2/256) - end - end) + return char(255,254) .. utfgsub(str,".",little) else - return char(254,255) .. utfgsub(str,".",function(c) - local b = byte(c) - if b < 0x10000 then - return char(b/256,b%256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1/256,b1%256,b2/256,b2%256) - end - end) + return char(254,255) .. utfgsub(str,".",big) end end --- filename : l-math.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit -if not versions then versions = { } end versions['l-math'] = 1.001 +if not modules then modules = { } end modules ['l-math'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -local floor = math.floor +local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan if not math.round then function math.round(x) @@ -2500,14 +2667,34 @@ if not math.mod then end end +local pipi = 2*math.pi/360 + +function math.sind(d) + return sin(d*pipi) +end + +function math.cosd(d) + return cos(d*pipi) +end + +function math.tand(d) + return tan(d*pipi) +end + + +end -- of closure --- filename : l-utils.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-utils'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-utils'] = 1.001 +-- hm, quite unreadable if not utils then utils = { } end if not utils.merger then utils.merger = { } end @@ -2572,24 +2759,52 @@ function utils.merger._self_swap_(data,code) end end +--~ stripper: +--~ +--~ data = string.gsub(data,"%-%-~[^\n]*\n","") +--~ data = string.gsub(data,"\n\n+","\n") + function utils.merger._self_libs_(libs,list) - local result, f = { }, nil + local result, f, frozen = { }, nil, false + result[#result+1] = "\n" if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end + local foundpath = nil for _, lib in ipairs(libs) do for _, pth in ipairs(list) do - local name = string.gsub(pth .. "/" .. lib,"\\","/") - f = io.open(name) - if f then - utils.report("merging library %s",name) - result[#result+1] = f:read("*all") - f:close() - list = { pth } -- speed up the search - break + pth = string.gsub(pth,"\\","/") -- file.clean_path + utils.report("checking library path %s",pth) + local name = pth .. "/" .. lib + if lfs.isfile(name) then + foundpath = pth + end + end + if foundpath then break end + end + if foundpath then + utils.report("using library path %s",foundpath) + local right, wrong = { }, { } + for _, lib in ipairs(libs) do + local fullname = foundpath .. "/" .. lib + if lfs.isfile(fullname) then + -- right[#right+1] = lib + utils.report("merging library %s",fullname) + result[#result+1] = "do -- create closure to overcome 200 locals limit" + result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = "end -- of closure" else - utils.report("no library %s",name) + -- wrong[#wrong+1] = lib + utils.report("no library %s",fullname) end end + if #right > 0 then + utils.report("merged libraries: %s",table.concat(right," ")) + end + if #wrong > 0 then + utils.report("skipped libraries: %s",table.concat(wrong," ")) + end + else + utils.report("no valid library path found") end return table.concat(result, "\n\n") end @@ -2643,15 +2858,257 @@ end -if not modules then modules = { } end modules ['luat-lib'] = { +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { version = 1.001, + comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", + license = "see context related readme files" +} + +-- the tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +debugger = debugger or { } + +local counters = { } +local names = { } +local getinfo = debug.getinfo +local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in pairs(counters) do + if count > threshold then + local name = getname(func) + if not name:find("for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in pairs(counters) do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +trackers = trackers or { } + +local data, done = { }, { } + +local function set(what,value) + for w in gmatch(lower(what),"[^, ]+") do + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](value) + end + end + end + end +end + +local function reset() + for d, f in next, data do + for i=1,#f do + f[i](false) + end + end +end + +function trackers.register(what,...) + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(fnc,value,nesting) end + end + end +end + +function trackers.enable(what) + done = { } + set(what,true) +end + +function trackers.disable(what) + done = { } + if not what or what == "" then + trackers.reset(what) + else + set(what,false) + end +end + +function trackers.reset(what) + done = { } + reset() +end + +function trackers.list() -- pattern + local list = table.sortedkeys(data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-env'] = { + version = 1.001, comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" } --- most code already moved to the l-*.lua and other luat-*.lua files +-- A former version provided functionality for non embeded core +-- scripts i.e. runtime library loading. Given the amount of +-- Lua code we use now, this no longer makes sense. Much of this +-- evolved before bytecode arrays were available and so a lot of +-- code has disappeared already. + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + +local format = string.format + +-- precautions os.setlocale(nil,nil) -- useless feature and even dangerous in luatex @@ -2659,15 +3116,27 @@ function os.setlocale() -- no way you can mess with it end +-- dirty tricks + if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end +if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then + profiler.start("luatex-profile.log") +end + +-- environment + environment = environment or { } environment.arguments = { } environment.files = { } environment.sortedflags = nil +if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end +if not environment.version or environment.version == "" then environment.version = "unknown" end +if not environment.jobname then environment.jobname = "unknown" end + function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil @@ -2689,24 +3158,20 @@ function environment.initialize_arguments(arg) environment.ownname = environment.ownname or arg[0] or 'unknown.lua' end -function environment.showarguments() - for k,v in pairs(environment.arguments) do - print(k .. " : " .. tostring(v)) - end - if #environment.files > 0 then - print("files : " .. table.concat(environment.files, " ")) - end -end - function environment.setargument(name,value) environment.arguments[name] = value end -function environment.argument(name) -- todo: default (plus typecheck on default) +-- todo: defaults, better checks e.g on type (boolean versus string) +-- +-- tricky: too many hits when we support partials unless we add +-- a registration of arguments so from now on we have 'partial' + +function environment.argument(name,partial) local arguments, sortedflags = environment.arguments, environment.sortedflags if arguments[name] then return arguments[name] - else + elseif partial then if not sortedflags then sortedflags = { } for _,v in pairs(table.sortedkeys(arguments)) do @@ -2714,6 +3179,7 @@ function environment.argument(name) -- todo: default (plus typecheck on default) end environment.sortedflags = sortedflags end + -- example of potential clash: ^mode ^modefile for _,v in ipairs(sortedflags) do if name:find(v) then return arguments[v:sub(2,#v)] @@ -2737,43 +3203,17 @@ function environment.split_arguments(separator) -- rather special, cut-off befor return before, after end ---~ function environment.reconstruct_commandline(arg) ---~ if not arg then arg = environment.original_arguments end ---~ local result = { } ---~ for _,a in ipairs(arg) do -- ipairs 1 .. #n ---~ local kk, vv = a:match("^(%-+.-)=(.+)$") ---~ if kk and vv then ---~ if vv:find(" ") then ---~ vv = vv:unquote() ---~ vv = vv:gsub('"','\\"') ---~ result[#result+1] = kk .. "=" .. vv:quote() ---~ else ---~ a = a:unquote() ---~ a = a:gsub('"','\\"') ---~ result[#result+1] = a ---~ end ---~ elseif a:find(" ") then ---~ a = a:unquote() ---~ a = a:gsub('"','\\"') ---~ result[#result+1] = a:quote() ---~ else ---~ result[#result+1] = a ---~ end ---~ end ---~ return table.join(result," ") ---~ end - function environment.reconstruct_commandline(arg,noquote) - if not arg then arg = environment.original_arguments end + arg = arg or environment.original_arguments if noquote and #arg == 1 then local a = arg[1] - a = input.resolve(a) + a = resolvers.resolve(a) a = a:unquote() return a - elseif #arg == 1 then + elseif next(arg) then local result = { } for _,a in ipairs(arg) do -- ipairs 1 .. #n - a = input.resolve(a) + a = resolvers.resolve(a) a = a:unquote() a = a:gsub('"','\\"') -- tricky if a:find(" ") then @@ -2783,6 +3223,8 @@ function environment.reconstruct_commandline(arg,noquote) end end return table.join(result," ") + else + return "" end end @@ -2818,279 +3260,163 @@ if arg then end +-- weird place ... depends on a not yet loaded module -if not modules then modules = { } end modules ['luat-inp'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", - comment = "companion to luat-lib.tex", -} - --- TODO: os.getenv -> os.env[] --- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) --- TODO: check escaping in find etc, too much, too slow +function environment.texfile(filename) + return resolvers.find_file(filename,'tex') +end --- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this --- module in components once we're done with prototyping. This is the --- first code I wrote for LuaTeX, so it needs some cleanup. Before changing --- something in this module one can best check with Taco or Hans first; there --- is some nasty trickery going on that relates to traditional kpse support. +function environment.luafile(filename) + local resolved = resolvers.find_file(filename,'tex') or "" + if resolved ~= "" then + return resolved + end + resolved = resolvers.find_file(filename,'texmfscripts') or "" + if resolved ~= "" then + return resolved + end + return resolvers.find_file(filename,'luatexlibs') or "" +end --- To be considered: hash key lowercase, first entry in table filename --- (any case), rest paths (so no need for optimization). Or maybe a --- separate table that matches lowercase names to mixed case when --- present. In that case the lower() cases can go away. I will do that --- only when we run into problems with names ... well ... Iwona-Regular. +environment.loadedluacode = loadfile -- can be overloaded --- Beware, loading and saving is overloaded in luat-tmp! +--~ function environment.loadedluacode(name) +--~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then +--~ local chunk = loadstring(io.loaddata("texluac.luc")) +--~ os.remove("texluac.luc") +--~ return chunk +--~ else +--~ environment.loadedluacode = loadfile -- can be overloaded +--~ return loadfile(name) +--~ end +--~ end -if not input then input = { } end -if not input.suffixes then input.suffixes = { } end -if not input.formats then input.formats = { } end -if not input.aux then input.aux = { } end - -if not input.suffixmap then input.suffixmap = { } end - -if not input.locators then input.locators = { } end -- locate databases -if not input.hashers then input.hashers = { } end -- load databases -if not input.generators then input.generators = { } end -- generate databases -if not input.filters then input.filters = { } end -- conversion filters - -local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys - -input.locators.notfound = { nil } -input.hashers.notfound = { nil } -input.generators.notfound = { nil } - -input.cacheversion = '1.0.1' -input.banner = nil -input.verbose = false -input.debug = false -input.cnfname = 'texmf.cnf' -input.luaname = 'texmfcnf.lua' -input.lsrname = 'ls-R' -input.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' - ---~ input.luasuffix = 'tma' ---~ input.lucsuffix = 'tmc' - --- for the moment we have .local but this will disappear -input.cnfdefault = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - --- chances are low that the cnf file is in the bin path -input.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - --- we use a cleaned up list / format=any is a wildcard, as is *name - -input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } -input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } -input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } -input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } -input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } -input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } -input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } -input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' -input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } -input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } -input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } -input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } -input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } -input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } -input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } -input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } -input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } - -input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } -input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } - -input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new -input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' - -input.formats ['lua'] = 'LUAINPUTS' -- new -input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } +function environment.luafilechunk(filename) -- used for loading lua bytecode in the format + filename = file.replacesuffix(filename, "lua") + local fullname = environment.luafile(filename) + if fullname and fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading file %s", fullname) + end + return environment.loadedluacode(fullname) + else + if trace_verbose then + logs.report("fileio","unknown file %s", filename) + end + return nil + end +end --- here we catch a few new thingies (todo: add these paths to context.tmf) --- --- FONTFEATURES = .;$TEXMF/fonts/fea// --- FONTCIDMAPS = .;$TEXMF/fonts/cid// +-- the next ones can use the previous ones / combine -function input.checkconfigdata() -- not yet ok, no time for debugging now - local instance = input.instance - local function fix(varname,default) - local proname = varname .. "." .. instance.progname or "crap" - local p = instance.environment[proname] - local v = instance.environment[varname] - if not ((p and p ~= "") or (v and v ~= "")) then - instance.variables[varname] = default -- or environment? +function environment.loadluafile(filename, version) + local lucname, luaname, chunk + local basename = file.removesuffix(filename) + if basename == filename then + lucname, luaname = basename .. ".luc", basename .. ".lua" + else + lucname, luaname = nil, basename -- forced suffix + end + -- when not overloaded by explicit suffix we look for a luc file first + local fullname = (lucname and environment.luafile(lucname)) or "" + if fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + end + if chunk then + assert(chunk)() + if version then + -- we check of the version number of this chunk matches + local v = version -- can be nil + if modules and modules[filename] then + v = modules[filename].version -- new method + elseif versions and versions[filename] then + v = versions[filename] -- old method + end + if v == version then + return true + else + if trace_verbose then + logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) + end + environment.loadluafile(filename) + end + else + return true end end - local name = os.name - if name == "windows" then - fix("OSFONTDIR", "c:/windows/fonts//") - elseif name == "macosx" then - fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") - else - -- bad luck + fullname = (luaname and environment.luafile(luaname)) or "" + if fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + if not chunk then + if verbose then + logs.report("fileio","unknown file %s", filename) + end + else + assert(chunk)() + return true + end end - fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm - fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + return false end --- backward compatible ones -input.alternatives = { } +end -- of closure -input.alternatives['map files'] = 'map' -input.alternatives['enc files'] = 'enc' -input.alternatives['cid files'] = 'cid' -input.alternatives['fea files'] = 'fea' -input.alternatives['opentype fonts'] = 'otf' -input.alternatives['truetype fonts'] = 'ttf' -input.alternatives['truetype collections'] = 'ttc' -input.alternatives['type1 fonts'] = 'pfb' +do -- create closure to overcome 200 locals limit --- obscure ones +if not modules then modules = { } end modules ['trac-inf'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -input.formats ['misc fonts'] = '' -input.suffixes['misc fonts'] = { } +local format = string.format -input.formats ['sfd'] = 'SFDFONTS' -input.suffixes ['sfd'] = { 'sfd' } -input.alternatives['subfont definition files'] = 'sfd' +local statusinfo, n, registered = { }, 0, { } --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical pusposes we now avoid this and use a --- instance variable. - -function input.newinstance() - - local instance = { } - - instance.rootpath = '' - instance.treepath = '' - instance.progname = 'context' - instance.engine = 'luatex' - instance.format = '' - instance.environment = { } - instance.variables = { } - instance.expansions = { } - instance.files = { } - instance.remap = { } - instance.configuration = { } - instance.setup = { } - instance.order = { } - instance.found = { } - instance.foundintrees = { } - instance.kpsevars = { } - instance.hashes = { } - instance.cnffiles = { } - instance.luafiles = { } - instance.lists = { } - instance.remember = true - instance.diskcache = true - instance.renewcache = false - instance.scandisk = true - instance.cachepath = nil - instance.loaderror = false - instance.smallcache = false - instance.sortdata = false - instance.savelists = true - instance.cleanuppaths = true - instance.allresults = false - instance.pattern = nil -- lists - instance.kpseonly = false -- lists - instance.loadtime = 0 - instance.starttime = 0 - instance.stoptime = 0 - instance.validfile = function(path,name) return true end - instance.data = { } -- only for loading - instance.force_suffixes = true - instance.dummy_path_expr = "^!*unset/*$" - instance.fakepaths = { } - instance.lsrmode = false - - -- store once, freeze and faster (once reset we can best use instance.environment) - - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) - end - - -- cross referencing, delayed because we can add suffixes - - for k, v in pairs(input.suffixes) do - for _, vv in pairs(v) do - if vv then - input.suffixmap[vv] = k - end - end - end - - return instance - -end - -input.instance = input.instance or nil - -function input.reset() - input.instance = input.newinstance() - return input.instance -end - -function input.reset_hashes() - input.instance.lists = { } - input.instance.found = { } -end - -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) -end +statistics = statistics or { } -function input.settrace(n) - input.trace = tonumber(n or 0) - if input.trace > 0 then - input.verbose = true - end -end +statistics.enable = true +statistics.threshold = 0.05 -input.log = (texio and texio.write_nl) or print +-- timing functions -function input.report(...) - if input.verbose then - input.log("<<"..format(...)..">>") - end -end +local clock = os.gettimeofday or os.clock -function input.report(...) - if input.trace > 0 then -- extra test - input.log("<<"..format(...)..">>") - end +function statistics.hastimer(instance) + return instance and instance.starttime end -input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) - --- These functions can be used to test the performance, especially --- loading the database files. - -do - local clock = os.gettimeofday or os.clock - - function input.starttiming(instance) - if instance then +function statistics.starttiming(instance) + if instance then + local it = instance.timing + if not it then + it = 0 + end + if it == 0 then instance.starttime = clock() if not instance.loadtime then instance.loadtime = 0 end end + instance.timing = it + 1 end +end - function input.stoptiming(instance, report) - if instance then +function statistics.stoptiming(instance, report) + if instance then + local it = instance.timing + if it > 1 then + instance.timing = it - 1 + else local starttime = instance.starttime if starttime then local stoptime = clock() @@ -3098,1032 +3424,727 @@ do instance.stoptime = stoptime instance.loadtime = instance.loadtime + loadtime if report then - input.report("load time %0.3f",loadtime) + statistics.report("load time %0.3f",loadtime) end + instance.timing = 0 return loadtime end end - return 0 end - + return 0 end -function input.elapsedtime(instance) +function statistics.elapsedtime(instance) return format("%0.3f",(instance and instance.loadtime) or 0) end -function input.report_loadtime(instance) - if instance then - input.report('total load time %s', input.elapsedtime(instance)) - end +function statistics.elapsedindeed(instance) + local t = (instance and instance.loadtime) or 0 + return t > statistics.threshold end -input.loadtime = input.elapsedtime +-- general function -function input.env(key) - return input.instance.environment[key] or input.osenv(key) +function statistics.register(tag,fnc) + if statistics.enable and type(fnc) == "function" then + local rt = registered[tag] or (#statusinfo + 1) + statusinfo[rt] = { tag, fnc } + registered[tag] = rt + if #tag > n then n = #tag end + end end -function input.osenv(key) - local ie = input.instance.environment - local value = ie[key] - if value == nil then - -- local e = os.getenv(key) - local e = os.env[key] - if e == nil then - -- value = "" -- false - else - value = input.bare_variable(e) +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end + -- this code will move + local register = statistics.register + register("luatex banner", function() + return string.lower(status.banner) + end) + register("control sequences", function() + return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) + end) + register("callbacks", function() + local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 + return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) + end) + register("current memory usage", statistics.memused) + register("runtime",statistics.runtime) +-- -- + for i=1,#statusinfo do + local s = statusinfo[i] + local r = s[2]() + if r then + reporter(s[1],r,n) + end end - ie[key] = value + statistics.enable = false end - return value or "" end --- we follow a rather traditional approach: --- --- (1) texmf.cnf given in TEXMFCNF --- (2) texmf.cnf searched in default variable --- --- also we now follow the stupid route: if not set then just assume *one* --- cnf file under texmf (i.e. distribution) +function statistics.show_job_stat(tag,data,n) + texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data)) +end -input.ownpath = input.ownpath or nil -input.ownbin = input.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -input.autoselfdir = true -- false may be handy for debugging +function statistics.memused() -- no math.round yet -) + local round = math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +end -function input.getownpath() - if not input.ownpath then - if input.autoselfdir and os.selfdir then - input.ownpath = os.selfdir - else - local binary = input.ownbin - if os.platform == "windows" then - binary = file.replacesuffix(binary,"exe") - end - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do - local b = file.join(p,binary) - if lfs.isfile(b) then - -- we assume that after changing to the path the currentdir function - -- resolves to the real location and use this side effect here; this - -- trick is needed because on the mac installations use symlinks in the - -- path instead of real locations - local olddir = lfs.currentdir() - if lfs.chdir(p) then - local pp = lfs.currentdir() - if input.verbose and p ~= pp then - input.report("following symlink %s to %s",p,pp) - end - input.ownpath = pp - lfs.chdir(olddir) - else - if input.verbose then - input.report("unable to check path %s",p) - end - input.ownpath = p - end - break - end - end - end - if not input.ownpath then input.ownpath = '.' end - end - return input.ownpath +if statistics.runtime then + -- already loaded and set +elseif luatex and luatex.starttime then + statistics.starttime = luatex.starttime + statistics.loadtime = 0 + statistics.timing = 0 +else + statistics.starttiming(statistics) end -function input.identify_own() - local instance = input.instance - local ownpath = input.getownpath() or lfs.currentdir() - local ie = instance.environment - if ownpath then - if input.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end - if input.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end - if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end - else - input.verbose = true - input.report("error: unable to locate ownpath") - os.exit() - end - if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end - if input.env('TEXOS') == "" then os.env['TEXOS'] = input.env('SELFAUTODIR') end - if input.env('TEXROOT') == "" then os.env['TEXROOT'] = input.env('SELFAUTOPARENT') end - if input.verbose then - for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do - input.report("variable %s set to %s",v,input.env(v) or "unknown") - end - end - function input.identify_own() end +function statistics.runtime() + statistics.stoptiming(statistics) + return statistics.formatruntime(statistics.elapsedtime(statistics)) end -function input.identify_cnf() - local instance = input.instance - if #instance.cnffiles == 0 then - -- fallback - input.identify_own() - -- the real search - input.expand_variables() - local t = input.split_path(input.env('TEXMFCNF')) - t = input.aux.expanded_path(t) - input.aux.expand_vars(t) -- redundant - local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) - if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end - end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - end +function statistics.formatruntime(runtime) + return format("%s seconds", statistics.elapsedtime(statistics)) end -function input.load_cnf() - local instance = input.instance - local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(fname) - end - end - -- instance.cnffiles contain complete names now ! - if #instance.cnffiles == 0 then - input.report("no cnf files found (TEXMFCNF may not be set/known)") - else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = input.normalize_name(instance.rootpath) - if instance.lsrmode then - loadoldconfigdata() - elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance.cnffiles) - if instance.loaderror then - loadoldconfigdata() - input.saveoldconfig() - end - else - loadoldconfigdata() - if instance.renewcache then - input.saveoldconfig() - end - end - input.aux.collapse_cnf_data() - end - input.checkconfigdata() +function statistics.timed(action,report) + local timer = { } + report = report or logs.simple + statistics.starttiming(timer) + action() + statistics.stoptiming(timer) + report("total runtime: %s",statistics.elapsedtime(timer)) end -function input.load_lua() - local instance = input.instance - if #instance.luafiles == 0 then - -- yet harmless - else - instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = input.normalize_name(instance.rootpath) - input.loadnewconfig() - input.aux.collapse_cnf_data() + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- this is old code that needs an overhaul + +local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + +if texlua then + write_nl = print + write = io.write +end + +--[[ldx-- +

This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.

+--ldx]]-- + +logs = logs or { } +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +--[[ldx-- +

This looks pretty ugly but we need to speed things up a bit.

+--ldx]]-- + +logs.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4, +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', + 'start_run', 'stop_run', + 'start_page_number', 'stop_page_number', + 'report_output_pages', 'report_output_log', + 'report_tex_stat', 'report_job_stat', + 'show_open', 'show_close', 'show_load', +} + +logs.tracers = { +} + +logs.level = 0 +logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in next, logs.functions do + logs[v] = logs[method][v] or function() end end - input.checkconfigdata() end -function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - local instance = input.instance - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) - end - end - end +-- tex logging + +function logs.tex.report(category,fmt,...) -- new + if fmt then + write_nl(category .. " | " .. format(fmt,...)) + else + write_nl(category .. " |") end end -function input.aux.load_cnf(fname) - local instance = input.instance - fname = input.clean_path(fname) - local lname = file.replacesuffix(fname,'lua') - local f = io.open(lname) - if f then -- this will go - f:close() - local dname = file.dirname(fname) - if not instance.configuration[dname] then - input.aux.load_configuration(dname,lname) - instance.order[#instance.order+1] = instance.configuration[dname] - end +function logs.tex.line(fmt,...) -- new + if fmt then + write_nl(format(fmt,...)) else - f = io.open(fname) - if f then - input.report("loading %s", fname) - local line, data, n, k, v - local dname = file.dirname(fname) - if not instance.configuration[dname] then - instance.configuration[dname] = { } - instance.order[#instance.order+1] = instance.configuration[dname] - end - local data = instance.configuration[dname] - while true do - local line, n = f:read(), 0 - if line then - while true do -- join lines - line, n = line:gsub("\\%s*$", "") - if n > 0 then - line = line .. f:read() - else - break - end - end - if not line:find("^[%%#]") then - local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") - if k and v and not data[k] then - data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") - instance.kpsevars[k] = true - end - end - else - break - end - end - f:close() - else - input.report("skipping %s", fname) - end + write_nl("") end end --- database loading +local texcount = tex and tex.count -function input.load_hash() - local instance = input.instance - input.locatelists() - if instance.lsrmode then - input.loadlists() - elseif instance.diskcache and not instance.renewcache then - input.loadfiles() - if instance.loaderror then - input.loadlists() - input.savefiles() +function logs.tex.start_page_number() + local real, user, sub = texcount[0], texcount[1], texcount[2] + if real > 0 then + if user > 0 then + if sub > 0 then + write(format("[%s.%s.%s",real,user,sub)) + else + write(format("[%s.%s",real,user)) + end + else + write(format("[%s",real)) end else - input.loadlists() - if instance.renewcache then - input.savefiles() - end + write("[-") end end -function input.aux.append_hash(type,tag,name) - if input.trace > 0 then - input.logger("= hash append: %s",tag) - end - table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +function logs.tex.stop_page_number() + write("]") end -function input.aux.prepend_hash(type,tag,name) - if input.trace > 0 then - input.logger("= hash prepend: %s",tag) +logs.tex.report_job_stat = statistics.show_job_stat + +-- xml logging + +function logs.xml.report(category,fmt,...) -- new + if fmt then + write_nl(format("%s",category,format(fmt,...))) + else + write_nl(format("",category)) end - table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end - -function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash - local instance = input.instance --- local t = input.expanded_path_list('TEXMF') -- full expansion - local t = input.split_path(input.env('TEXMF')) - table.insert(t,1,specification) - local newspec = table.join(t,";") - if instance.environment["TEXMF"] then - instance.environment["TEXMF"] = newspec - elseif instance.variables["TEXMF"] then - instance.variables["TEXMF"] = newspec +function logs.xml.line(fmt,...) -- new + if fmt then + write_nl(format("%s",format(fmt,...))) else - -- weird + write_nl("") end - input.expand_variables() - input.reset_hashes() end --- locators +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("") end end +function logs.xml.push () if logs.level > 0 then tw("" ) end end -function input.locatelists() - local instance = input.instance - for _, path in pairs(input.clean_path_list('TEXMF')) do - input.report("locating list of %s",path) - input.locatedatabase(input.normalize_name(path)) - end +function logs.xml.start_run() + write_nl("") + write_nl("") -- xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng' + write_nl("") end -function input.locatedatabase(specification) - return input.methodhandler('locators', specification) +function logs.xml.stop_run() + write_nl("") end -function input.locators.tex(specification) - if specification and specification ~= '' and lfs.isdir(specification) then - if input.trace > 0 then - input.logger('! tex locator found: %s',specification) - end - input.aux.append_hash('file',specification,filename) - elseif input.trace > 0 then - input.logger('? tex locator not found: %s',specification) - end +function logs.xml.start_page_number() + write_nl(format("

") + write_nl("") +end -function input.hashdatabase(tag,name) - return input.methodhandler('hashers',tag,name) +function logs.xml.report_output_pages(p,b) + write_nl(format("", p)) + write_nl(format("", b)) + write_nl("") end -function input.loadfiles() - local instance = input.instance - instance.loaderror = false - instance.files = { } - if not instance.renewcache then - for _, hash in ipairs(instance.hashes) do - input.hashdatabase(hash.tag,hash.name) - if instance.loaderror then break end - end - end +function logs.xml.report_output_log() end -function input.hashers.tex(tag,name) - input.aux.load_files(tag) +function logs.xml.report_tex_stat(k,v) + texiowrite_nl("log",""..tostring(v).."") end --- generators: +local level = 0 -function input.loadlists() - for _, hash in ipairs(input.instance.hashes) do - input.generatedatabase(hash.tag) - end +function logs.xml.show_open(name) + level = level + 1 + texiowrite_nl(format("",level,name)) end -function input.generatedatabase(specification) - return input.methodhandler('generators', specification) +function logs.xml.show_close(name) + texiowrite(" ") + level = level - 1 end -local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) - -function input.generators.tex(specification) - local instance = input.instance - local tag = specification - if not instance.lsrmode and lfs.dir then - input.report("scanning path %s",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' - else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == 'directory' then - m = m + 1 - if path then - action(path..'/'..name) - else - action(name) - end - elseif path and mode == 'file' then - n = n + 1 - local f = files[name] - if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end - end - else - files[name] = path - local lower = name:lower() - if name ~= lower then - files["remap:"..lower] = name - r = r + 1 - end - end - end - end - end - end - action() - input.report("%s files found on %s directories with %s uppercase remappings",n,m,r) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file %s",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then - if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table - else - fl[#fl+1] = path - end - end - else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line - end - end - else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line - end - end - f:close() - end - end +function logs.xml.show_load(name) + texiowrite_nl(format("",level+1,name)) end --- savers, todo - -function input.savefiles() - input.aux.save_data('files', function(k,v) - return input.instance.validfile(k,v) -- path, name - end) -end +-- --- A config (optionally) has the paths split in tables. Internally --- we join them and split them after the expansion has taken place. This --- is more convenient. +local name, banner = 'report', 'context' -function input.splitconfig() - for i,c in ipairs(input.instance) do - for k,v in pairs(c) do - if type(v) == 'string' then - local t = file.split_path(v) - if #t > 1 then - c[k] = t - end - end - end +local function report(category,fmt,...) + if fmt then + write_nl(format("%s | %s: %s",name,category,format(fmt,...))) + elseif category then + write_nl(format("%s | %s",name,category)) + else + write_nl(format("%s |",name)) end end -function input.joinconfig() - for i,c in ipairs(input.instance.order) do - for k,v in pairs(c) do - if type(v) == 'table' then - c[k] = file.join_path(v) - end - end +local function simple(fmt,...) + if fmt then + write_nl(format("%s | %s",name,format(fmt,...))) + else + write_nl(format("%s |",name)) end end -function input.split_path(str) - if type(str) == 'table' then - return str - else - return file.split_path(str) + +function logs.setprogram(_name_,_banner_,_verbose_) + name, banner = _name_, _banner_ + if _verbose_ then + trackers.enable("resolvers.verbose") end + logs.set_method("tex") + logs.report = report -- also used in libraries + logs.simple = simple -- only used in scripts ! + if utils then + utils.report = simple + end + logs.verbose = _verbose_ end -function input.join_path(str) - if type(str) == 'table' then - return file.join_path(str) + +function logs.setverbose(what) + if what then + trackers.enable("resolvers.verbose") else - return str + trackers.disable("resolvers.verbose") end + logs.verbose = what or false end -function input.splitexpansions() - local ie = input.instance.expansions - for k,v in pairs(ie) do - local t, h = { }, { } - for _,vv in pairs(file.split_path(v)) do - if vv ~= "" and not h[vv] then - t[#t+1] = vv - h[vv] = true - end - end - if #t > 1 then - ie[k] = t - else - ie[k] = t[1] - end +function logs.extendbanner(_banner_,_verbose_) + banner = banner .. " | ".. _banner_ + if _verbose_ ~= nil then + logs.setverbose(what) end end --- end of split/join code +logs.verbose = false +logs.report = logs.tex.report +logs.simple = logs.tex.report -function input.saveoldconfig() - input.splitconfig() - input.aux.save_data('configuration', nil) - input.joinconfig() +function logs.reportlines(str) -- todo: + for line in str:gmatch("(.-)[\n\r]") do + logs.report(line) + end end -input.configbanner = [[ --- This is a Luatex configuration file created by 'luatools.lua' or --- 'luatex.exe' directly. For comment, suggestions and questions you can --- contact the ConTeXt Development Team. This configuration file is --- not copyrighted. [HH & TH] -]] - -function input.serialize(files) - -- This version is somewhat optimized for the kind of - -- tables that we deal with, so it's much faster than - -- the generic serializer. This makes sense because - -- luatools and mtxtools are called frequently. Okay, - -- we pay a small price for properly tabbed tables. - local t = { } - local function dump(k,v,m) - if type(v) == 'string' then - return m .. "['" .. k .. "']='" .. v .. "'," - elseif #v == 1 then - return m .. "['" .. k .. "']='" .. v[1] .. "'," - else - return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," - end - end - t[#t+1] = "return {" - if input.instance.sortdata then - for _, k in pairs(sortedkeys(files)) do - local fk = files[k] - if type(fk) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do - t[#t+1] = dump(kk,fk[kk],"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,fk,"\t") - end - end - else - for k, v in pairs(files) do - if type(v) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do - t[#t+1] = dump(kk,vv,"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,v,"\t") - end - end - end - t[#t+1] = "}" - return concat(t,"\n") +function logs.reportline() -- for scripts too + logs.report() end -if not texmf then texmf = {} end -- no longer needed, at least not here +logs.simpleline = logs.reportline -function input.aux.save_data(dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(input.instance[dataname]) do - local name = (makename or file.join)(cachename,dataname) - local luaname, lucname = name .. ".lua", name .. ".luc" - input.report("preparing %s for %s",dataname,cachename) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false - end - end - local data = { - type = dataname, - root = cachename, - version = input.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = files, - } - local ok = io.savedata(luaname,input.serialize(data)) - if ok then - input.report("%s saved in %s",dataname,luaname) - if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - input.report("%s compiled to %s",dataname,lucname) - else - input.report("compiling failed for %s, deleting file %s",dataname,lucname) - os.remove(lucname) - end - else - input.report("unable to save %s in %s (access error)",dataname,luaname) - end +function logs.help(message,option) + logs.report(banner) + logs.reportline() + logs.reportlines(message) + local moreinfo = logs.moreinfo or "" + if moreinfo ~= "" and option ~= "nomoreinfo" then + logs.reportline() + logs.reportlines(moreinfo) end end -function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload - local instance = input.instance - filename = ((not filename or (filename == "")) and dataname) or filename - filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) - local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") - if blob then - local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading %s for %s from %s",dataname,pathname,filename) - instance[dataname][pathname] = data.content +logs.set_level('error') +logs.set_method('tex') + +function logs.system(whereto,process,jobname,category,...) + for i=1,10 do + local f = io.open(whereto,"a") + if f then + f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...))) + f:close() + break else - input.report("skipping %s for %s from %s",dataname,pathname,filename) - instance[dataname][pathname] = { } - instance.loaderror = true + sleep(0.1) end - else - input.report("skipping %s for %s from %s",dataname,pathname,filename) end end --- some day i'll use the nested approach, but not yet (actually we even drop --- engine/progname support since we have only luatex now) +--~ local syslogname = "oeps.xxx" +--~ +--~ for i=1,10 do +--~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + comment = "companion to luat-lib.tex", +} + +-- After a few years using the code the large luat-inp.lua file +-- has been split up a bit. In the process some functionality was +-- dropped: -- --- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- * support for reading lsr files +-- * selective scanning (subtrees) +-- * some public auxiliary functions were made private -- --- return { --- TEXMFBOGUS = 'effe checken of dit werkt', --- } +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow -function input.aux.load_texmfcnf(dataname,pathname) - local instance = input.instance - local filename = file.join(pathname,input.luaname) - local blob = loadfile(filename) - if blob then - local data = blob() - if data then - input.report("loading configuration file %s",filename) - if true then - -- flatten to variable.progname - local t = { } - for k, v in pairs(data) do -- v = progname - if type(v) == "string" then - t[k] = v - else - for kk, vv in pairs(v) do -- vv = variable - if type(vv) == "string" then - t[vv.."."..v] = kk - end - end - end - end - instance[dataname][pathname] = t - else - instance[dataname][pathname] = data - end - else - input.report("skipping configuration file %s",filename) - instance[dataname][pathname] = { } - instance.loaderror = true - end - else - input.report("skipping configuration file %s",filename) - end -end +-- This lib is multi-purpose and can be loaded again later on so that +-- additional functionality becomes available. We will split thislogs.report("fileio", +-- module in components once we're done with prototyping. This is the +-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing +-- something in this module one can best check with Taco or Hans first; there +-- is some nasty trickery going on that relates to traditional kpse support. -function input.aux.load_configuration(dname,lname) - input.aux.load_data(dname,'configuration',lname and file.basename(lname)) -end -function input.aux.load_files(tag) - input.aux.load_data(tag,'files') -end +-- To be considered: hash key lowercase, first entry in table filename +-- (any case), rest paths (so no need for optimization). Or maybe a +-- separate table that matches lowercase names to mixed case when +-- present. In that case the lower() cases can go away. I will do that +-- only when we run into problems with names ... well ... Iwona-Regular. -function input.resetconfig() - input.identify_own() - local instance = input.instance - instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +-- Beware, loading and saving is overloaded in luat-tmp! + +local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch +local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys +local next, type = next, type + +local trace_locating, trace_detail, trace_verbose = false, false, false + +trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) + +if not resolvers then + resolvers = { + suffixes = { }, + formats = { }, + dangerous = { }, + suffixmap = { }, + alternatives = { }, + locators = { }, -- locate databases + hashers = { }, -- load databases + generators = { }, -- generate databases + } end -function input.loadnewconfig() - local instance = input.instance - for _, cnf in ipairs(instance.luafiles) do - local dname = file.dirname(cnf) - input.aux.load_texmfcnf('setup',dname) - instance.order[#instance.order+1] = instance.setup[dname] - if instance.loaderror then break end - end -end +local resolvers = resolvers + +resolvers.locators .notfound = { nil } +resolvers.hashers .notfound = { nil } +resolvers.generators.notfound = { nil } + +resolvers.cacheversion = '1.0.1' +resolvers.cnfname = 'texmf.cnf' +resolvers.luaname = 'texmfcnf.lua' +resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +local dummy_path_expr = "^!*unset/*$" + +local formats = resolvers.formats +local suffixes = resolvers.suffixes +local dangerous = resolvers.dangerous +local suffixmap = resolvers.suffixmap +local alternatives = resolvers.alternatives + +formats['afm'] = 'AFMFONTS' suffixes['afm'] = { 'afm' } +formats['enc'] = 'ENCFONTS' suffixes['enc'] = { 'enc' } +formats['fmt'] = 'TEXFORMATS' suffixes['fmt'] = { 'fmt' } +formats['map'] = 'TEXFONTMAPS' suffixes['map'] = { 'map' } +formats['mp'] = 'MPINPUTS' suffixes['mp'] = { 'mp' } +formats['ocp'] = 'OCPINPUTS' suffixes['ocp'] = { 'ocp' } +formats['ofm'] = 'OFMFONTS' suffixes['ofm'] = { 'ofm', 'tfm' } +formats['otf'] = 'OPENTYPEFONTS' suffixes['otf'] = { 'otf' } -- 'ttf' +formats['opl'] = 'OPLFONTS' suffixes['opl'] = { 'opl' } +formats['otp'] = 'OTPINPUTS' suffixes['otp'] = { 'otp' } +formats['ovf'] = 'OVFFONTS' suffixes['ovf'] = { 'ovf', 'vf' } +formats['ovp'] = 'OVPFONTS' suffixes['ovp'] = { 'ovp' } +formats['tex'] = 'TEXINPUTS' suffixes['tex'] = { 'tex' } +formats['tfm'] = 'TFMFONTS' suffixes['tfm'] = { 'tfm' } +formats['ttf'] = 'TTFONTS' suffixes['ttf'] = { 'ttf', 'ttc' } +formats['pfb'] = 'T1FONTS' suffixes['pfb'] = { 'pfb', 'pfa' } +formats['vf'] = 'VFFONTS' suffixes['vf'] = { 'vf' } + +formats['fea'] = 'FONTFEATURES' suffixes['fea'] = { 'fea' } +formats['cid'] = 'FONTCIDMAPS' suffixes['cid'] = { 'cid', 'cidmap' } + +formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +formats ['lua'] = 'LUAINPUTS' -- new +suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } -function input.loadoldconfig() - local instance = input.instance - if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do - local dname = file.dirname(cnf) - input.aux.load_configuration(dname) - instance.order[#instance.order+1] = instance.configuration[dname] - if instance.loaderror then break end - end - end - input.joinconfig() -end +-- backward compatible ones -function input.expand_variables() - local instance = input.instance - local expansions, environment, variables = { }, instance.environment, instance.variables - local env = input.env - instance.expansions = expansions - if instance.engine ~= "" then environment['engine'] = instance.engine end - if instance.progname ~= "" then environment['progname'] = instance.progname end - for k,v in pairs(environment) do - local a, b = k:match("^(%a+)%_(.*)%s*$") - if a and b then - expansions[a..'.'..b] = v - else - expansions[k] = v - end - end - for k,v in pairs(environment) do -- move environment to expansions - if not expansions[k] then expansions[k] = v end - end - for k,v in pairs(variables) do -- move variables to expansions - if not expansions[k] then expansions[k] = v end - end - while true do - local busy = false - for k,v in pairs(expansions) do - local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) - busy = true - return expansions[a] or env(a) - end) - local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) - busy = true - return expansions[a] or env(a) - end) - if n > 0 or m > 0 then - expansions[k]= s - end - end - if not busy then break end - end - for k,v in pairs(expansions) do - expansions[k] = v:gsub("\\", '/') - end -end +alternatives['map files'] = 'map' +alternatives['enc files'] = 'enc' +alternatives['cid files'] = 'cid' +alternatives['fea files'] = 'fea' +alternatives['opentype fonts'] = 'otf' +alternatives['truetype fonts'] = 'ttf' +alternatives['truetype collections'] = 'ttc' +alternatives['type1 fonts'] = 'pfb' -function input.aux.expand_vars(lst) -- simple vars - local instance = input.instance - local variables, env = instance.variables, input.env - for k,v in pairs(lst) do - lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return variables[a] or env(a) - end) - end -end +-- obscure ones -function input.aux.expanded_var(var) -- simple vars - local instance = input.instance - return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(a) - end) -end +formats ['misc fonts'] = '' +suffixes['misc fonts'] = { } -function input.aux.entry(entries,name) - if name and (name ~= "") then - local instance = input.instance - name = name:gsub('%$','') - local result = entries[name..'.'..instance.progname] or entries[name] - if result then - return result - else - result = input.env(name) - if result then - instance.variables[name] = result - input.expand_variables() - return instance.expansions[name] or "" - end - end - end - return "" -end -function input.variable(name) - return input.aux.entry(input.instance.variables,name) -end -function input.expansion(name) - return input.aux.entry(input.instance.expansions,name) -end +formats ['sfd'] = 'SFDFONTS' +suffixes ['sfd'] = { 'sfd' } +alternatives['subfont definition files'] = 'sfd' -function input.aux.is_entry(entries,name) - if name and name ~= "" then - name = name:gsub('%$','') - return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil - else - return false +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. + +-- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// + +-- we always have one instance active + +resolvers.instance = resolvers.instance or nil -- the current one (slow access) +local instance = resolvers.instance or nil -- the current one (fast access) + +function resolvers.newinstance() + + -- store once, freeze and faster (once reset we can best use + -- instance.environment) maybe better have a register suffix + -- function + + for k, v in next, suffixes do + for i=1,#v do + local vi = v[i] + if vi then + suffixmap[vi] = k + end + end + end + + -- because vf searching is somewhat dangerous, we want to prevent + -- too liberal searching esp because we do a lookup on the current + -- path anyway; only tex (or any) is safe + + for k, v in next, formats do + dangerous[k] = true + end + dangerous.tex = nil + + -- the instance + + local newinstance = { + rootpath = '', + treepath = '', + progname = 'context', + engine = 'luatex', + format = '', + environment = { }, + variables = { }, + expansions = { }, + files = { }, + remap = { }, + configuration = { }, + setup = { }, + order = { }, + found = { }, + foundintrees = { }, + kpsevars = { }, + hashes = { }, + cnffiles = { }, + luafiles = { }, + lists = { }, + remember = true, + diskcache = true, + renewcache = false, + scandisk = true, + cachepath = nil, + loaderror = false, + sortdata = false, + savelists = true, + cleanuppaths = true, + allresults = false, + pattern = nil, -- lists + data = { }, -- only for loading + force_suffixes = true, + fakepaths = { }, + } + + local ne = newinstance.environment + + for k,v in next, os.env do + ne[k] = resolvers.bare_variable(v) end -end -function input.is_variable(name) - return input.aux.is_entry(input.instance.variables,name) -end + return newinstance -function input.is_expansion(name) - return input.aux.is_entry(input.instance.expansions,name) end -function input.unexpanded_path_list(str) - local pth = input.variable(str) - local lst = input.split_path(pth) - return input.aux.expanded_path(lst) +function resolvers.setinstance(someinstance) + instance = someinstance + resolvers.instance = someinstance + return someinstance end -function input.unexpanded_path(str) - return file.join_path(input.unexpanded_path_list(str)) +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) end -do - local done = { } +local function reset_hashes() + instance.lists = { } + instance.found = { } +end - function input.reset_extra_path() - local instance = input.instance - local ep = instance.extra_paths - if not ep then - ep, done = { }, { } - instance.extra_paths = ep - elseif #ep > 0 then - instance.lists, done = { }, { } +local function check_configuration() -- not yet ok, no time for debugging now + local ie = instance.environment + local function fix(varname,default) + local proname = varname .. "." .. instance.progname or "crap" + local p, v = ie[proname], ie[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? end end - - function input.register_extra_path(paths,subpaths) - local instance = input.instance - local ep = instance.extra_paths or { } - local n = #ep - if paths and paths ~= "" then - if subpaths and subpaths ~= "" then - for p in paths:gmatch("[^,]+") do - -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do - local ps = p .. "/" .. s - if not done[ps] then - ep[#ep+1] = input.clean_path(ps) - done[ps] = true - end - end - end - else - for p in paths:gmatch("[^,]+") do - if not done[p] then - ep[#ep+1] = input.clean_path(p) - done[p] = true - end - end - end - elseif subpaths and subpaths ~= "" then - for i=1,n do - -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do - local ps = ep[i] .. "/" .. s - if not done[ps] then - ep[#ep+1] = input.clean_path(ps) - done[ps] = true - end - end - end - end - if #ep > 0 then - instance.extra_paths = ep -- register paths - end - if #ep > n then - instance.lists = { } -- erase the cache - end + local name = os.name + if name == "windows" then + fix("OSFONTDIR", "c:/windows/fonts//") + elseif name == "macosx" then + fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + else + -- bad luck end + fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") +end +function resolvers.bare_variable(str) -- assumes str is a string + return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2")) end -function input.expanded_path_list(str) - local instance = input.instance - local function made_list(list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new = { }, { } - -- honour . .. ../.. but only when at the start - for k, v in ipairs(list) do - if not done[v] then - if v:find("^[%.%/]$") then - done[v] = true - new[#new+1] = v - else - break - end - end - end - -- first the extra paths - for k, v in ipairs(ep) do - if not done[v] then - done[v] = true - new[#new+1] = v - end - end - -- next the formal paths - for k, v in ipairs(list) do - if not done[v] then - done[v] = true - new[#new+1] = v - end - end - return new - end - end - if not str then - return ep or { } - elseif instance.savelists then - -- engine+progname hash - str = str:gsub("%$","") - if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(str))) - instance.lists[str] = input.aux.expanded_path(lst) - end - return instance.lists[str] - else - local lst = input.split_path(input.expansion(str)) - return made_list(input.aux.expanded_path(lst)) +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) end end +resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) -function input.clean_path_list(str) - local t = input.expanded_path_list(str) - if t then - for i=1,#t do - t[i] = file.collapse_path(input.clean_path(t[i])) +function resolvers.osenv(key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = resolvers.bare_variable(e) end + ie[key] = value end - return t + return value or "" end -function input.expand_path(str) - return file.join_path(input.expanded_path_list(str)) +function resolvers.env(key) + return instance.environment[key] or resolvers.osenv(key) end -function input.expanded_path_list_from_var(str) -- brrr - local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) - if tmp ~= "" then - return input.expanded_path_list(str) - else - return input.expanded_path_list(tmp) - end -end -function input.expand_path_from_var(str) - return file.join_path(input.expanded_path_list_from_var(str)) -end +-- -function input.format_of_var(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' -end -function input.format_of_suffix(str) - return input.suffixmap[file.extname(str)] or 'tex' +local function expand_vars(lst) -- simple vars + local variables, env = instance.variables, resolvers.env + local function resolve(a) + return variables[a] or env(a) + end + for k=1,#lst do + lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve) + end end -function input.variable_of_format(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +local function expanded_var(var) -- simple vars + local function resolve(a) + return instance.variables[a] or resolvers.env(a) + end + return (gsub(var,"%$([%a%d%_%-]+)",resolve)) end -function input.var_of_format_or_suffix(str) - local v = input.formats[str] - if v then - return v - end - v = input.formats[input.alternatives[str]] - if v then - return v - end - v = input.suffixmap[file.extname(str)] - if v then - return input.formats[isf] +local function entry(entries,name) + if name and (name ~= "") then + name = gsub(name,'%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = resolvers.env(name) + if result then + instance.variables[name] = result + resolvers.expand_variables() + return instance.expansions[name] or "" + end + end end - return '' + return "" end -function input.expand_braces(str) -- output variable and brace expansion of STRING - local ori = input.variable(str) - local pth = input.aux.expanded_path(input.split_path(ori)) - return file.join_path(pth) +local function is_entry(entries,name) + if name and name ~= "" then + name = gsub(name,'%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false + end end -- {a,b,c,d} @@ -4146,2363 +4167,2168 @@ end -- work that well; the parsing is ok, but dealing with the resulting -- table is a pain because we need to work inside-out recursively -function input.aux.splitpathexpr(str, t, validate) - -- no need for optimization, only called a few times, we can use lpeg for the sub +local function splitpathexpr(str, t, validate) + -- no need for further optimization as it is only called a + -- few times, we can use lpeg for the sub; we could move + -- the local functions outside the body t = t or { } - str = str:gsub(",}",",@}") - str = str:gsub("{,","{@,") + str = gsub(str,",}",",@}") + str = gsub(str,"{,","{@,") -- str = "@" .. str .. "@" + local ok, done + local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" + end + local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" + end + local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb + end + end + return "{" .. concat(t,",") .. "}" + end + local function do_three(a,b,c) + return a .. b.. c + end while true do - local done = false + done = false while true do - local ok = false - str = str:gsub("([^{},]+){([^{}]+)}", function(a,b) - local t = { } - for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end + str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first) + if ok > 0 then done = true else break end end while true do - local ok = false - str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b) - local t = { } - for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end + str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second) + if ok > 0 then done = true else break end end while true do - local ok = false - str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b) - local t = { } - for sa in a:gmatch("[^,]+") do - for sb in b:gmatch("[^,]+") do - t[#t+1] = sa .. sb - end - end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end + str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both) + if ok > 0 then done = true else break end end - str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c) - done = true - return a .. b.. c - end) + str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three) + if ok > 0 then done = true end if not done then break end end - str = str:gsub("[{}]", "") - str = str:gsub("@","") + str = gsub(str,"[{}]", "") + str = gsub(str,"@","") if validate then - for s in str:gmatch("[^,]+") do + for s in gmatch(str,"[^,]+") do s = validate(s) if s then t[#t+1] = s end end else - for s in str:gmatch("[^,]+") do + for s in gmatch(str,"[^,]+") do t[#t+1] = s end end return t end -function input.aux.expanded_path(pathlist) -- maybe not a list, just a path - local instance = input.instance +local function expanded_path_from_list(pathlist) -- maybe not a list, just a path -- a previous version fed back into pathlist local newlist, ok = { }, false - for _,v in ipairs(pathlist) do - if v:find("[{}]") then + for k=1,#pathlist do + if find(pathlist[k],"[{}]") then ok = true break end end if ok then - for _, v in ipairs(pathlist) do - input.aux.splitpathexpr(v, newlist, function(s) - s = file.collapse_path(s) - return s ~= "" and not s:find(instance.dummy_path_expr) and s - end) + local function validate(s) + s = file.collapse_path(s) + return s ~= "" and not find(s,dummy_path_expr) and s + end + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) end else - for _,v in ipairs(pathlist) do - for vv in string.gmatch(v..',',"(.-),") do - vv = file.collapse_path(v) - if vv ~= "" then newlist[#newlist+1] = vv end + for k=1,#pathlist do + for p in gmatch(pathlist[k],"([^,]+)") do + p = file.collapse_path(p) + if p ~= "" then newlist[#newlist+1] = p end end end end return newlist end -input.is_readable = { } +-- we follow a rather traditional approach: +-- +-- (1) texmf.cnf given in TEXMFCNF +-- (2) texmf.cnf searched in default variable +-- +-- also we now follow the stupid route: if not set then just assume *one* +-- cnf file under texmf (i.e. distribution) + +resolvers.ownpath = resolvers.ownpath or nil +resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +resolvers.autoselfdir = true -- false may be handy for debugging -function input.aux.is_readable(readable, name) - if input.trace > 2 then - if readable then - input.logger("+ readable: %s",name) +function resolvers.getownpath() + if not resolvers.ownpath then + if resolvers.autoselfdir and os.selfdir then + resolvers.ownpath = os.selfdir else - input.logger("- readable: %s", name) + local binary = resolvers.ownbin + if os.platform == "windows" then + binary = file.replacesuffix(binary,"exe") + end + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,binary) + if lfs.isfile(b) then + -- we assume that after changing to the path the currentdir function + -- resolves to the real location and use this side effect here; this + -- trick is needed because on the mac installations use symlinks in the + -- path instead of real locations + local olddir = lfs.currentdir() + if lfs.chdir(p) then + local pp = lfs.currentdir() + if trace_verbose and p ~= pp then + logs.report("fileio","following symlink %s to %s",p,pp) + end + resolvers.ownpath = pp + lfs.chdir(olddir) + else + if trace_verbose then + logs.report("fileio","unable to check path %s",p) + end + resolvers.ownpath = p + end + break + end + end end + if not resolvers.ownpath then resolvers.ownpath = '.' end end - return readable -end - -function input.is_readable.file(name) - return input.aux.is_readable(lfs.isfile(name), name) + return resolvers.ownpath end -input.is_readable.tex = input.is_readable.file - --- name --- name/name +local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } -function input.aux.collect_files(names) - local instance = input.instance - local filelist = { } - for _, fname in pairs(names) do - if fname then - if input.trace > 2 then - input.logger("? blobpath asked: %s",fname) - end - local bname = file.basename(fname) +local function identify_own() + local ownpath = resolvers.getownpath() or lfs.currentdir() + local ie = instance.environment + if ownpath then + if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if resolvers.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end + else + logs.report("fileio","error: unable to locate ownpath") + os.exit() + end + if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end + if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end + if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end + if trace_verbose then + for i=1,#own_places do + local v = own_places[i] + logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") + end + end + identify_own = function() end +end + +function resolvers.identify_cnf() + if #instance.cnffiles == 0 then + -- fallback + identify_own() + -- the real search + resolvers.expand_variables() + local t = resolvers.split_path(resolvers.env('TEXMFCNF')) + t = expanded_path_from_list(t) + expand_vars(t) -- redundant + local function locate(filename,list) + for i=1,#t do + local ti = t[i] + local texmfcnf = file.collapse_path(file.join(ti,filename)) + if lfs.isfile(texmfcnf) then + list[#list+1] = texmfcnf + end + end + end + locate(resolvers.luaname,instance.luafiles) + locate(resolvers.cnfname,instance.cnffiles) + end +end + +local function load_cnf_file(fname) + fname = resolvers.clean_path(fname) + local lname = file.replacesuffix(fname,'lua') + local f = io.open(lname) + if f then -- this will go + f:close() + local dname = file.dirname(fname) + if not instance.configuration[dname] then + resolvers.load_data(dname,'configuration',lname and file.basename(lname)) + instance.order[#instance.order+1] = instance.configuration[dname] + end + else + f = io.open(fname) + if f then + if trace_verbose then + logs.report("fileio","loading %s", fname) + end + local line, data, n, k, v local dname = file.dirname(fname) - if dname == "" or dname:find("^%.") then - dname = false - else - dname = "/" .. dname .. "$" + if not instance.configuration[dname] then + instance.configuration[dname] = { } + instance.order[#instance.order+1] = instance.configuration[dname] end - for _, hash in ipairs(instance.hashes) do - local blobpath = hash.tag - local files = blobpath and instance.files[blobpath] - if files then - if input.trace > 2 then - input.logger('? blobpath do: %s (%s)',blobpath,bname) - end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] + local data = instance.configuration[dname] + while true do + local line, n = f:read(), 0 + if line then + while true do -- join lines + line, n = gsub(line,"\\%s*$", "") + if n > 0 then + line = line .. f:read() + else + break end end - if blobfile then - if type(blobfile) == 'string' then - if not dname or blobfile:find(dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,blobfile,bname), -- search - input.concatinators[hash.type](blobpath,blobfile,bname) -- result - } - end - else - for _, vv in pairs(blobfile) do - if not dname or vv:find(dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,vv,bname), -- search - input.concatinators[hash.type](blobpath,vv,bname) -- result - } - end - end + if not find(line,"^[%%#]") then + local l = gsub(line,"%s*%%.*$","") + local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$") + if k and v and not data[k] then + v = gsub(v,"[%%#].*",'') + data[k] = gsub(v,"~","$HOME") + instance.kpsevars[k] = true end end - elseif input.trace > 1 then - input.logger('! blobpath no: %s (%s)',blobpath,bname) + else + break end end + f:close() + elseif trace_verbose then + logs.report("fileio","skipping %s", fname) end end - if #filelist > 0 then - return filelist - else - return nil +end + +local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in next, c do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = resolvers.bare_variable(v) + end + end + end end end -function input.suffix_of_format(str) - if input.suffixes[str] then - return input.suffixes[str][1] +function resolvers.load_cnf() + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + load_cnf_file(fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + if trace_verbose then + logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") + end else - return "" + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + if instance.diskcache and not instance.renewcache then + resolvers.loadoldconfig(instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + resolvers.saveoldconfig() + end + else + loadoldconfigdata() + if instance.renewcache then + resolvers.saveoldconfig() + end + end + collapse_cnf_data() end + check_configuration() end -function input.suffixes_of_format(str) - if input.suffixes[str] then - return input.suffixes[str] +function resolvers.load_lua() + if #instance.luafiles == 0 then + -- yet harmless else - return {} + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + resolvers.loadnewconfig() + collapse_cnf_data() end + check_configuration() end -do +-- database loading - -- called about 700 times for an empty doc (font initializations etc) - -- i need to weed the font files for redundant calls +function resolvers.load_hash() + resolvers.locatelists() + if instance.diskcache and not instance.renewcache then + resolvers.loadfiles() + if instance.loaderror then + resolvers.loadlists() + resolvers.savefiles() + end + else + resolvers.loadlists() + if instance.renewcache then + resolvers.savefiles() + end + end +end - local letter = lpeg.R("az","AZ") - local separator = lpeg.P("://") +function resolvers.append_hash(type,tag,name) + if trace_locating then + logs.report("fileio","= hash append: %s",tag) + end + insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) +end - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local rootbased = lpeg.P("/") + letter*lpeg.P(":") +function resolvers.prepend_hash(type,tag,name) + if trace_locating then + logs.report("fileio","= hash prepend: %s",tag) + end + insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) +end - -- ./name ../name /name c: :// - function input.aux.qualified_path(filename) - return qualified:match(filename) +function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash +-- local t = resolvers.expanded_path_list('TEXMF') -- full expansion + local t = resolvers.split_path(resolvers.env('TEXMF')) + insert(t,1,specification) + local newspec = concat(t,";") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"] = newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"] = newspec + else + -- weird end - function input.aux.rootbased_path(filename) - return rootbased:match(filename) + resolvers.expand_variables() + reset_hashes() +end + +-- locators + +function resolvers.locatelists() + for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do + if trace_verbose then + logs.report("fileio","locating list of %s",path) + end + resolvers.locatedatabase(file.collapse_path(path)) end +end + +function resolvers.locatedatabase(specification) + return resolvers.methodhandler('locators', specification) +end - function input.normalize_name(original) - return original +function resolvers.locators.tex(specification) + if specification and specification ~= '' and lfs.isdir(specification) then + if trace_locating then + logs.report("fileio",'! tex locator found: %s',specification) + end + resolvers.append_hash('file',specification,filename) + elseif trace_locating then + logs.report("fileio",'? tex locator not found: %s',specification) end +end - input.normalize_name = file.collapse_path +-- hashers +function resolvers.hashdatabase(tag,name) + return resolvers.methodhandler('hashers',tag,name) end -function input.aux.register_in_trees(name) - if not name:find("^%.") then - local instance = input.instance - instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one +function resolvers.loadfiles() + instance.loaderror = false + instance.files = { } + if not instance.renewcache then + for _, hash in ipairs(instance.hashes) do + resolvers.hashdatabase(hash.tag,hash.name) + if instance.loaderror then break end + end end end --- split the next one up, better for jit +function resolvers.hashers.tex(tag,name) + resolvers.load_data(tag,'files') +end -function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) - local instance = input.instance - local result = { } - local stamp = nil - filename = input.normalize_name(filename) -- elsewhere - filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere - -- speed up / beware: format problem - if instance.remember then - stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format - if instance.found[stamp] then - if input.trace > 0 then - input.logger('! remembered: %s',filename) - end - return instance.found[stamp] - end +-- generators: + +function resolvers.loadlists() + for _, hash in ipairs(instance.hashes) do + resolvers.generatedatabase(hash.tag) end - if filename:find('%*') then - if input.trace > 0 then - input.logger('! wildcard: %s', filename) - end - result = input.find_wildcard_files(filename) - elseif input.aux.qualified_path(filename) then - if input.is_readable.file(filename) then - if input.trace > 0 then - input.logger('! qualified: %s', filename) - end - result = { filename } +end + +function resolvers.generatedatabase(specification) + return resolvers.methodhandler('generators', specification) +end + +-- starting with . or .. etc or funny char + +local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) + +function resolvers.generators.tex(specification) + local tag = specification + if trace_verbose then + logs.report("fileio","scanning path %s",specification) + end + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local function action(path) + local full + if path then + full = spec .. path .. '/' else - local forcedname, ok = "", false - if file.extname(filename) == "" then - if instance.format == "" then - forcedname = filename .. ".tex" - if input.is_readable.file(forcedname) then - if input.trace > 0 then - input.logger('! no suffix, forcing standard filetype: tex') - end - result, ok = { forcedname }, true - end - else - for _, s in pairs(input.suffixes_of_format(instance.format)) do - forcedname = filename .. "." .. s - if input.is_readable.file(forcedname) then - if input.trace > 0 then - input.logger('! no suffix, forcing format filetype: %s', s) + full = spec + end + for name in directory(full) do + if not weird:match(name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + if path then + n = n + 1 + local f = files[name] + if f then + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path + end + else -- probably unique anyway + files[name] = path + local lower = lower(name) + if name ~= lower then + files["remap:"..lower] = name + r = r + 1 end - result, ok = { forcedname }, true - break end end + elseif mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) + else + action(name) + end end end - if not ok and input.trace > 0 then - input.logger('? qualified: %s', filename) - end end - else - -- search spec - local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) - if ext == "" then - if not instance.force_suffixes then - wantedfiles[#wantedfiles+1] = filename - end - else - wantedfiles[#wantedfiles+1] = filename - end - if instance.format == "" then - if ext == "" then - local forcedname = filename .. '.tex' - wantedfiles[#wantedfiles+1] = forcedname - filetype = input.format_of_suffix(forcedname) - if input.trace > 0 then - input.logger('! forcing filetype: %s',filetype) - end - else - filetype = input.format_of_suffix(filename) - if input.trace > 0 then - input.logger('! using suffix based filetype: %s',filetype) + end + action() + if trace_verbose then + logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) + end +end + +-- savers, todo + +function resolvers.savefiles() + resolvers.save_data('files') +end + +-- A config (optionally) has the paths split in tables. Internally +-- we join them and split them after the expansion has taken place. This +-- is more convenient. + +function resolvers.splitconfig() + for i,c in ipairs(instance) do + for k,v in pairs(c) do + if type(v) == 'string' then + local t = file.split_path(v) + if #t > 1 then + c[k] = t end end - else - if ext == "" then - for _, s in pairs(input.suffixes_of_format(instance.format)) do - wantedfiles[#wantedfiles+1] = filename .. "." .. s - end + end + end +end + +function resolvers.joinconfig() + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do -- ipairs? + if type(v) == 'table' then + c[k] = file.join_path(v) end - filetype = instance.format - if input.trace > 0 then - input.logger('! using given filetype: %s',filetype) + end + end +end +function resolvers.split_path(str) + if type(str) == 'table' then + return str + else + return file.split_path(str) + end +end +function resolvers.join_path(str) + if type(str) == 'table' then + return file.join_path(str) + else + return str + end +end + +function resolvers.splitexpansions() + local ie = instance.expansions + for k,v in next, ie do + local t, h = { }, { } + for _,vv in ipairs(file.split_path(v)) do + if vv ~= "" and not h[vv] then + t[#t+1] = vv + h[vv] = true end end - local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(typespec) - if not pathlist or #pathlist == 0 then - -- no pathlist, access check only / todo == wildcard - if input.trace > 2 then - input.logger('? filename: %s',filename) - input.logger('? filetype: %s',filetype or '?') - input.logger('? wanted files: %s',concat(wantedfiles," | ")) - end - for _, fname in pairs(wantedfiles) do - if fname and input.is_readable.file(fname) then - filename, done = fname, true - result[#result+1] = file.join('.',fname) - break - end - end - -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(wantedfiles) - local fl = filelist and filelist[1] - if fl then - filename = fl[3] - result[#result+1] = filename - done = true - end + if #t > 1 then + ie[k] = t else - -- list search - local filelist = input.aux.collect_files(wantedfiles) - local doscan, recurse - if input.trace > 2 then - input.logger('? filename: %s',filename) - -- if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end - -- if filelist then input.logger('? file list: %s',concat(filelist," | ")) end - end - -- a bit messy ... esp the doscan setting here - for _, path in pairs(pathlist) do - if path:find("^!!") then doscan = false else doscan = true end - if path:find("//$") then recurse = true else recurse = false end - local pathname = path:gsub("^!+", '') - done = false - -- using file list - if filelist and not (done and not instance.allresults) and recurse then - -- compare list entries with permitted pattern - pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences - pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless - local expr = "^" .. pathname - -- input.debug('?',expr) - for _, fl in ipairs(filelist) do - local f = fl[2] - if f:find(expr) then - -- input.debug('T',' '..f) - if input.trace > 2 then - input.logger('= found in hash: %s',f) - end - --- todo, test for readable - result[#result+1] = fl[3] - input.aux.register_in_trees(f) -- for tracing used files - done = true - if not instance.allresults then break end - else - -- input.debug('F',' '..f) - end - end - end - if not done and doscan then - -- check if on disk / unchecked / does not work at all / also zips - if input.method_is_file(pathname) then -- ? - local pname = pathname:gsub("%.%*$",'') - if not pname:find("%*") then - local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(ppname) then - for _, w in pairs(wantedfiles) do - local fname = file.join(ppname,w) - if input.is_readable.file(fname) then - if input.trace > 2 then - input.logger('= found by scanning: %s',fname) - end - result[#result+1] = fname - done = true - if not instance.allresults then break end - end - end - else - -- no access needed for non existing path, speedup (esp in large tree with lots of fake) - end - end - end - end - if not done and doscan then - -- todo: slow path scanning - end - if done and not instance.allresults then break end - end + ie[k] = t[1] end end - for k,v in pairs(result) do - result[k] = file.collapse_path(v) - end - if instance.remember then - instance.found[stamp] = result - end - return result end -input.aux._find_file_ = input.aux.find_file -- frozen variant +-- end of split/join code -function input.aux.find_file(filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(filename) - if #result == 0 then - local lowered = filename:lower() - if filename ~= lowered then - return input.aux._find_file_(lowered) - end - end - return result +function resolvers.saveoldconfig() + resolvers.splitconfig() + resolvers.save_data('configuration') + resolvers.joinconfig() end -function input.aux.can_be_dir(name) - local instance = input.instance - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory +resolvers.configbanner = [[ +-- This is a Luatex configuration file created by 'luatools.lua' or +-- 'luatex.exe' directly. For comment, suggestions and questions you can +-- contact the ConTeXt Development Team. This configuration file is +-- not copyrighted. [HH & TH] +]] + +function resolvers.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local function dump(k,v,m) -- could be moved inline + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," else - instance.fakepaths[name] = 2 -- no directory + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," end end - return (instance.fakepaths[name] == 1) -end - -if not input.concatinators then input.concatinators = { } end - -input.concatinators.tex = file.join -input.concatinators.file = input.concatinators.tex - -function input.find_files(filename,filetype,mustexist) - local instance = input.instance - if type(mustexist) == boolean then - -- all set - elseif type(filetype) == 'boolean' then - filetype, mustexist = nil, false - elseif type(filetype) ~= 'string' then - filetype, mustexist = nil, false - end - instance.format = filetype or '' - local t = input.aux.find_file(filename,true) - instance.format = '' - return t -end - -function input.find_file(filename,filetype,mustexist) - return (input.find_files(filename,filetype,mustexist)[1] or "") -end - -function input.find_given_files(filename) - local instance = input.instance - local bname, result = file.basename(filename), { } - for k, hash in ipairs(instance.hashes) do - local files = instance.files[hash.tag] - local blist = files[bname] - if not blist then - local rname = "remap:"..bname - blist = files[rname] - if blist then - bname = files[rname] - blist = files[bname] + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sortedkeys(files)) do -- ipairs + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sortedkeys(fk)) do -- ipairs + t[#t+1] = dump(kk,fk[kk],"\t\t") + end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") end end - if blist then - if type(blist) == 'string' then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" - if not instance.allresults then break end - else - for kk,vv in pairs(blist) do - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" - if not instance.allresults then break end + else + for k, v in next, files do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in next, v do + t[#t+1] = dump(kk,vv,"\t\t") end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,v,"\t") end end end - return result -end - -function input.find_given_file(filename) - return (input.find_given_files(filename)[1] or "") + t[#t+1] = "}" + return concat(t,"\n") end -function input.find_wildcard_files(filename) -- todo: remap: - local instance = input.instance - local result = { } - local bname, dname = file.basename(filename), file.dirname(filename) - local path = dname:gsub("^*/","") - path = path:gsub("*",".*") - path = path:gsub("-","%%-") - if dname == "" then - path = ".*" - end - local name = bname - name = name:gsub("*",".*") - name = name:gsub("-","%%-") - path = path:lower() - name = name:lower() - local function doit(blist,bname,hash,allresults) - local done = false - if blist then - if type(blist) == 'string' then - -- make function and share code - if (blist:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" - done = true - end - else - for kk,vv in pairs(blist) do - if (vv:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" - done = true - if not allresults then break end - end - end +function resolvers.save_data(dataname, makename) -- untested without cache overload + for cachename, files in next, instance[dataname] do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. ".lua", name .. ".luc" + if trace_verbose then + logs.report("fileio","preparing %s for %s",dataname,cachename) + end + for k, v in next, files do + if type(v) == "table" and #v == 1 then + files[k] = v[1] end end - return done - end - local files, allresults, done = instance.files, instance.allresults, false - if name:find("%*") then - for k, hash in ipairs(instance.hashes) do - for kk, hh in pairs(files[hash.tag]) do - if not kk:find("^remap:") then - if (kk:lower()):find(name) then - if doit(hh,kk,hash,allresults) then done = true end - if done and not allresults then break end - end + local data = { + type = dataname, + root = cachename, + version = resolvers.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local ok = io.savedata(luaname,resolvers.serialize(data)) + if ok then + if trace_verbose then + logs.report("fileio","%s saved in %s",dataname,luaname) + end + if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip + if trace_verbose then + logs.report("fileio","%s compiled to %s",dataname,lucname) end + else + if trace_verbose then + logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + end + os.remove(lucname) end - end - else - for k, hash in ipairs(instance.hashes) do - if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end - if done and not allresults then break end + elseif trace_verbose then + logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) end end - -- we can consider also searching the paths not in the database, but then - -- we end up with a messy search (all // in all path specs) - return result -end - -function input.find_wildcard_file(filename) - return (input.find_wildcard_files(filename)[1] or "") end --- main user functions - -function input.save_used_files_in_trees(filename,jobname) - local instance = input.instance - if not filename then filename = 'luatex.jlg' end - local f = io.open(filename,'w') - if f then - f:write("\n") - f:write("\n") - if jobname then - f:write("\t" .. jobname .. "\n") - end - f:write("\t\n") - for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs - f:write("\t\t" .. v .. "\n") +function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then + if trace_verbose then + logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + end + instance[dataname][pathname] = data.content + else + if trace_verbose then + logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + end + instance[dataname][pathname] = { } + instance.loaderror = true end - f:write("\t\n") - f:write("\n") - f:close() + elseif trace_verbose then + logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) end end -function input.automount() - -- implemented later -end +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } -function input.load() - input.starttiming(input.instance) - input.resetconfig() - input.identify_cnf() - input.load_lua() - input.expand_variables() - input.load_cnf() - input.expand_variables() - input.load_hash() - input.automount() - input.stoptiming(input.instance) +function resolvers.resetconfig() + identify_own() + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false end -function input.for_files(command, files, filetype, mustexist) - if files and #files > 0 then - local function report(str) - if input.verbose then - input.report(str) -- has already verbose - else - print(str) - end - end - if input.verbose then - report('') - end - for _, file in pairs(files) do - local result = command(file,filetype,mustexist) - if type(result) == 'string' then - report(result) +function resolvers.loadnewconfig() + for _, cnf in ipairs(instance.luafiles) do + local pathname = file.dirname(cnf) + local filename = file.join(pathname,resolvers.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + if trace_verbose then + logs.report("fileio","loading configuration file %s",filename) + end + if true then + -- flatten to variable.progname + local t = { } + for k, v in next, data do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in next, v do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance['setup'][pathname] = t + else + instance['setup'][pathname] = data + end else - for _,v in pairs(result) do - report(v) + if trace_verbose then + logs.report("fileio","skipping configuration file %s",filename) end + instance['setup'][pathname] = { } + instance.loaderror = true end + elseif trace_verbose then + logs.report("fileio","skipping configuration file %s",filename) end + instance.order[#instance.order+1] = instance.setup[pathname] + if instance.loaderror then break end end end --- strtab - -input.var_value = input.variable -- output the value of variable $STRING. -input.expand_var = input.expansion -- output variable expansion of STRING. - -function input.show_path(str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(input.format_of_var(str))) +function resolvers.loadoldconfig() + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + resolvers.load_data(dname,'configuration') + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end + end + resolvers.joinconfig() end --- input.find_file(filename) --- input.find_file(filename, filetype, mustexist) --- input.find_file(filename, mustexist) --- input.find_file(filename, filetype) - -function input.aux.register_file(files, name, path) - if files[name] then - if type(files[name]) == 'string' then - files[name] = { files[name], path } +function resolvers.expand_variables() + local expansions, environment, variables = { }, instance.environment, instance.variables + local env = resolvers.env + instance.expansions = expansions + if instance.engine ~= "" then environment['engine'] = instance.engine end + if instance.progname ~= "" then environment['progname'] = instance.progname end + for k,v in next, environment do + local a, b = match(k,"^(%a+)%_(.*)%s*$") + if a and b then + expansions[a..'.'..b] = v else - files[name] = path + expansions[k] = v end - else - files[name] = path end -end - -if not input.finders then input.finders = { } end -if not input.openers then input.openers = { } end -if not input.loaders then input.loaders = { } end - -input.finders.notfound = { nil } -input.openers.notfound = { nil } -input.loaders.notfound = { false, nil, 0 } - -function input.splitmethod(filename) - if not filename then - return { } -- safeguard - elseif type(filename) == "table" then - return filename -- already split - elseif not filename:find("://") then - return { scheme="file", path = filename, original=filename } -- quick hack - else - return url.hashed(filename) + for k,v in next, environment do -- move environment to expansions + if not expansions[k] then expansions[k] = v end end -end - -function input.method_is_file(filename) - return input.splitmethod(filename).scheme == 'file' -end - -function table.sequenced(t,sep) -- temp here - local s = { } - for k, v in pairs(t) do - s[#s+1] = k .. "=" .. v + for k,v in next, variables do -- move variables to expansions + if not expansions[k] then expansions[k] = v end end - return concat(s, sep or " | ") -end - -function input.methodhandler(what, filename, filetype) -- ... - local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb - local scheme = specification.scheme - if input[what][scheme] then - if input.trace > 0 then - input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + local busy = false + local function resolve(a) + busy = true + return expansions[a] or env(a) + end + while true do + busy = false + for k,v in next, expansions do + local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve) + local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve) + if n > 0 or m > 0 then + expansions[k]= s + end end - return input[what][scheme](filename,filetype) -- todo: specification - else - return input[what].tex(filename,filetype) -- todo: specification + if not busy then break end end -end - --- also inside next test? - -function input.findtexfile(filename, filetype) - return input.methodhandler('finders',input.normalize_name(filename), filetype) -end -function input.opentexfile(filename) - return input.methodhandler('openers',input.normalize_name(filename)) -end - -function input.findbinfile(filename, filetype) - return input.methodhandler('finders',input.normalize_name(filename), filetype) -end -function input.openbinfile(filename) - return input.methodhandler('loaders',input.normalize_name(filename)) -end - -function input.loadbinfile(filename, filetype) - local fname = input.findbinfile(input.normalize_name(filename), filetype) - if fname and fname ~= "" then - return input.openbinfile(fname) - else - return unpack(input.loaders.notfound) + for k,v in next, expansions do + expansions[k] = gsub(v,"\\", '/') end end -function input.texdatablob(filename, filetype) - local ok, data, size = input.loadbinfile(filename, filetype) - return data or "" +function resolvers.variable(name) + return entry(instance.variables,name) end -input.loadtexfile = input.texdatablob - -function input.openfile(filename) - local fullname = input.findtexfile(filename) - if fullname and (fullname ~= "") then - return input.opentexfile(fullname) - else - return nil - end +function resolvers.expansion(name) + return entry(instance.expansions,name) end -function input.logmode() - return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() +function resolvers.is_variable(name) + return is_entry(instance.variables,name) end --- this is a prelude to engine/progname specific configuration files --- in which case we can omit files meant for other programs and --- packages - ---- ctx - --- maybe texinputs + font paths --- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc - -input.validators = { } -input.validators.visibility = { } - -function input.validators.visibility.default(path, name) - return true +function resolvers.is_expansion(name) + return is_entry(instance.expansions,name) end -function input.validators.visibility.context(path, name) - path = path[1] or path -- some day a loop - return not ( - path:find("latex") or --- path:find("doc") or - path:find("tex4ht") or - path:find("source") or --- path:find("config") or --- path:find("metafont") or - path:find("lists$") or - name:find("%.tpm$") or - name:find("%.bak$") - ) +function resolvers.unexpanded_path_list(str) + local pth = resolvers.variable(str) + local lst = resolvers.split_path(pth) + return expanded_path_from_list(lst) end --- todo: describe which functions are public (maybe input.private. ... ) +function resolvers.unexpanded_path(str) + return file.join_path(resolvers.unexpanded_path_list(str)) +end --- beware: i need to check where we still need a / on windows: +do -- no longer needed -function input.clean_path(str) - if str then - str = str:gsub("\\","/") - str = str:gsub("^!+","") - str = str:gsub("^~",input.homedir) - return str - else - return nil - end -end + local done = { } -function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(name)) do - func("^"..input.clean_path(v)) + function resolvers.reset_extra_path() + local ep = instance.extra_paths + if not ep then + ep, done = { }, { } + instance.extra_paths = ep + elseif #ep > 0 then + instance.lists, done = { }, { } + end end -end - -function input.do_with_var(name,func) - func(input.aux.expanded_var(name)) -end -function input.with_files(pattern,handle) - local instance = input.instance - for _, hash in ipairs(instance.hashes) do - local blobpath = hash.tag - local blobtype = hash.type - if blobpath then - local files = instance.files[blobpath] - if files then - for k,v in pairs(files) do - if k:find("^remap:") then - k = files[k] - v = files[k] -- chained - end - if k:find(pattern) then - if type(v) == "string" then - handle(blobtype,blobpath,v,k) - else - for _,vv in pairs(v) do - handle(blobtype,blobpath,vv,k) - end + function resolvers.register_extra_path(paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep + if paths and paths ~= "" then + if subpaths and subpaths ~= "" then + for p in gmatch(paths,"[^,]+") do + -- we gmatch each step again, not that fast, but used seldom + for s in gmatch(subpaths,"[^,]+") do + local ps = p .. "/" .. s + if not done[ps] then + ep[#ep+1] = resolvers.clean_path(ps) + done[ps] = true end end end - end - end - end -end - -function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = input.clean_path(oldname) - input.report("to be replaced old script %s", oldscript) - local newscripts = input.find_files(newname) or { } - if #newscripts == 0 then - input.report("unable to locate new script") - else - for _, newscript in ipairs(newscripts) do - newscript = input.clean_path(newscript) - input.report("checking new script %s", newscript) - if oldscript == newscript then - input.report("old and new script are the same") - elseif not newscript:find(scriptpath) then - input.report("new script should come from %s",scriptpath) - elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then - input.report("invalid new script name") else - local newdata = io.loaddata(newscript) - if newdata then - input.report("old script content replaced by new content") - io.savedata(oldscript,newdata) - break - else - input.report("unable to load new script") + for p in gmatch(paths,"[^,]+") do + if not done[p] then + ep[#ep+1] = resolvers.clean_path(p) + done[p] = true + end + end + end + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in gmatch(subpaths,"[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = resolvers.clean_path(ps) + done[ps] = true + end end end end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end end -end - ---~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) - --- command line resolver: - ---~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) - -do - - local resolvers = { } +end - resolvers.environment = function(str) - return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") - end - resolvers.relative = function(str,n) - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break +local function made_list(instance,list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k=1,#list do + local v = list[k] + if not done[v] then + if find(v,"^[%.%/]$") then + done[v] = true + new[#new+1] = v else - p = p .. "../" + break end end end - return input.clean_path(str) - end - resolvers.locate = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path((fullname ~= "" and fullname) or str) - end - resolvers.filename = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) - end - resolvers.pathname = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) - end - - resolvers.env = resolvers.environment - resolvers.rel = resolvers.relative - resolvers.loc = resolvers.locate - resolvers.kpse = resolvers.locate - resolvers.full = resolvers.locate - resolvers.file = resolvers.filename - resolvers.path = resolvers.pathname - - local function resolve(str) - if type(str) == "table" then - for k, v in pairs(str) do - str[k] = resolve(v) or v + -- first the extra paths + for k=1,#ep do + local v = ep[k] + if not done[v] then + done[v] = true + new[#new+1] = v end - elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target) - if resolvers[method] then - return resolvers[method](target) - else - return method .. ":" .. target - end - end) end - return str - end - - if os.uname then - for k, v in pairs(os.uname()) do - if not resolvers[k] then - resolvers[k] = function() return v end + -- next the formal paths + for k=1,#list do + local v = list[k] + if not done[v] then + done[v] = true + new[#new+1] = v end end - end - - input.resolve = resolve - -end - -function input.boolean_variable(str,default) - local b = input.expansion(str) - if b == "" then - return default - else - b = toboolean(b) - return (b == nil and default) or b - end -end - - -if not modules then modules = { } end modules ['luat-log'] = { - version = 1.001, - comment = "companion to luat-lib.tex", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -

This is a prelude to a more extensive logging module. For the sake -of parsing log files, in addition to the standard logging we will -provide an structured file. Actually, any logging that -is hooked into callbacks will be \XML\ by default.

---ldx]]-- - --- input.logger -> special tracing, driven by log level (only input) --- input.report -> goes to terminal, depends on verbose, has banner --- logs.report -> module specific tracing and reporting, no banner but class - - -input = input or { } -logs = logs or { } - ---[[ldx-- -

This looks pretty ugly but we need to speed things up a bit.

---ldx]]-- - -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4 -} - -logs.functions = { - 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' -} - -logs.callbacks = { - 'start_page_number', - 'stop_page_number', - 'report_output_pages', - 'report_output_log' -} - -logs.tracers = { -} - -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } - -logs.level = 0 - -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format - -if texlua then - write_nl = print - write = io.write -end - -function logs.xml.report(category,fmt,...) -- new - write_nl(format("%s",category,format(fmt,...))) -end -function logs.xml.line(fmt,...) -- new - write_nl(format("%s",format(fmt,...))) -end - -function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end -function logs.xml.stop () if logs.level > 0 then tw("") end end -function logs.xml.push () if logs.level > 0 then tw("" ) end end - -function logs.tex.report(category,fmt,...) -- new - -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . - write_nl(category .. " | " .. format(fmt,...)) + return new + end end -function logs.tex.line(fmt,...) -- new - write_nl(format(fmt,...)) + +function resolvers.clean_path_list(str) + local t = resolvers.expanded_path_list(str) + if t then + for i=1,#t do + t[i] = file.collapse_path(resolvers.clean_path(t[i])) + end + end + return t end -function logs.set_level(level) - logs.level = logs.levels[level] or level +function resolvers.expand_path(str) + return file.join_path(resolvers.expanded_path_list(str)) end -function logs.set_method(method) - for _, v in pairs(logs.functions) do - logs[v] = logs[method][v] or function() end - end - if callback and input[method] then - for _, cb in pairs(logs.callbacks) do - callback.register(cb, input[method][cb]) +function resolvers.expanded_path_list(str) + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = gsub(str,"%$","") + if not instance.lists[str] then -- cached + local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str))) + instance.lists[str] = expanded_path_from_list(lst) end + return instance.lists[str] + else + local lst = resolvers.split_path(resolvers.expansion(str)) + return made_list(instance,expanded_path_from_list(lst)) end end -function logs.xml.start_page_number() - write_nl(format("

") - write_nl("") +function resolvers.expand_path_from_var(str) + return file.join_path(resolvers.expanded_path_list_from_var(str)) end -function logs.xml.report_output_pages(p,b) - write_nl(format("", p)) - write_nl(format("", b)) - write_nl("") +function resolvers.format_of_var(str) + return formats[str] or formats[alternatives[str]] or '' end - -function logs.xml.report_output_log() +function resolvers.format_of_suffix(str) + return suffixmap[file.extname(str)] or 'tex' end -function input.logger(...) -- assumes test for input.trace > n - if input.trace > 0 then - logs.report(...) - end +function resolvers.variable_of_format(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.report(fmt,...) - if input.verbose then - logs.report(input.banner or "report",format(fmt,...)) +function resolvers.var_of_format_or_suffix(str) + local v = formats[str] + if v then + return v end -end - -function input.reportlines(str) -- todo: - for line in str:gmatch("(.-)[\n\r]") do - logs.report(input.banner or "report",line) + v = formats[alternatives[str]] + if v then + return v + end + v = suffixmap[file.extname(str)] + if v then + return formats[isf] end + return '' end -input.moreinfo = [[ -more information about ConTeXt and the tools that come with it can be found at: +function resolvers.expand_braces(str) -- output variable and brace expansion of STRING + local ori = resolvers.variable(str) + local pth = expanded_path_from_list(resolvers.split_path(ori)) + return file.join_path(pth) +end -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] +resolvers.isreadable = { } -function input.help(banner,message) - if not input.verbose then - input.verbose = true - -- input.report(banner,"\n") - end - input.report(banner,"\n") - input.report("") - input.reportlines(message) - if input.moreinfo and input.moreinfo ~= "" then - input.report("") - input.reportlines(input.moreinfo) +function resolvers.isreadable.file(name) + local readable = lfs.isfile(name) -- brrr + if trace_detail then + if readable then + logs.report("fileio","+ readable: %s",name) + else + logs.report("fileio","- readable: %s", name) + end end + return readable end -logs.set_level('error') -logs.set_method('tex') - - -if not modules then modules = { } end modules ['luat-tmp'] = { - version = 1.001, - comment = "companion to luat-lib.tex", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -

This module deals with caching data. It sets up the paths and -implements loaders and savers for tables. Best is to set the -following variable. When not set, the usual paths will be -checked. Personally I prefer the (users) temporary path.

- - -TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. - - -

Currently we do no locking when we write files. This is no real -problem because most caching involves fonts and the chance of them -being written at the same time is small. We also need to extend -luatools with a recache feature.

---ldx]]-- - -local format = string.format - -caches = caches or { } -dir = dir or { } -texmf = texmf or { } +resolvers.isreadable.tex = resolvers.isreadable.file -caches.path = caches.path or nil -caches.base = caches.base or "luatex-cache" -caches.more = caches.more or "context" -caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false -caches.tree = false -caches.paths = caches.paths or nil -caches.force = false -caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } +-- name +-- name/name -function caches.temp() - local cachepath = nil - local function check(list,isenv) - if not cachepath then - for _, v in ipairs(list) do - cachepath = (isenv and (os.env[v] or "")) or v or "" - if cachepath == "" then - -- next - else - cachepath = input.clean_path(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" - break - elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then - break +local function collect_files(names) + local filelist = { } + for k=1,#names do + local fname = names[k] + if trace_detail then + logs.report("fileio","? blobpath asked: %s",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or find(dname,"^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + local hashes = instance.hashes + for h=1,#hashes do + local hash = hashes[h] + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if trace_detail then + logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] + end + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or find(blobfile,dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result + } + end + else + for kk=1,#blobfile do + local vv = blobfile[kk] + if not dname or find(vv,dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + resolvers.concatinators[hash.type](blobpath,vv,bname) -- result + } + end end end end - cachepath = nil + elseif trace_locating then + logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) end end end - check(input.clean_path_list("TEXMFCACHE") or { }) - check(caches.defaults,true) - if not cachepath then - print("\nfatal error: there is no valid (writable) cache path defined\n") - os.exit() - elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" - print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) - os.exit() - end - cachepath = input.normalize_name(cachepath) - function caches.temp() - return cachepath + if #filelist > 0 then + return filelist + else + return nil end - return cachepath end -function caches.configpath() - return table.concat(input.instance.cnffiles,";") +function resolvers.suffix_of_format(str) + if suffixes[str] then + return suffixes[str][1] + else + return "" + end end -function caches.hashed(tree) - return md5.hex((tree:lower()):gsub("[\\\/]+","/")) +function resolvers.suffixes_of_format(str) + if suffixes[str] then + return suffixes[str] + else + return {} + end end ---~ tracing: +function resolvers.register_in_trees(name) + if not find(name,"^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one + end +end ---~ function caches.hashed(tree) ---~ tree = (tree:lower()):gsub("[\\\/]+","/") ---~ local hash = md5.hex(tree) ---~ if input.verbose then -- temp message ---~ input.report("hashing %s => %s",tree,hash) ---~ end ---~ return hash ---~ end +-- split the next one up for readability (bu this module needs a cleanup anyway) -function caches.treehash() - local tree = caches.configpath() - if not tree or tree == "" then - return false - else - return caches.hashed(tree) +local function can_be_dir(name) -- can become local + local fakepaths = instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name] = 1 -- directory + else + fakepaths[name] = 2 -- no directory + end end + return (fakepaths[name] == 1) end -function caches.setpath(...) - if not caches.path then - if not caches.path then - caches.path = caches.temp() +local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) + local result = collected or { } + local stamp = nil + filename = file.collapse_path(filename) -- elsewhere + filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere + -- speed up / beware: format problem + if instance.remember then + stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format + if instance.found[stamp] then + if trace_locating then + logs.report("fileio",'! remembered: %s',filename) + end + return instance.found[stamp] + end + end + if not dangerous[instance.format or "?"] then + if resolvers.isreadable.file(filename) then + if trace_detail then + logs.report("fileio",'= found directly: %s',filename) + end + instance.found[stamp] = { filename } + return { filename } + end + end + if find(filename,'%*') then + if trace_locating then + logs.report("fileio",'! wildcard: %s', filename) + end + result = resolvers.find_wildcard_files(filename) + elseif file.is_qualified_path(filename) then + if resolvers.isreadable.file(filename) then + if trace_locating then + logs.report("fileio",'! qualified: %s', filename) + end + result = { filename } + else + local forcedname, ok, suffix = "", false, file.extname(filename) + if suffix == "" then -- why + if instance.format == "" then + forcedname = filename .. ".tex" + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio",'! no suffix, forcing standard filetype: tex') + end + result, ok = { forcedname }, true + end + else + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do + forcedname = filename .. "." .. s + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio",'! no suffix, forcing format filetype: %s', s) + end + result, ok = { forcedname }, true + break + end + end + end + end + if not ok and suffix ~= "" then + -- try to find in tree (no suffix manipulation), here we search for the + -- matching last part of the name + local basename = file.basename(filename) + local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local savedformat = instance.format + local format = savedformat or "" + if format == "" then + instance.format = resolvers.format_of_suffix(suffix) + end + if not format then + instance.format = "othertextfiles" -- kind of everything, maybe texinput is better + end + -- + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end + end + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if rr:find(pattern) then + result[#result+1], ok = rr, true + end + end + -- a real wildcard: + -- + -- if not ok then + -- local filelist = collect_files({basename}) + -- for f=1,#filelist do + -- local ff = filelist[f][3] or "" + -- if ff:find(pattern) then + -- result[#result+1], ok = ff, true + -- end + -- end + -- end + end + if not ok and trace_locating then + logs.report("fileio",'? qualified: %s', filename) + end + end + else + -- search spec + local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename) + if ext == "" then + if not instance.force_suffixes then + wantedfiles[#wantedfiles+1] = filename + end + else + wantedfiles[#wantedfiles+1] = filename end - caches.path = input.clean_path(caches.path) -- to be sure - if lfs then - caches.tree = caches.tree or caches.treehash() - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + if instance.format == "" then + if ext == "" then + local forcedname = filename .. '.tex' + wantedfiles[#wantedfiles+1] = forcedname + filetype = resolvers.format_of_suffix(forcedname) + if trace_locating then + logs.report("fileio",'! forcing filetype: %s',filetype) + end else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) + filetype = resolvers.format_of_suffix(filename) + if trace_locating then + logs.report("fileio",'! using suffix based filetype: %s',filetype) + end + end + else + if ext == "" then + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do + wantedfiles[#wantedfiles+1] = filename .. "." .. s + end + end + filetype = instance.format + if trace_locating then + logs.report("fileio",'! using given filetype: %s',filetype) + end + end + local typespec = resolvers.variable_of_format(filetype) + local pathlist = resolvers.expanded_path_list(typespec) + if not pathlist or #pathlist == 0 then + -- no pathlist, access check only / todo == wildcard + if trace_detail then + logs.report("fileio",'? filename: %s',filename) + logs.report("fileio",'? filetype: %s',filetype or '?') + logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + end + for k=1,#wantedfiles do + local fname = wantedfiles[k] + if fname and resolvers.isreadable.file(fname) then + filename, done = fname, true + result[#result+1] = file.join('.',fname) + break + end + end + -- this is actually 'other text files' or 'any' or 'whatever' + local filelist = collect_files(wantedfiles) + local fl = filelist and filelist[1] + if fl then + filename = fl[3] + result[#result+1] = filename + done = true + end + else + -- list search + local filelist = collect_files(wantedfiles) + local doscan, recurse + if trace_detail then + logs.report("fileio",'? filename: %s',filename) + end + -- a bit messy ... esp the doscan setting here + for k=1,#pathlist do + local path = pathlist[k] + if find(path,"^!!") then doscan = false else doscan = true end + if find(path,"//$") then recurse = true else recurse = false end + local pathname = gsub(path,"^!+", '') + done = false + -- using file list + if filelist and not (done and not instance.allresults) and recurse then + -- compare list entries with permitted pattern + pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences + pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname + pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless + local expr = "^" .. pathname + for k=1,#filelist do + local fl = filelist[k] + local f = fl[2] + if find(f,expr) then + if trace_detail then + logs.report("fileio",'= found in hash: %s',f) + end + --- todo, test for readable + result[#result+1] = fl[3] + resolvers.register_in_trees(f) -- for tracing used files + done = true + if not instance.allresults then break end + end + end + end + if not done and doscan then + -- check if on disk / unchecked / does not work at all / also zips + if resolvers.splitmethod(pathname).scheme == 'file' then -- ? + local pname = gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + local ppname = gsub(pname,"/+$","") + if can_be_dir(ppname) then + for k=1,#wantedfiles do + local w = wantedfiles[k] + local fname = file.join(ppname,w) + if resolvers.isreadable.file(fname) then + if trace_detail then + logs.report("fileio",'= found by scanning: %s',fname) + end + result[#result+1] = fname + done = true + if not instance.allresults then break end + end + end + else + -- no access needed for non existing path, speedup (esp in large tree with lots of fake) + end + end + end + end + if not done and doscan then + -- todo: slow path scanning + end + if done and not instance.allresults then break end end end end - if not caches.path then - caches.path = '.' - end - caches.path = input.clean_path(caches.path) - if lfs and not table.is_empty({...}) then - local pth = dir.mkdirs(caches.path,...) - return pth + for k=1,#result do + result[k] = file.collapse_path(result[k]) end - caches.path = dir.expand_name(caches.path) - return caches.path -end - -function caches.definepath(category,subcategory) - return function() - return caches.setpath(category,subcategory) + if instance.remember then + instance.found[stamp] = result end + return result end -function caches.setluanames(path,name) - return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" -end - -function caches.loaddata(path,name) - local tmaname, tmcname = caches.setluanames(path,name) - local loader = loadfile(tmcname) or loadfile(tmaname) - if loader then - return loader() - else - return false - end -end +if not resolvers.concatinators then resolvers.concatinators = { } end -function caches.is_writable(filepath,filename) - local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.is_writable(tmaname) -end +resolvers.concatinators.tex = file.join +resolvers.concatinators.file = resolvers.concatinators.tex -function caches.savedata(filepath,filename,data,raw) - local tmaname, tmcname = caches.setluanames(filepath,filename) - local reduce, simplify = true, true - if raw then - reduce, simplify = false, false +function resolvers.find_files(filename,filetype,mustexist) + if type(mustexist) == boolean then + -- all set + elseif type(filetype) == 'boolean' then + filetype, mustexist = nil, false + elseif type(filetype) ~= 'string' then + filetype, mustexist = nil, false end - if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true,false)) -- no hex - else - table.tofile(tmaname, data,'return',true,true,false) -- maybe not the last true + instance.format = filetype or '' + local result = collect_instance_files(filename) + if #result == 0 then + local lowered = lower(filename) + if filename ~= lowered then + return collect_instance_files(lowered) + end end - local cleanup = input.boolean_variable("PURGECACHE", false) - local strip = input.boolean_variable("LUACSTRIP", true) - utils.lua.compile(tmaname, tmcname, cleanup, strip) + instance.format = '' + return result end --- here we use the cache for format loading (texconfig.[formatname|jobname]) - ---~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and input.instance then - if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath("formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") +function resolvers.find_file(filename,filetype,mustexist) + return (resolvers.find_files(filename,filetype,mustexist)[1] or "") end ---[[ldx-- -

Once we found ourselves defining similar cache constructs -several times, containers were introduced. Containers are used -to collect tables in memory and reuse them when possible based -on (unique) hashes (to be provided by the calling function).

- -

Caching to disk is disabled by default. Version numbers are -stored in the saved table which makes it possible to change the -table structures without bothering about the disk cache.

- -

Examples of usage can be found in the font related code.

---ldx]]-- - -containers = { } -containers.trace = false - -do -- local report - - local function report(container,tag,name) - if caches.trace or containers.trace or container.trace then - logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') - end - end - - local allocated = { } - - -- tracing - - function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil +function resolvers.find_given_files(filename) + local bname, result = file.basename(filename), { } + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local files = instance.files[hash.tag] + local blist = files[bname] + if not blist then + local rname = "remap:"..bname + blist = files[rname] + if blist then + bname = files[rname] + blist = files[bname] end end - end - - function containers.is_usable(container, name) - return container.enabled and caches.is_writable(container.path, name) - end - - function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version - else - return false - end - end - - function containers.read(container,name) - if container.enabled and not container.storage[name] then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then - report(container,"loaded",name) + if blist then + if type(blist) == 'string' then + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or "" + if not instance.allresults then break end else - container.storage[name] = nil - end - end - if container.storage[name] then - report(container,"reusing",name) - end - return container.storage[name] - end - - function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared + for kk=1,#blist do + local vv = blist[kk] + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or "" + if not instance.allresults then break end + end end - report(container,"stored",name) - container.storage[name] = data end - return data - end - - function containers.content(container,name) - return container.storage[name] end - + return result end --- since we want to use the cache instead of the tree, we will now --- reimplement the saver. - -local save_data = input.aux.save_data -local load_data = input.aux.load_data - -input.cachepath = nil -- public, for tracing -input.usecache = true -- public, for tracing - -function input.aux.save_data(dataname, check) - save_data(dataname, check, function(cachename,dataname) - input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) - if input.usecache then - input.cachepath = input.cachepath or caches.definepath("trees") - return file.join(input.cachepath(),caches.hashed(cachename)) - else - return file.join(cachename,dataname) - end - end) +function resolvers.find_given_file(filename) + return (resolvers.find_given_files(filename)[1] or "") end -function input.aux.load_data(pathname,dataname,filename) - load_data(pathname,dataname,filename,function(dataname,filename) - input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) - if input.usecache then - input.cachepath = input.cachepath or caches.definepath("trees") - return file.join(input.cachepath(),caches.hashed(pathname)) +local function doit(path,blist,bname,tag,kind,result,allresults) + local done = false + if blist and kind then + if type(blist) == 'string' then + -- make function and share code + if find(lower(blist),path) then + result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or "" + done = true + end else - if not filename or (filename == "") then - filename = dataname + for kk=1,#blist do + local vv = blist[kk] + if find(lower(vv),path) then + result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or "" + done = true + if not allresults then break end + end end - return file.join(pathname,filename) end - end) + end + return done end --- we will make a better format, maybe something xml or just text or lua - -input.automounted = input.automounted or { } - -function input.automount(usecache) - local mountpaths = input.clean_path_list(input.expansion('TEXMFMOUNT')) - if table.is_empty(mountpaths) and usecache then - mountpaths = { caches.setpath("mount") } +function resolvers.find_wildcard_files(filename) -- todo: remap: + local result = { } + local bname, dname = file.basename(filename), file.dirname(filename) + local path = gsub(dname,"^*/","") + path = gsub(path,"*",".*") + path = gsub(path,"-","%%-") + if dname == "" then + path = ".*" end - if not table.is_empty(mountpaths) then - input.starttiming(input.instance) - for k, root in pairs(mountpaths) do - local f = io.open(root.."/url.tmi") - if f then - for line in f:lines() do - if line then - if line:find("^[%%#%-]") then -- or %W - -- skip - elseif line:find("^zip://") then - input.report("mounting %s",line) - table.insert(input.automounted,line) - input.usezipfile(line) - end + local name = bname + name = gsub(name,"*",".*") + name = gsub(name,"-","%%-") + path = lower(path) + name = lower(name) + local files, allresults, done = instance.files, instance.allresults, false + if find(name,"%*") then + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + for kk, hh in next, files[hash.tag] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,tag,kind,result,allresults) then done = true end + if done and not allresults then break end end end - f:close() end end - input.stoptiming(input.instance) + else + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end + if done and not allresults then break end + end end + -- we can consider also searching the paths not in the database, but then + -- we end up with a messy search (all // in all path specs) + return result end --- store info in format +function resolvers.find_wildcard_file(filename) + return (resolvers.find_wildcard_files(filename)[1] or "") +end -input.storage = { } -input.storage.data = { } -input.storage.min = 0 -- 500 -input.storage.max = input.storage.min - 1 -input.storage.trace = false -- true -input.storage.done = input.storage.done or 0 -input.storage.evaluators = { } --- (evaluate,message,names) +-- main user functions -function input.storage.register(...) - input.storage.data[#input.storage.data+1] = { ... } +function resolvers.automount() + -- implemented later end -function input.storage.evaluate(name) - input.storage.evaluators[#input.storage.evaluators+1] = name +function resolvers.load(option) + statistics.starttiming(instance) + resolvers.resetconfig() + resolvers.identify_cnf() + resolvers.load_lua() + resolvers.expand_variables() + resolvers.load_cnf() + resolvers.expand_variables() + if option ~= "nofiles" then + resolvers.load_hash() + resolvers.automount() + end + statistics.stoptiming(instance) end -function input.storage.finalize() -- we can prepend the string with "evaluate:" - for _, t in ipairs(input.storage.evaluators) do - for i, v in pairs(t) do - if type(v) == "string" then - t[i] = loadstring(v)() - elseif type(v) == "table" then - for _, vv in pairs(v) do - if type(vv) == "string" then - t[i] = loadstring(vv)() - end - end +function resolvers.for_files(command, files, filetype, mustexist) + if files and #files > 0 then + local function report(str) + if trace_verbose then + logs.report("fileio",str) -- has already verbose + else + print(str) end end - end -end - -function input.storage.dump() - for name, data in ipairs(input.storage.data) do - local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] - local name, initialize, finalize, code = nil, "", "", "" - for str in target:gmatch("([^%.]+)") do - if name then - name = name .. "." .. str + if trace_verbose then + report('') + end + for _, file in ipairs(files) do + local result = command(file,filetype,mustexist) + if type(result) == 'string' then + report(result) else - name = str + for _,v in ipairs(result) do + report(v) + end end - initialize = format("%s %s = %s or {} ", initialize, name, name) - end - if evaluate then - finalize = "input.storage.evaluate(" .. name .. ")" - end - input.storage.max = input.storage.max + 1 - if input.storage.trace then - logs.report('storage','saving %s in slot %s',message,input.storage.max) - code = - initialize .. - format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. - table.serialize(original,name) .. - finalize - else - code = initialize .. table.serialize(original,name) .. finalize end - lua.bytecode[input.storage.max] = loadstring(code) - end -end - --- we also need to count at generation time (nicer for message) - -if lua.bytecode then -- from 0 upwards - local i = input.storage.min - while lua.bytecode[i] do - lua.bytecode[i]() - lua.bytecode[i] = nil - i = i + 1 end - input.storage.done = i end +-- strtab --- filename : luat-zip.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-zip'] = 1.001 - -local format = string.format +resolvers.var_value = resolvers.variable -- output the value of variable $STRING. +resolvers.expand_var = resolvers.expansion -- output variable expansion of STRING. -if zip and input then - zip.supported = true -else - zip = { } - zip.supported = false +function resolvers.show_path(str) -- output search path for file type NAME + return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str))) end -if not zip.supported then +-- resolvers.find_file(filename) +-- resolvers.find_file(filename, filetype, mustexist) +-- resolvers.find_file(filename, mustexist) +-- resolvers.find_file(filename, filetype) - if not input then input = { } end -- will go away - - function zip.openarchive (...) return nil end -- needed ? - function zip.closenarchive (...) end -- needed ? - function input.usezipfile (...) end -- needed ? - -else - - -- zip:///oeps.zip?name=bla/bla.tex - -- zip:///oeps.zip?tree=tex/texmf-local - - local function validzip(str) - if not str:find("^zip://") then - return "zip:///" .. str +function resolvers.register_file(files, name, path) + if files[name] then + if type(files[name]) == 'string' then + files[name] = { files[name], path } else - return str + files[name] = path end + else + files[name] = path end +end - zip.archives = { } - zip.registeredfiles = { } - - function zip.openarchive(name) - if not name or name == "" then - return nil - else - local arch = zip.archives[name] - if arch then - return arch - else - local full = input.find_file(name) or "" - local arch = (full ~= "" and zip.open(full)) or false - zip.archives[name] = arch - return arch - end - end +function resolvers.splitmethod(filename) + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not find(filename,"://") then + return { scheme="file", path = filename, original=filename } -- quick hack + else + return url.hashed(filename) end +end - function zip.closearchive(name) - if not name or name == "" and zip.archives[name] then - zip.close(zip.archives[name]) - zip.archives[name] = nil - end +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do -- pairs? + s[#s+1] = k .. "=" .. v end + return concat(s, sep or " | ") +end - -- zip:///texmf.zip?tree=/tex/texmf - -- zip:///texmf.zip?tree=/tex/texmf-local - -- zip:///texmf-mine.zip?tree=/tex/texmf-projects - - function input.locators.zip(specification) -- where is this used? startup zips (untested) - specification = input.splitmethod(specification) - local zipfile = specification.path - local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree - if input.trace > 0 then - if zfile then - input.logger('! zip locator, found: %s',specification.original) - else - input.logger('? zip locator, not found: %s',specification.original) - end +function resolvers.methodhandler(what, filename, filetype) -- ... + local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if resolvers[what][scheme] then + if trace_locating then + logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) end + return resolvers[what][scheme](filename,filetype) -- todo: specification + else + return resolvers[what].tex(filename,filetype) -- todo: specification end +end - function input.hashers.zip(tag,name) - input.report("loading zip file %s as %s",name,tag) - input.usezipfile(tag .."?tree=" .. name) +function resolvers.clean_path(str) + if str then + str = gsub(str,"\\","/") + str = gsub(str,"^!+","") + str = gsub(str,"^~",resolvers.homedir) + return str + else + return nil end +end - function input.concatinators.zip(tag,path,name) - if not path or path == "" then - return tag .. '?name=' .. name - else - return tag .. '?name=' .. path .. "/" .. name - end +function resolvers.do_with_path(name,func) + for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? + func("^"..resolvers.clean_path(v)) end +end - function input.is_readable.zip(name) - return true - end +function resolvers.do_with_var(name,func) + func(expanded_var(name)) +end - function input.finders.zip(specification,filetype) - specification = input.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) - if zfile then - if input.trace > 0 then - input.logger('! zip finder, path: %s',specification.path) +function resolvers.with_files(pattern,handle) + for _, hash in ipairs(instance.hashes) do + local blobpath = hash.tag + local blobtype = hash.type + if blobpath then + local files = instance.files[blobpath] + if files then + for k,v in next, files do + if find(k,"^remap:") then + k = files[k] + v = files[k] -- chained end - local dfile = zfile:open(q.name) - if dfile then - dfile = zfile:close() - if input.trace > 0 then - input.logger('+ zip finder, name: %s',q.name) + if find(k,pattern) then + if type(v) == "string" then + handle(blobtype,blobpath,v,k) + else + for _,vv in pairs(v) do -- ipairs? + handle(blobtype,blobpath,vv,k) + end end - return specification.original end - elseif input.trace > 0 then - input.logger('? zip finder, path %s',specification.path) end end end - if input.trace > 0 then - input.logger('- zip finder, name: %s',filename) - end - return unpack(input.finders.notfound) end +end - function input.openers.zip(specification) - local zipspecification = input.splitmethod(specification) - if zipspecification.path then - local q = url.query(zipspecification.query) - if q.name then - local zfile = zip.openarchive(zipspecification.path) - if zfile then - if input.trace > 0 then - input.logger('+ zip starter, path: %s',zipspecification.path) - end - local dfile = zfile:open(q.name) - if dfile then - input.show_open(specification) - return input.openers.text_opener(specification,dfile,'zip') - end - elseif input.trace > 0 then - input.logger('- zip starter, path %s',zipspecification.path) - end - end - end - if input.trace > 0 then - input.logger('- zip opener, name: %s',filename) +function resolvers.locate_format(name) + local barename, fmtname = name:gsub("%.%a+$",""), "" + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" + end + if fmtname == "" then + fmtname = resolvers.find_files(barename..".fmt")[1] or "" + end + fmtname = resolvers.clean_path(fmtname) + if fmtname ~= "" then + local barename = file.removesuffix(fmtname) + local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" + if lfs.isfile(luiname) then + return barename, luiname + elseif lfs.isfile(lucname) then + return barename, lucname + elseif lfs.isfile(luaname) then + return barename, luaname end - return unpack(input.openers.notfound) end + return nil, nil +end - function input.loaders.zip(specification) - specification = input.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) - if zfile then - if input.trace > 0 then - input.logger('+ zip starter, path: %s',specification.path) - end - local dfile = zfile:open(q.name) - if dfile then - input.show_load(filename) - if input.trace > 0 then - input.logger('+ zip loader, name: %s',filename) - end - local s = dfile:read("*all") - dfile:close() - return true, s, #s - end - elseif input.trace > 0 then - input.logger('- zip starter, path: %s',specification.path) - end - end - end - if input.trace > 0 then - input.logger('- zip loader, name: %s',filename) - end - return unpack(input.openers.notfound) +function resolvers.boolean_variable(str,default) + local b = resolvers.expansion(str) + if b == "" then + return default + else + b = toboolean(b) + return (b == nil and default) or b end +end - -- zip:///somefile.zip - -- zip:///somefile.zip?tree=texmf-local -> mount +texconfig.kpse_init = false - function input.usezipfile(zipname) - zipname = validzip(zipname) - if input.trace > 0 then - input.logger('! zip use, file: %s',zipname) - end - local specification = input.splitmethod(zipname) - local zipfile = specification.path - if zipfile and not zip.registeredfiles[zipname] then - local tree = url.query(specification.query).tree or "" - if input.trace > 0 then - input.logger('! zip register, file: %s',zipname) - end - local z = zip.openarchive(zipfile) - if z then - local instance = input.instance - if input.trace > 0 then - input.logger("= zipfile, registering: %s",zipname) - end - input.starttiming(instance) - input.aux.prepend_hash('zip',zipname,zipfile) - input.aux.extend_texmf_var(zipname) -- resets hashes too - zip.registeredfiles[zipname] = z - instance.files[zipname] = input.aux.register_zip_file(z,tree or "") - input.stoptiming(instance) - elseif input.trace > 0 then - input.logger("? zipfile, unknown: %s",zipname) - end - elseif input.trace > 0 then - input.logger('! zip register, no file: %s',zipname) - end - end +kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } ) - function input.aux.register_zip_file(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" - else - filter = "^"..tree.."/(.+)/(.-)$" - end - if input.trace > 0 then - input.logger('= zip filter: %s',filter) - end - local register, n = input.aux.register_file, 0 - for i in z:files() do - local path, name = i.filename:match(filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory - end - else - register(files, i.filename, '') - n = n + 1 - end - end - input.logger('= zip entries: %s',n) - return files - end +-- for a while -end +input = resolvers --- filename : luat-zip.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-tmp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['luat-tex'] = 1.001 +--[[ldx-- +

This module deals with caching data. It sets up the paths and +implements loaders and savers for tables. Best is to set the +following variable. When not set, the usual paths will be +checked. Personally I prefer the (users) temporary path.

--- special functions that deal with io + +TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;. + -local format = string.format +

Currently we do no locking when we write files. This is no real +problem because most caching involves fonts and the chance of them +being written at the same time is small. We also need to extend +luatools with a recache feature.

+--ldx]]-- -if texconfig and not texlua then +local format, lower, gsub = string.format, string.lower, string.gsub - input.level = input.level or 0 +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) - if input.logmode() == 'xml' then - function input.show_open(name) - input.level = input.level + 1 - texio.write_nl("") - end - function input.show_close(name) - texio.write(" ") - input.level = input.level - 1 - end - function input.show_load(name) - texio.write_nl("") -- level? - end - else - function input.show_open () end - function input.show_close() end - function input.show_load () end - end +caches = caches or { } - function input.finders.generic(tag,filename,filetype) - local foundname = input.find_file(filename,filetype) - if foundname and foundname ~= "" then - if input.trace > 0 then - input.logger('+ finder: %s, file: %s', tag,filename) - end - return foundname - else - if input.trace > 0 then - input.logger('- finder: %s, file: %s', tag,filename) - end - return unpack(input.finders.notfound) - end - end +caches.path = caches.path or nil +caches.base = caches.base or "luatex-cache" +caches.more = caches.more or "context" +caches.direct = false -- true is faster but may need huge amounts of memory +caches.tree = false +caches.paths = caches.paths or nil +caches.force = false +caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } - input.filters.dynamic_translator = nil - input.filters.frozen_translator = nil - input.filters.utf_translator = nil +function caches.cleanname(name) + return (gsub(lower(name),"[^%w%d]+","-")) +end - function input.openers.text_opener(filename,file_handle,tag) - local u = unicode.utftype(file_handle) - local t = { } - if u > 0 then - if input.trace > 0 then - input.logger('+ opener: %s (%s), file: %s',tag,unicode.utfname[u],filename) - end - local l - if u > 2 then - l = unicode.utf32_to_utf8(file_handle:read("*a"),u==4) - else - l = unicode.utf16_to_utf8(file_handle:read("*a"),u==2) - end - file_handle:close() - t = { - utftype = u, -- may go away - lines = l, - current = 0, -- line number, not really needed - handle = nil, - noflines = #l, - close = function() - if input.trace > 0 then - input.logger('= closer: %s (%s), file: %s',tag,unicode.utfname[u],filename) - end - input.show_close(filename) - t = nil - end, ---~ getline = function(n) ---~ local line = t.lines[n] ---~ if not line or line == "" then ---~ return "" ---~ else ---~ local translator = input.filters.utf_translator ---~ return (translator and translator(line)) or line ---~ end ---~ end, - reader = function(self) - self = self or t - local current, lines = self.current, self.lines - if current >= #lines then - return nil - else - current = current + 1 - self.current = current - local line = lines[current] - if line == "" then - return "" - else - local translator = input.filters.utf_translator - -- return (translator and translator(line)) or line - if translator then - return translator(line) - else - return line - end +function caches.temp() + local cachepath = nil + local function check(list,isenv) + if not cachepath then + for k=1,#list do + local v = list[k] + cachepath = (isenv and (os.env[v] or "")) or v or "" + if cachepath == "" then + -- next + else + cachepath = resolvers.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + break + elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then + break end end end - } - else - if input.trace > 0 then - input.logger('+ opener: %s, file: %s',tag,filename) + cachepath = nil end - -- todo: file;name -> freeze / eerste regel scannen -> freeze - local filters = input.filters - t = { - reader = function(self) - local line = file_handle:read() - if line == "" then - return "" - end - local translator = filters.utf_translator - if translator then - return translator(line) - end - translator = filters.dynamic_translator - if translator then - return translator(line) - end - return line - end, - close = function() - if input.trace > 0 then - input.logger('= closer: %s, file: %s',tag,filename) - end - input.show_close(filename) - file_handle:close() - t = nil - end, - handle = function() - return file_handle - end, - noflines = function() - t.noflines = io.noflines(file_handle) - return t.noflines - end - } end - return t end + check(resolvers.clean_path_list("TEXMFCACHE") or { }) + check(caches.defaults,true) + if not cachepath then + print("\nfatal error: there is no valid (writable) cache path defined\n") + os.exit() + elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" + print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + cachepath = file.collapse_path(cachepath) + function caches.temp() + return cachepath + end + return cachepath +end - function input.openers.generic(tag,filename) - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - input.show_open(filename) - return input.openers.text_opener(filename,f,tag) - end - end - if input.trace > 0 then - input.logger('- opener: %s, file: %s',tag,filename) - end - return unpack(input.openers.notfound) +function caches.configpath() + return table.concat(resolvers.instance.cnffiles,";") +end + +function caches.hashed(tree) + return md5.hex(gsub(lower(tree),"[\\\/]+","/")) +end + +function caches.treehash() + local tree = caches.configpath() + if not tree or tree == "" then + return false + else + return caches.hashed(tree) end +end - function input.loaders.generic(tag,filename) - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - input.show_load(filename) - if input.trace > 0 then - input.logger('+ loader: %s, file: %s',tag,filename) - end - local s = f:read("*a") - garbagecollector.check(s) - f:close() - if s then - return true, s, #s - end - end +function caches.setpath(...) + if not caches.path then + if not caches.path then + caches.path = caches.temp() end - if input.trace > 0 then - input.logger('- loader: %s, file: %s',tag,filename) + caches.path = resolvers.clean_path(caches.path) -- to be sure + caches.tree = caches.tree or caches.treehash() + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) end - return unpack(input.loaders.notfound) end - - function input.finders.tex(filename,filetype) - return input.finders.generic('tex',filename,filetype) - end - function input.openers.tex(filename) - return input.openers.generic('tex',filename) + if not caches.path then + caches.path = '.' end - function input.loaders.tex(filename) - return input.loaders.generic('tex',filename) + caches.path = resolvers.clean_path(caches.path) + if not table.is_empty({...}) then + local pth = dir.mkdirs(caches.path,...) + return pth end - + caches.path = dir.expand_name(caches.path) + return caches.path end --- callback into the file io and related things; disabling kpse - - -if texconfig and not texlua then do - - -- this is not the right place, because we refer to quite some not yet defined tables, but who cares ... +function caches.definepath(category,subcategory) + return function() + return caches.setpath(category,subcategory) + end +end - ctx = ctx or { } +function caches.setluanames(path,name) + return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" +end - function ctx.writestatus(a,b,c,...) - if c then - texio.write_nl(("%-15s: %s\n"):format(a,b:format(c,...))) - else - texio.write_nl(("%-15s: %s\n"):format(a,b)) -- b can have %'s - end +function caches.loaddata(path,name) + local tmaname, tmcname = caches.setluanames(path,name) + local loader = loadfile(tmcname) or loadfile(tmaname) + if loader then + return loader() + else + return false end +end - -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc - - local statusinfo, n = { }, 0 +--~ function caches.loaddata(path,name) +--~ local tmaname, tmcname = caches.setluanames(path,name) +--~ return dofile(tmcname) or dofile(tmaname) +--~ end - function ctx.register_statistics(tag,pattern,fnc) - statusinfo[#statusinfo+1] = { tag, pattern, fnc } - if #tag > n then n = #tag end - end +function caches.iswritable(filepath,filename) + local tmaname, tmcname = caches.setluanames(filepath,filename) + return file.iswritable(tmaname) +end - function ctx.memused() - -- collectgarbage("collect") - return string.format("%s MB (ctx: %s MB)",math.round(collectgarbage("count")), math.round(status.luastate_bytes/1000)) +function caches.savedata(filepath,filename,data,raw) + local tmaname, tmcname = caches.setluanames(filepath,filename) + local reduce, simplify = true, true + if raw then + reduce, simplify = false, false end - - function ctx.show_statistics() -- todo: move calls - local loadtime, register_statistics = input.loadtime, ctx.register_statistics - if caches then - register_statistics("used config path", "%s", function() return caches.configpath() end) - register_statistics("used cache path", "%s", function() return caches.temp() or "?" end) - end - if status.luabytecodes > 0 and input.storage and input.storage.done then - register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) - end - if input.instance then - register_statistics("input load time", "%s seconds", function() return loadtime(input.instance) end) - end - if fonts then - register_statistics("fonts load time","%s seconds", function() return loadtime(fonts) end) - end - if xml then - register_statistics("xml load time", "%s seconds, lpath calls: %s, cached calls: %s", function() - local stats = xml.statistics() - return loadtime(xml), stats.lpathcalls, stats.lpathcached - end) - register_statistics("lxml load time", "%s seconds preparation, backreferences: %i", function() - return loadtime(lxml), #lxml.self - end) - end - if mptopdf then - register_statistics("mps conversion time", "%s seconds", function() return loadtime(mptopdf) end) - end - if nodes then - register_statistics("node processing time", "%s seconds including kernel", function() return loadtime(nodes) end) - end - if kernel then - register_statistics("kernel processing time", "%s seconds", function() return loadtime(kernel) end) - end - if attributes then - register_statistics("attribute processing time", "%s seconds", function() return loadtime(attributes) end) - end - if languages then - register_statistics("language load time", "%s seconds, n=%s", function() return loadtime(languages), languages.hyphenation.n() end) - end - if figures then - register_statistics("graphics processing time", "%s seconds including tex, n=%s", function() return loadtime(figures), figures.n or "?" end) - end - if metapost then - register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return loadtime(metapost), loadtime(mplib), loadtime(metapost.exectime), metapost.n end) - end - if status.luastate_bytes and ctx.memused then - register_statistics("current memory usage", "%s", ctx.memused) - end - if nodes then - register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack - end - if status.node_mem_usage then - register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) - end - if languages then - register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) - end - if fonts then - register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) - end - if xml then -- so we are in mkiv, we need a different check - register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() - input.stoptiming(input.instance) - local runtime = loadtime(input.instance) - local shipped = tex.count['nofshipouts'] - local pages = tex.count['realpageno'] - 1 - local persecond = shipped / runtime - return runtime, pages, shipped, persecond - end) - end - for _, t in ipairs(statusinfo) do - local tag, pattern, fnc = t[1], t[2], t[3] - ctx.writestatus("mkiv lua stats", "%s - %s", tag:rpadd(n," "), pattern:format(fnc())) - end-- input.expanded_path_list("osfontdir") + if caches.direct then + file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex + else + table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true end + local cleanup = resolvers.boolean_variable("PURGECACHE", false) + local strip = resolvers.boolean_variable("LUACSTRIP", true) + utils.lua.compile(tmaname, tmcname, cleanup, strip) +end -end end - -if texconfig and not texlua then - - texconfig.kpse_init = false - texconfig.trace_file_names = input.logmode() == 'tex' - texconfig.max_print_line = 100000 +-- here we use the cache for format loading (texconfig.[formatname|jobname]) - -- if still present, we overload kpse (put it off-line so to say) +--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then + if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc + texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") +end - input.starttiming(input.instance) - if not input.instance then +end -- of closure - if not input.instance then -- prevent a second loading +do -- create closure to overcome 200 locals limit - input.instance = input.reset() - input.instance.progname = 'context' - input.instance.engine = 'luatex' - input.instance.validfile = input.validctxfile +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - input.load() +resolvers.finders = resolvers.finders or { } +resolvers.openers = resolvers.openers or { } +resolvers.loaders = resolvers.loaders or { } - end +resolvers.finders.notfound = { nil } +resolvers.openers.notfound = { nil } +resolvers.loaders.notfound = { false, nil, 0 } - if callback then - callback.register('find_read_file' , function(id,name) return input.findtexfile(name) end) - callback.register('open_read_file' , function( name) return input.opentexfile(name) end) - end - if callback then - callback.register('find_data_file' , function(name) return input.findbinfile(name,"tex") end) - callback.register('find_enc_file' , function(name) return input.findbinfile(name,"enc") end) - callback.register('find_font_file' , function(name) return input.findbinfile(name,"tfm") end) - callback.register('find_format_file' , function(name) return input.findbinfile(name,"fmt") end) - callback.register('find_image_file' , function(name) return input.findbinfile(name,"tex") end) - callback.register('find_map_file' , function(name) return input.findbinfile(name,"map") end) - callback.register('find_ocp_file' , function(name) return input.findbinfile(name,"ocp") end) - callback.register('find_opentype_file' , function(name) return input.findbinfile(name,"otf") end) - callback.register('find_output_file' , function(name) return name end) - callback.register('find_pk_file' , function(name) return input.findbinfile(name,"pk") end) - callback.register('find_sfd_file' , function(name) return input.findbinfile(name,"sfd") end) - callback.register('find_truetype_file' , function(name) return input.findbinfile(name,"ttf") end) - callback.register('find_type1_file' , function(name) return input.findbinfile(name,"pfb") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(name,"vf") end) +end -- of closure - callback.register('read_data_file' , function(file) return input.loadbinfile(file,"tex") end) - callback.register('read_enc_file' , function(file) return input.loadbinfile(file,"enc") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(file,"tfm") end) - -- format - -- image - callback.register('read_map_file' , function(file) return input.loadbinfile(file,"map") end) - callback.register('read_ocp_file' , function(file) return input.loadbinfile(file,"ocp") end) - callback.register('read_opentype_file' , function(file) return input.loadbinfile(file,"otf") end) - -- output - callback.register('read_pk_file' , function(file) return input.loadbinfile(file,"pk") end) - callback.register('read_sfd_file' , function(file) return input.loadbinfile(file,"sfd") end) - callback.register('read_truetype_file' , function(file) return input.loadbinfile(file,"ttf") end) - callback.register('read_type1_file' , function(file) return input.loadbinfile(file,"pfb") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(file,"vf" ) end) - end +do -- create closure to overcome 200 locals limit - if input.aleph_mode == nil then environment.aleph_mode = true end -- some day we will drop omega font support +if not modules then modules = { } end modules ['data-out'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - if callback and input.aleph_mode then - callback.register('find_font_file' , function(name) return input.findbinfile(name,"ofm") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(file,"ofm") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(name,"ovf") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(file,"ovf") end) - end +outputs = outputs or { } - if callback then - callback.register('find_write_file' , function(id,name) return name end) - end - if callback and (not config or (#config == 0)) then - callback.register('find_format_file' , function(name) return name end) - end - if callback and false then - for k, v in pairs(callback.list()) do - if not v then texio.write_nl("callback "..k.." is not set") end - end - end +end -- of closure - if callback then +do -- create closure to overcome 200 locals limit - input.start_actions = { } - input.stop_actions = { } +if not modules then modules = { } end modules ['data-con'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - function input.register_start_actions(f) table.insert(input.start_actions, f) end - function input.register_stop_actions (f) table.insert(input.stop_actions, f) end +local format, lower, gsub = string.format, string.lower, string.gsub - --~ callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) - --~ callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end end) +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) +local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) - end +--[[ldx-- +

Once we found ourselves defining similar cache constructs +several times, containers were introduced. Containers are used +to collect tables in memory and reuse them when possible based +on (unique) hashes (to be provided by the calling function).

- if callback then +

Caching to disk is disabled by default. Version numbers are +stored in the saved table which makes it possible to change the +table structures without bothering about the disk cache.

- if input.logmode() == 'xml' then +

Examples of usage can be found in the font related code.

+--ldx]]-- - function input.start_page_number() - texio.write_nl("

") - texio.write_nl("") - end +containers = containers or { } - callback.register('start_page_number' , input.start_page_number) - callback.register('stop_page_number' , input.stop_page_number ) +containers.usecache = true - function input.report_output_pages(p,b) - texio.write_nl(""..p.."") - texio.write_nl(""..b.."") - texio.write_nl("") - end - function input.report_output_log() - end +local function report(container,tag,name) + if trace_cache or trace_containers then + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') + end +end - callback.register('report_output_pages', input.report_output_pages) - callback.register('report_output_log' , input.report_output_log ) +local allocated = { } - function input.start_run() - texio.write_nl("") - texio.write_nl("") - texio.write_nl("") - end - function input.stop_run() - texio.write_nl("") - end - function input.show_statistics() - for k,v in pairs(status.list()) do - texio.write_nl("log",""..tostring(v).."") - end - end +-- tracing - table.insert(input.start_actions, input.start_run) - table.insert(input.stop_actions , input.show_statistics) - table.insert(input.stop_actions , input.stop_run) +function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches and caches.setpath(category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end + end +end - else - table.insert(input.stop_actions , input.show_statistics) - end +function containers.is_usable(container, name) + return container.enabled and caches and caches.iswritable(container.path, name) +end - callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) - callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end ctx.show_statistics() end) +function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false + end +end +function containers.read(container,name) + if container.enabled and caches and not container.storage[name] and containers.usecache then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) + else + container.storage[name] = nil end - end + if container.storage[name] then + report(container,"reusing",name) + end + return container.storage[name] +end - if kpse then - - function kpse.find_file(filename,filetype,mustexist) - return input.find_file(filename,filetype,mustexist) +function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled and caches then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared end - function kpse.expand_path(variable) - return input.expand_path(variable) - end - function kpse.expand_var(variable) - return input.expand_var(variable) - end - function kpse.expand_braces(variable) - return input.expand_braces(variable) - end - + report(container,"stored",name) + container.storage[name] = data end + return data +end +function containers.content(container,name) + return container.storage[name] end --- program specific configuration (memory settings and alike) -if texconfig and not texlua then +end -- of closure - luatex = luatex or { } +do -- create closure to overcome 200 locals limit - luatex.variablenames = { - 'main_memory', 'extra_mem_bot', 'extra_mem_top', - 'buf_size','expand_depth', - 'font_max', 'font_mem_size', - 'hash_extra', 'max_strings', 'pool_free', 'pool_size', 'string_vacancies', - 'obj_tab_size', 'pdf_mem_size', 'dest_names_size', - 'nest_size', 'param_size', 'save_size', 'stack_size', - 'trie_size', 'hyph_size', 'max_in_open', - 'ocp_stack_size', 'ocp_list_size', 'ocp_buf_size' - } +if not modules then modules = { } end modules ['data-use'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - function luatex.variables() - local t, x = { }, nil - for _,v in pairs(luatex.variablenames) do - x = input.var_value(v) - if x and x:find("^%d+$") then - t[v] = tonumber(x) - end - end - return t - end +local format, lower, gsub = string.format, string.lower, string.gsub - function luatex.setvariables(tab) - for k,v in pairs(luatex.variables()) do - tab[k] = v - end - end +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) - if not luatex.variables_set then - luatex.setvariables(texconfig) - luatex.variables_set = true - end +-- since we want to use the cache instead of the tree, we will now +-- reimplement the saver. - texconfig.max_print_line = 100000 - texconfig.max_in_open = 127 +local save_data = resolvers.save_data +local load_data = resolvers.load_data -end +resolvers.cachepath = nil -- public, for tracing +resolvers.usecache = true -- public, for tracing --- some tex basics, maybe this will move to ctx +function resolvers.save_data(dataname) + save_data(dataname, function(cachename,dataname) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(cachename)) + else + return file.join(cachename,dataname) + end + end) +end -if tex then +function resolvers.load_data(pathname,dataname,filename) + load_data(pathname,dataname,filename,function(dataname,filename) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(pathname)) + else + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) + end + end) +end - local texsprint, texwrite = tex.sprint, tex.write +-- we will make a better format, maybe something xml or just text or lua - if not cs then cs = { } end +resolvers.automounted = resolvers.automounted or { } - function cs.def(k,v) - texsprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") +function resolvers.automount(usecache) + local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) + if table.is_empty(mountpaths) and usecache then + mountpaths = { caches.setpath("mount") } end - - function cs.chardef(k,v) - texsprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") + if not table.is_empty(mountpaths) then + statistics.starttiming(resolvers.instance) + for k, root in pairs(mountpaths) do + local f = io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if line:find("^[%%#%-]") then -- or %W + -- skip + elseif line:find("^zip://") then + if trace_locating then + logs.report("fileio","mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) + end + end + end + f:close() + end + end + statistics.stoptiming(resolvers.instance) end +end + +-- status info + +statistics.register("used config path", function() return caches.configpath() end) +statistics.register("used cache path", function() return caches.temp() or "?" end) + +-- experiment (code will move) - function cs.boolcase(b) - if b then texwrite(1) else texwrite(0) end +function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname + local enginebanner = status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname = file.replacesuffix(texname,"luv") + local luvdata = { + enginebanner = enginebanner, + formatbanner = formatbanner, + sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcefile = sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) end +end - function cs.testcase(b) - if b then - texsprint(tex.texcatcodes, "\\firstoftwoarguments") +function statistics.check_fmt_status(texname) + local enginebanner = status.list().banner + if enginebanner and texname then + local luvname = file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv = dofile(luvname) + if luv and luv.sourcefile then + local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + if luv.enginebanner and luv.enginebanner ~= enginebanner then + return "engine mismatch" + end + if luv.sourcehash and luv.sourcehash ~= sourcehash then + return "source mismatch" + end + else + return "invalid status file" + end else - texsprint(tex.texcatcodes, "\\secondoftwoarguments") + return "missing status file" end end - + return true end +end -- of closure + +do -- create closure to overcome 200 locals limit + if not modules then modules = { } end modules ['luat-kps'] = { version = 1.001, comment = "companion to luatools.lua", @@ -6522,90 +6348,218 @@ $SELFAUTODIR : /usr/tex/bin $SELFAUTOPARENT : /usr/tex -

How about just forgetting abou them?

+

How about just forgetting about them?

--ldx]]-- -input = input or { } -input.suffixes = input.suffixes or { } -input.formats = input.formats or { } - -input.suffixes['gf'] = { 'gf' } -input.suffixes['pk'] = { 'pk' } -input.suffixes['base'] = { 'base' } -input.suffixes['bib'] = { 'bib' } -input.suffixes['bst'] = { 'bst' } -input.suffixes['cnf'] = { 'cnf' } -input.suffixes['mem'] = { 'mem' } -input.suffixes['mf'] = { 'mf' } -input.suffixes['mfpool'] = { 'pool' } -input.suffixes['mft'] = { 'mft' } -input.suffixes['mppool'] = { 'pool' } -input.suffixes['graphic/figure'] = { 'eps', 'epsi' } -input.suffixes['texpool'] = { 'pool' } -input.suffixes['PostScript header'] = { 'pro' } -input.suffixes['ist'] = { 'ist' } -input.suffixes['web'] = { 'web', 'ch' } -input.suffixes['cweb'] = { 'w', 'web', 'ch' } -input.suffixes['cmap files'] = { 'cmap' } -input.suffixes['lig files'] = { 'lig' } -input.suffixes['bitmap font'] = { } -input.suffixes['MetaPost support'] = { } -input.suffixes['TeX system documentation'] = { } -input.suffixes['TeX system sources'] = { } -input.suffixes['dvips config'] = { } -input.suffixes['type42 fonts'] = { } -input.suffixes['web2c files'] = { } -input.suffixes['other text files'] = { } -input.suffixes['other binary files'] = { } -input.suffixes['opentype fonts'] = { 'otf' } - -input.suffixes['fmt'] = { 'fmt' } -input.suffixes['texmfscripts'] = { 'rb','lua','py','pl' } - -input.suffixes['pdftex config'] = { } -input.suffixes['Troff fonts'] = { } - -input.suffixes['ls-R'] = { } +local suffixes = resolvers.suffixes +local formats = resolvers.formats + +suffixes['gf'] = { 'gf' } +suffixes['pk'] = { 'pk' } +suffixes['base'] = { 'base' } +suffixes['bib'] = { 'bib' } +suffixes['bst'] = { 'bst' } +suffixes['cnf'] = { 'cnf' } +suffixes['mem'] = { 'mem' } +suffixes['mf'] = { 'mf' } +suffixes['mfpool'] = { 'pool' } +suffixes['mft'] = { 'mft' } +suffixes['mppool'] = { 'pool' } +suffixes['graphic/figure'] = { 'eps', 'epsi' } +suffixes['texpool'] = { 'pool' } +suffixes['PostScript header'] = { 'pro' } +suffixes['ist'] = { 'ist' } +suffixes['web'] = { 'web', 'ch' } +suffixes['cweb'] = { 'w', 'web', 'ch' } +suffixes['cmap files'] = { 'cmap' } +suffixes['lig files'] = { 'lig' } +suffixes['bitmap font'] = { } +suffixes['MetaPost support'] = { } +suffixes['TeX system documentation'] = { } +suffixes['TeX system sources'] = { } +suffixes['dvips config'] = { } +suffixes['type42 fonts'] = { } +suffixes['web2c files'] = { } +suffixes['other text files'] = { } +suffixes['other binary files'] = { } +suffixes['opentype fonts'] = { 'otf' } + +suffixes['fmt'] = { 'fmt' } +suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +suffixes['pdftex config'] = { } +suffixes['Troff fonts'] = { } + +suffixes['ls-R'] = { } --[[ldx--

If you wondered abou tsome of the previous mappings, how about the next bunch:

--ldx]]-- -input.formats['bib'] = '' -input.formats['bst'] = '' -input.formats['mft'] = '' -input.formats['ist'] = '' -input.formats['web'] = '' -input.formats['cweb'] = '' -input.formats['MetaPost support'] = '' -input.formats['TeX system documentation'] = '' -input.formats['TeX system sources'] = '' -input.formats['Troff fonts'] = '' -input.formats['dvips config'] = '' -input.formats['graphic/figure'] = '' -input.formats['ls-R'] = '' -input.formats['other text files'] = '' -input.formats['other binary files'] = '' - -input.formats['gf'] = '' -input.formats['pk'] = '' -input.formats['base'] = 'MFBASES' -input.formats['cnf'] = '' -input.formats['mem'] = 'MPMEMS' -input.formats['mf'] = 'MFINPUTS' -input.formats['mfpool'] = 'MFPOOL' -input.formats['mppool'] = 'MPPOOL' -input.formats['texpool'] = 'TEXPOOL' -input.formats['PostScript header'] = 'TEXPSHEADERS' -input.formats['cmap files'] = 'CMAPFONTS' -input.formats['type42 fonts'] = 'T42FONTS' -input.formats['web2c files'] = 'WEB2C' -input.formats['pdftex config'] = 'PDFTEXCONFIG' -input.formats['texmfscripts'] = 'TEXMFSCRIPTS' -input.formats['bitmap font'] = '' -input.formats['lig files'] = 'LIGFONTS' +formats['bib'] = '' +formats['bst'] = '' +formats['mft'] = '' +formats['ist'] = '' +formats['web'] = '' +formats['cweb'] = '' +formats['MetaPost support'] = '' +formats['TeX system documentation'] = '' +formats['TeX system sources'] = '' +formats['Troff fonts'] = '' +formats['dvips config'] = '' +formats['graphic/figure'] = '' +formats['ls-R'] = '' +formats['other text files'] = '' +formats['other binary files'] = '' + +formats['gf'] = '' +formats['pk'] = '' +formats['base'] = 'MFBASES' +formats['cnf'] = '' +formats['mem'] = 'MPMEMS' +formats['mf'] = 'MFINPUTS' +formats['mfpool'] = 'MFPOOL' +formats['mppool'] = 'MPPOOL' +formats['texpool'] = 'TEXPOOL' +formats['PostScript header'] = 'TEXPSHEADERS' +formats['cmap files'] = 'CMAPFONTS' +formats['type42 fonts'] = 'T42FONTS' +formats['web2c files'] = 'WEB2C' +formats['pdftex config'] = 'PDFTEXCONFIG' +formats['texmfscripts'] = 'TEXMFSCRIPTS' +formats['bitmap font'] = '' +formats['lig files'] = 'LIGFONTS' + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-aux'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local find = string.find + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) + +function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = resolvers.clean_path(oldname) + if trace_verbose then + logs.report("fileio","to be replaced old script %s", oldscript) + end + local newscripts = resolvers.find_files(newname) or { } + if #newscripts == 0 then + if trace_verbose then + logs.report("fileio","unable to locate new script") + end + else + for i=1,#newscripts do + local newscript = resolvers.clean_path(newscripts[i]) + if trace_verbose then + logs.report("fileio","checking new script %s", newscript) + end + if oldscript == newscript then + if trace_verbose then + logs.report("fileio","old and new script are the same") + end + elseif not find(newscript,scriptpath) then + if trace_verbose then + logs.report("fileio","new script should come from %s",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_verbose then + logs.report("fileio","invalid new script name") + end + else + local newdata = io.loaddata(newscript) + if newdata then + if trace_verbose then + logs.report("fileio","old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_verbose then + logs.report("fileio","unable to load new script") + end + end + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-lst'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- used in mtxrun + +local find, concat, upper, format = string.find, table.concat, string.upper, string.format + +resolvers.listers = resolvers.listers or { } + +local function tabstr(str) + if type(str) == 'table' then + return concat(str," | ") + else + return str + end +end + +local function list(list,report) + local instance = resolvers.instance + local pat = upper(pattern or "","") + local report = report or texio.write_nl + for _,key in pairs(table.sortedkeys(list)) do + if instance.pattern == "" or find(upper(key),pat) then + if instance.kpseonly then + if instance.kpsevars[key] then + report(format("%s=%s",key,tabstr(list[key]))) + end + else + report(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) + end + end + end +end + +function resolvers.listers.variables () list(resolvers.instance.variables ) end +function resolvers.listers.expansions() list(resolvers.instance.expansions) end + +function resolvers.listers.configurations(report) + local report = report or texio.write_nl + local instance = resolvers.instance + for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do + if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then + report(format("%s\n",key)) + for i,c in ipairs(instance.order) do + local str = c[key] + if str then + report(format("\t%s\t%s",i,str)) + end + end + report("") + end + end +end + +end -- of closure -- end library merge -- We initialize some characteristics of this program. We need to @@ -6623,21 +6577,33 @@ own.libs = { -- todo: check which ones are really needed 'l-number.lua', 'l-set.lua', 'l-os.lua', - 'l-md5.lua', 'l-file.lua', + 'l-md5.lua', 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-unicode.lua', 'l-math.lua', 'l-utils.lua', - 'luat-lib.lua', - 'luat-inp.lua', - 'luat-log.lua', - 'luat-tmp.lua', - 'luat-zip.lua', - 'luat-tex.lua', - 'luat-kps.lua', + 'trac-tra.lua', + 'luat-env.lua', + 'trac-inf.lua', + 'trac-log.lua', + 'data-res.lua', + 'data-tmp.lua', +-- 'data-pre.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-tex.lua', +-- 'data-bin.lua', +-- 'data-zip.lua', +-- 'data-crl.lua', +-- 'data-lua.lua', + 'data-kps.lua', -- so that we can replace kpsewhich + 'data-aux.lua', -- updater + 'data-lst.lua', -- lister } -- We need this hack till luatex is fixed. @@ -6674,11 +6640,11 @@ function locate_libs() end end -if not input then +if not resolvers then locate_libs() end -if not input then +if not resolvers then print("") print("Luatools is unable to start up due to lack of libraries. You may") print("try to run 'lua luatools.lua --selfmerge' in the path where this") @@ -6687,63 +6653,65 @@ if not input then os.exit() end -input.instance = input.reset() -input.verbose = environment.arguments["verbose"] or false -input.banner = 'LuaTools' -utils.report = input.report - -input.defaultlibs = { -- not all are needed - 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', - 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-dimen.lua', - 'luat-lib.lua', 'luat-inp.lua', 'luat-env.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' -} +logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false) --- todo: use environment.argument() instead of environment.arguments[] +local instance = resolvers.reset() -local instance = input.instance +resolvers.defaultlibs = { -- not all are needed + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-boolean.lua', + 'l-number.lua', + 'l-unicode.lua', + 'l-os.lua', + 'l-io.lua', + 'l-file.lua', + 'l-md5.lua', + 'l-url.lua', + 'l-dir.lua', + 'l-utils.lua', + 'l-dimen.lua', + 'trac-inf.lua', + 'trac-tra.lua', + 'trac-log.lua', + 'luat-env.lua', -- here ? + 'data-res.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-tmp.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-pre.lua', + 'data-tex.lua', + 'data-bin.lua', +-- 'data-zip.lua', +-- 'data-clr.lua', + 'data-lua.lua', + 'data-ctx.lua', + 'luat-fio.lua', + 'luat-cnf.lua', +} instance.engine = environment.arguments["engine"] or 'luatex' instance.progname = environment.arguments["progname"] or 'context' instance.luaname = environment.arguments["luafile"] or "" -- environment.ownname or "" -instance.lualibs = environment.arguments["lualibs"] or table.concat(input.defaultlibs,",") +instance.lualibs = environment.arguments["lualibs"] or table.concat(resolvers.defaultlibs,",") instance.allresults = environment.arguments["all"] or false instance.pattern = environment.arguments["pattern"] or nil instance.sortdata = environment.arguments["sort"] or false instance.kpseonly = not environment.arguments["all"] or false instance.my_format = environment.arguments["format"] or instance.format -instance.lsrmode = environment.arguments["lsr"] or false if type(instance.pattern) == 'boolean' then - input.report("invalid pattern specification") -- toto, force verbose for one message + logs.simple("invalid pattern specification") instance.pattern = nil end -if environment.arguments["trace"] then input.settrace(environment.arguments["trace"]) end - -if environment.arguments["minimize"] then - if input.validators.visibility[instance.progname] then - instance.validfile = input.validators.visibility[instance.progname] - end -end - -function input.my_prepare_a() - input.resetconfig() - input.identify_cnf() - input.load_lua() - input.expand_variables() - input.load_cnf() - input.expand_variables() -end - -function input.my_prepare_b() - input.my_prepare_a() - input.load_hash() - input.automount() -end - --- barename +if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end -if not messages then messages = { } end +runners = runners or { } +messages = messages or { } messages.no_ini_file = [[ There is no lua initialization file found. This file can be forced by the @@ -6769,21 +6737,18 @@ messages.help = [[ --luafile=str lua inifile (default is .lua) --lualibs=list libraries to assemble (optional when --compile) --compile assemble and compile lua inifile ---mkii force context mkii mode (only for testing, not usable!) --verbose give a bit more info ---minimize optimize lists for format --all show all found files --sort sort cached data --engine=str target engine --progname=str format or backend --pattern=str filter variables ---lsr use lsr and cnf directly ]] -function input.my_make_format(texname) - local instance = input.instance +function runners.make_format(texname) + local instance = resolvers.instance if texname and texname ~= "" then - if input.usecache then + if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform if path and lfs then lfs.chdir(path) @@ -6793,22 +6758,22 @@ function input.my_make_format(texname) if barename == texname then texname = texname .. ".tex" end - local fullname = input.find_files(texname)[1] or "" + local fullname = resolvers.find_files(texname)[1] or "" if fullname == "" then - input.report("no tex file with name: %s",texname) + logs.simple("no tex file with name: %s",texname) else local luaname, lucname, luapath, lualibs = "", "", "", { } -- the following is optional, since context.lua can also -- handle this collect and compile business if environment.arguments["compile"] then if luaname == "" then luaname = barename end - input.report("creating initialization file: %s",luaname) + logs.simple("creating initialization file: %s",luaname) luapath = file.dirname(luaname) if luapath == "" then luapath = file.dirname(texname) end if luapath == "" then - luapath = file.dirname(input.find_files(texname)[1] or "") + luapath = file.dirname(resolvers.find_files(texname)[1] or "") end lualibs = string.split(instance.lualibs,",") luaname = file.basename(barename .. ".lua") @@ -6818,83 +6783,86 @@ function input.my_make_format(texname) if lualibs[1] then local firstlib = file.join(luapath,lualibs[1]) if not lfs.isfile(firstlib) then - local foundname = input.find_files(lualibs[1])[1] + local foundname = resolvers.find_files(lualibs[1])[1] if foundname then - input.report("located library path: %s",luapath) + logs.simple("located library path: %s",luapath) luapath = file.dirname(foundname) end end end - input.report("using library path: %s",luapath) - input.report("using lua libraries: %s",table.join(lualibs," ")) + logs.simple("using library path: %s",luapath) + logs.simple("using lua libraries: %s",table.join(lualibs," ")) utils.merger.selfcreate(lualibs,luapath,luaname) - local strip = input.boolean_variable("LUACSTRIP", true) + local strip = resolvers.boolean_variable("LUACSTRIP", true) if utils.lua.compile(luaname,lucname,false,strip) and io.exists(lucname) then luaname = lucname - input.report("using compiled initialization file: %s",lucname) + logs.simple("using compiled initialization file: %s",lucname) else - input.report("using uncompiled initialization file: %s",luaname) + logs.simple("using uncompiled initialization file: %s",luaname) end else for _, v in pairs({instance.luaname, instance.progname, barename}) do v = string.gsub(v..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then - luaname = input.find_files(v)[1] or "" + luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then break end end end end + if environment.arguments["noluc"] then + luaname = luaname:gsub("%.luc$",".lua") -- make this an option + end if luaname == "" then - input.reportlines(messages.no_ini_file) - input.report("texname : %s",texname) - input.report("luaname : %s",instance.luaname) - input.report("progname: %s",instance.progname) - input.report("barename: %s",barename) + if logs.verbose then + logs.simplelines(messages.no_ini_file) + logs.simple("texname : %s",texname) + logs.simple("luaname : %s",instance.luaname) + logs.simple("progname: %s",instance.progname) + logs.simple("barename: %s",barename) + end else - input.report("using lua initialization file: %s",luaname) - local mp = dir.glob(file.stripsuffix(file.basename(luaname)).."-*.mem") + logs.simple("using lua initialization file: %s",luaname) + local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") if mp and #mp > 0 then for _, name in ipairs(mp) do - input.report("removing related mplib format %s", file.basename(name)) + logs.simple("removing related mplib format %s", file.basename(name)) os.remove(name) end end - local flags = { "--ini" } - if environment.arguments["mkii"] then - flags[#flags+1] = "--progname=" .. instance.progname - else - flags[#flags+1] = "--lua=" .. string.quote(luaname) - end + local flags = { + "--ini", + "--lua=" .. string.quote(luaname) + } local bs = (os.platform == "unix" and "\\\\") or "\\" -- todo: make a function local command = "luatex ".. table.concat(flags," ") .. " " .. string.quote(fullname) .. " " .. bs .. "dump" - input.report("running command: %s\n",command) + logs.simple("running command: %s\n",command) os.spawn(command) -- todo: do a dummy run that generates the related metafun and mfplain formats end end else - input.report("no tex file given") + logs.simple("no tex file given") end end -function input.my_run_format(name,data,more) +function runners.run_format(name,data,more) -- hm, rather old code here; we can now use the file.whatever functions if name and (name ~= "") then local barename = name:gsub("%.%a+$","") local fmtname = "" - if input.usecache then + if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" end if fmtname == "" then - fmtname = input.find_files(barename..".fmt")[1] or "" + fmtname = resolvers.find_files(barename..".fmt")[1] or "" end - fmtname = input.clean_path(fmtname) + fmtname = resolvers.clean_path(fmtname) barename = fmtname:gsub("%.%a+$","") if fmtname == "" then - input.report("no format with name: %s",name) + logs.simple("no format with name: %s",name) else local luaname = barename .. ".luc" local f = io.open(luaname) @@ -6904,151 +6872,104 @@ function input.my_run_format(name,data,more) end if f then f:close() - local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. string.quote(more) - input.report("running command: %s",command) + local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. (more ~= "" and string.quote(more) or "") + logs.simple("running command: %s",command) os.spawn(command) else - input.report("using format name: %s",fmtname) - input.report("no luc/lua with name: %s",barename) - end - end - end -end - --- helpers for verbose lists - -input.listers = input.listers or { } - -local function tabstr(str) - if type(str) == 'table' then - return table.concat(str," | ") - else - return str - end -end - -local function list(list) - local instance = input.instance - local pat = string.upper(pattern or "","") - for _,key in pairs(table.sortedkeys(list)) do - if instance.pattern == "" or string.find(key:upper(),pat) then - if instance.kpseonly then - if instance.kpsevars[key] then - print(format("%s=%s",key,tabstr(list[key]))) - end - else - print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) - end - end - end -end - -function input.listers.variables () list(input.instance.variables ) end -function input.listers.expansions() list(input.instance.expansions) end - -function input.listers.configurations() - local instance = input.instance - for _,key in pairs(table.sortedkeys(instance.kpsevars)) do - if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then - print(key.."\n") - for i,c in ipairs(instance.order) do - local str = c[key] - if str then - print(format("\t%s\t%s",i,str)) - end + logs.simple("using format name: %s",fmtname) + logs.simple("no luc/lua with name: %s",barename) end - print() end end end -input.report("%s\n",banner) - local ok = true +-- private option --noluc for testing errors in the stub + if environment.arguments["find-file"] then - input.my_prepare_b() + resolvers.load() instance.format = environment.arguments["format"] or instance.format if instance.pattern then instance.allresults = true - input.for_files(input.find_files, { instance.pattern }, instance.my_format) + resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format) else - input.for_files(input.find_files, environment.files, instance.my_format) + resolvers.for_files(resolvers.find_files, environment.files, instance.my_format) end elseif environment.arguments["find-path"] then - input.my_prepare_b() - local path = input.find_file(environment.files[1], instance.my_format) - if input.verbose then - input.report(file.dirname(path)) + resolvers.load() + local path = resolvers.find_file(environment.files[1], instance.my_format) + if logs.verbose then + logs.simple(file.dirname(path)) else print(file.dirname(path)) end elseif environment.arguments["run"] then - input.my_prepare_a() -- ! no need for loading databases - input.verbose = true - input.my_run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") + resolvers.load("nofiles") -- ! no need for loading databases + logs.setverbose(true) + runners.run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") elseif environment.arguments["fmt"] then - input.my_prepare_a() -- ! no need for loading databases - input.verbose = true - input.my_run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") + resolvers.load("nofiles") -- ! no need for loading databases + logs.setverbose(true) + runners.run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") elseif environment.arguments["expand-braces"] then - input.my_prepare_a() - input.for_files(input.expand_braces, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_braces, environment.files) elseif environment.arguments["expand-path"] then - input.my_prepare_a() - input.for_files(input.expand_path, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_path, environment.files) elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then - input.my_prepare_a() - input.for_files(input.expand_var, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_var, environment.files) elseif environment.arguments["show-path"] or environment.arguments["path-value"] then - input.my_prepare_a() - input.for_files(input.show_path, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.show_path, environment.files) elseif environment.arguments["var-value"] or environment.arguments["show-value"] then - input.my_prepare_a() - input.for_files(input.var_value, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.var_value, environment.files) elseif environment.arguments["format-path"] then - input.my_prepare_b() - input.report(caches.setpath("format")) + resolvers.load() + logs.simple(caches.setpath("format")) elseif instance.pattern then -- brrr - input.my_prepare_b() + resolvers.load() instance.format = environment.arguments["format"] or instance.format instance.allresults = true - input.for_files(input.find_files, { instance.pattern }, instance.my_format) + resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format) elseif environment.arguments["generate"] then instance.renewcache = true - input.verbose = true - input.my_prepare_b(instance) + logs.setverbose(true) + resolvers.load() elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then - input.my_prepare_b(instance) - input.verbose = true - input.my_make_format(environment.files[1] or "") + resolvers.load() + logs.setverbose(true) + runners.make_format(environment.files[1] or "") elseif environment.arguments["selfmerge"] then utils.merger.selfmerge(own.name,own.libs,own.list) elseif environment.arguments["selfclean"] then utils.merger.selfclean(own.name) elseif environment.arguments["selfupdate"] then - input.my_prepare_b() - input.verbose = true - input.update_script(own.name,"luatools") + resolvers.load() + logs.setverbose(true) + resolvers.update_script(own.name,"luatools") elseif environment.arguments["variables"] or environment.arguments["show-variables"] then - input.my_prepare_a() - input.listers.variables() + resolvers.load("nofiles") + resolvers.listers.variables() elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then - input.my_prepare_a() - input.listers.expansions() + resolvers.load("nofiles") + resolvers.listers.expansions() elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then - input.my_prepare_a() - input.listers.configurations() + resolvers.load("nofiles") + resolvers.listers.configurations() elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then - input.help(banner,messages.help) + logs.help(messages.help) else - input.my_prepare_b() - input.for_files(input.find_files, environment.files, instance.my_format) + resolvers.load() + resolvers.for_files(resolvers.find_files, environment.files, instance.my_format) end -if input.verbose then - input.report("") - input.report("runtime: %0.3f seconds",os.runtime()) +if logs.verbose then + logs.simpleline() + logs.simple("runtime: %0.3f seconds",os.runtime()) end if os.platform == "unix" then diff --git a/scripts/context/lua/mtx-babel.lua b/scripts/context/lua/mtx-babel.lua index 92b1cd597..8d7765643 100644 --- a/scripts/context/lua/mtx-babel.lua +++ b/scripts/context/lua/mtx-babel.lua @@ -385,22 +385,22 @@ do local structure = environment.argument("structure") or "document" converter = converter[structure] if converter then - input.report("converting '%s' using language '%s' with structure '%s'", filename, language, structure) + logs.simple("converting '%s' using language '%s' with structure '%s'", filename, language, structure) data = converter:match(data) local newfilename = filename .. ".utf" io.savedata(newfilename, data) - input.report("converted data saved in '%s'", newfilename) + logs.simple("converted data saved in '%s'", newfilename) else - input.report("unknown structure '%s' language '%s'", structure, language) + logs.simple("unknown structure '%s' language '%s'", structure, language) end else - input.report("no converter for language '%s'", language) + logs.simple("no converter for language '%s'", language) end else - input.report("provide language") + logs.simple("provide language") end else - input.report("no data in '%s'",filename) + logs.simple("no data in '%s'",filename) end end end @@ -413,7 +413,7 @@ do end -banner = banner .. " | babel conversion tools | version 1.2" +logs.extendbanner("Babel Conversion Tools 1.2",true) messages.help = [[ --language=string conversion language (e.g. greek) @@ -421,10 +421,8 @@ messages.help = [[ --convert convert babel codes into utf ]] -input.verbose = true - if environment.argument("convert") then scripts.babel.convert(environment.files[1] or "") else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-cache.lua b/scripts/context/lua/mtx-cache.lua index 0432168ee..a1fbed825 100644 --- a/scripts/context/lua/mtx-cache.lua +++ b/scripts/context/lua/mtx-cache.lua @@ -76,7 +76,7 @@ function scripts.cache.list(all) end) end -banner = banner .. " | cache tools " +logs.extendbanner("Cache Tools 0.10") messages.help = [[ --purge remove not used files @@ -93,5 +93,5 @@ elseif environment.argument("erase") then elseif environment.argument("list") then scripts.cache.list(environment.argument("all")) else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-chars.lua b/scripts/context/lua/mtx-chars.lua index f4f751377..d4330ca30 100644 --- a/scripts/context/lua/mtx-chars.lua +++ b/scripts/context/lua/mtx-chars.lua @@ -6,38 +6,12 @@ if not modules then modules = { } end modules ['mtx-chars'] = { license = "see context related readme files" } -local format, concat = string.format, table.concat +local format, concat, utfchar, upper = string.format, table.concat, unicode.utf8.char, string.upper scripts = scripts or { } scripts.chars = scripts.chars or { } -function scripts.chars.stixtomkiv(inname,outname) - if inname == "" then - logs.report("aquiring math data","invalid datafilename") - end - local f = io.open(inname) - if not f then - logs.report("aquiring math data","invalid datafile") - else - logs.report("aquiring math data","processing " .. inname) - if not outname or outname == "" then - outname = "char-mth.lua" - end - local classes = { - N = "normal", - A = "alphabetic", - D = "diacritic", - P = "punctuation", - B = "binary", - R = "relation", - L = "large", - O = "opening", - C = "closing", - F = "fence" - } - local valid, done = false, { } - local g = io.open(outname,'w') - g:write([[ +local banner = [[ -- filename : char-mth.lua -- comment : companion to char-mth.tex (in ConTeXt) -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -46,38 +20,70 @@ function scripts.chars.stixtomkiv(inname,outname) if not versions then versions = { } end versions['char-mth'] = 1.001 if not characters then characters = { } end - ]]) - g:write(format("\ncharacters.math = {\n")) - for l in f:lines() do - if not valid then - valid = l:find("AMS/TeX name") - end - if valid then - local unicode = l:sub(2,6) - if unicode:sub(1,1) ~= " " and unicode ~= "" and not done[unicode] then - local mathclass, adobename, texname = l:sub(57,57) or "", l:sub(13,36) or "", l:sub(84,109) or "" - texname, adobename = texname:gsub("[\\ ]",""), adobename:gsub("[\\ ]","") - local t = { } - if mathclass ~= "" then t[#t+1] = format("mathclass='%s'", classes[mathclass] or "unknown") end - if adobename ~= "" then t[#t+1] = format("adobename='%s'", adobename ) end - if texname ~= "" then t[#t+1] = format("texname='%s'" , texname ) end - if #t > 0 then - g:write(format("\t[0x%s] = { %s },\n",unicode, concat(t,", "))) - end - done[unicode] = true - end - end - end - if not valid then - g:write("\t-- The data file is corrupt, invalid or maybe the format has changed.\n") - logs.report("aquiring math data","problems with data table") - else - logs.report("aquiring math data","table saved in " .. outname) - end - g:write("}\n") - g:close() - f:close() - end +]] + +--~ function scripts.chars.stixtomkiv(inname,outname) +--~ if inname == "" then +--~ logs.report("aquiring math data","invalid datafilename") +--~ end +--~ local f = io.open(inname) +--~ if not f then +--~ logs.report("aquiring math data","invalid datafile") +--~ else +--~ logs.report("aquiring math data","processing " .. inname) +--~ if not outname or outname == "" then +--~ outname = "char-mth.lua" +--~ end +--~ local classes = { +--~ N = "normal", +--~ A = "alphabetic", +--~ D = "diacritic", +--~ P = "punctuation", +--~ B = "binary", +--~ R = "relation", +--~ L = "large", +--~ O = "opening", +--~ C = "closing", +--~ F = "fence" +--~ } +--~ local valid, done = false, { } +--~ local g = io.open(outname,'w') +--~ g:write(banner) +--~ g:write(format("\ncharacters.math = {\n")) +--~ for l in f:lines() do +--~ if not valid then +--~ valid = l:find("AMS/TeX name") +--~ end +--~ if valid then +--~ local unicode = l:sub(2,6) +--~ if unicode:sub(1,1) ~= " " and unicode ~= "" and not done[unicode] then +--~ local mathclass, adobename, texname = l:sub(57,57) or "", l:sub(13,36) or "", l:sub(84,109) or "" +--~ texname, adobename = texname:gsub("[\\ ]",""), adobename:gsub("[\\ ]","") +--~ local t = { } +--~ if mathclass ~= "" then t[#t+1] = format("mathclass='%s'", classes[mathclass] or "unknown") end +--~ if adobename ~= "" then t[#t+1] = format("adobename='%s'", adobename ) end +--~ if texname ~= "" then t[#t+1] = format("texname='%s'" , texname ) end +--~ if #t > 0 then +--~ g:write(format("\t[0x%s] = { %s },\n",unicode, concat(t,", "))) +--~ end +--~ done[unicode] = true +--~ end +--~ end +--~ end +--~ if not valid then +--~ g:write("\t-- The data file is corrupt, invalid or maybe the format has changed.\n") +--~ logs.report("aquiring math data","problems with data table") +--~ else +--~ logs.report("aquiring math data","table saved in " .. outname) +--~ end +--~ g:write("}\n") +--~ g:close() +--~ f:close() +--~ end +--~ end + +function scripts.chars.stixtomkiv(inname,outname) + logs.report("we no longer use this options but use our own tables instead") end local banner_pdf_1 = [[ @@ -95,7 +101,7 @@ local banner_pdf_2 = [[ ]] function scripts.chars.makepdfr() - local chartable = input.find_file("char-def.lua") or "" + local chartable = resolvers.find_file("char-def.lua") or "" if chartable ~= "" then dofile(chartable) if characters and characters.data then @@ -117,93 +123,189 @@ function scripts.chars.makepdfr() end end -local banner_utf_1 = [[ -% filename : enco-utf.tex -% comment : generated by mtxrun --script chars --utf -% author : Hans Hagen, PRAGMA-ADE, Hasselt NL -% copyright: PRAGMA ADE / ConTeXt Development Team -% license : see context related readme files +local banner_utf_module = [[ +%% filename : %s +%% comment : generated by mtxrun --script chars --xtx +%% author : Hans Hagen, PRAGMA-ADE, Hasselt NL +%% copyright: PRAGMA ADE / ConTeXt Development Team +%% license : see context related readme files +]] -\ifx\setcclcucx\undefined +local banner_utf_mappings = [[ - \def\setcclcucx #1 #2 #3 % - {\global\catcode"#1=11 - \global\lccode "#1="#2 - \global\uccode "#1="#3 } +% lc/uc/catcode mappings -\fi ]] -local banner_utf_2 = [[ +local banner_utf_patch = [[ -% lc/uc/catcode mappings +% patch needed for turkish +\setXTXcharcodes "201C "201C "201C +\setXTXcharcodes "201D "201D "201D ]] -local banner_utf_3 = [[ +local banner_utf_names = [[ % named characters mapped onto utf (\\char is needed for accents) ]] -local banner_utf_4 = [[ +local banner_utf_classes = [[ + +% some character classes for xetex; seems to be rather hard coded, these numbers +% and also a mix of several classes; here we do linebreaks + +]] + +local banner_utf_finish = [[ \endinput ]] +local xtxclasses = { + id = 1, + ex = 3, + is = 3, + cm = 256, + op = 2, + ns = 3, + cl = 3, +} + function scripts.chars.makeencoutf() - local chartable = input.find_file("char-def.lua") or "" + local chartable = resolvers.find_file("char-def.lua") or "" if chartable ~= "" then dofile(chartable) - if characters and characters.data then - local f = io.open("enco-utf.tex", 'w') + local function open(name,banner) + local f = io.open(name,'w') + if f then + logs.simple("writing '%s'",name) + f:write(format(banner_utf_module,name)) + f:write(banner) + f:write() + return f + end + end + local function close(f) + f:write(banner_utf_finish) + f:close() + end + local data = characters and characters.data + if data then + local list = table.sortedkeys(characters.data) + local f = open("xetx-utf.tex",banner_utf_mappings) if f then - f:write(banner_utf_1) - f:write(banner_utf_2) - local list = table.sortedkeys(characters.data) - local length = 0 for i=1,#list do local code = list[i] if code <= 0xFFFF then - local chr = characters.data[code] + local chr = data[code] local cc = chr.category if cc == 'll' or cc == 'lu' or cc == 'lt' then if not chr.lccode then chr.lccode = code end if not chr.uccode then chr.uccode = code end - f:write(format("\\setcclcucx %04X %04X %04X %% %s\n",code,chr.lccode,chr.uccode,chr.description)) + f:write(format('\\setXTXcharcodes "%05X "%05X "%05X %% %s\n',code,chr.lccode,chr.uccode,chr.description)) + end + end + end + f:write("\n") + for i=1,#list do + local code = list[i] + local chr = data[code] + if chr and chr.range then + local cc = chr.category + if cc == 'lo' then + f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharcodes\\recurselevel\\recurselevel\\recurselevel}\n',code,chr.range)) end - if #(chr.contextname or "") > length then + end + end + f:write(banner_utf_patch) + close(f) + end + local f = open("xetx-chr.tex",banner_utf_names) + if f then + local length = 0 + for i=1,#list do + local code = list[i] + if code > 0x5B and code <= 0xFFFF then + local chr = data[code] + if chr and #(chr.contextname or "") > length then length = #chr.contextname end end end - f:write(banner_utf_3) for i=1,#list do local code = list[i] if code > 0x5B and code <= 0xFFFF then - local chr = characters.data[code] - if chr.contextname then - local ch = char(code) ---~ if ch:find("[~#$%%^&{}\\]") then - f:write(format("\\def\\%s{\\char\"%04X } %% %s: %s\n", chr.contextname:rpadd(length," "), code, chr.description, ch)) ---~ else ---~ f:write(format("\\def\\%s{%s} %% %s\n", chr.contextname:rpadd(length," "), ch,chr.description)) ---~ end + local chr = data[code] + if chr and chr.contextname then + local ch = utfchar(code) + f:write(format("\\def\\%s{\\char\"%05X } %% %s: %s\n", chr.contextname:rpadd(length," "), code, chr.description, ch)) end end end - f:write(banner_utf_4) - f:close() + close(f) + end + local f = open("xetx-cls.tex",banner_utf_classes) + if f then + for k, v in pairs(xtxclasses) do + f:write(format("\\defineXTXcharinjectionclass[lb:%s]\n",k)) + end + f:write("\n") + local i_first, i_last, i_clb = nil, nil, nil + local function flush() + if i_first then + if i_first == i_last then + f:write(format('\\dosetXTXcharacterclass{"%05X}{lb:%s}\n',i_first,i_clb)) + else + f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n',i_first,i_last,i_clb)) + end + end + i_first, i_last, i_clb = nil, nil, nil + end + for i=1,#list do + local code = list[i] + local code_next = list[i+1] + local chr = data[code] + local chr_next = data[code_next] + local clb = chr and chr.linebreak + local lbc = xtxclasses[clb] + if not lbc then + flush() + elseif clb == i_clb then + if i_first then + i_last = code + else + i_first, i_last, i_clb = code, code, clb + end + else + flush() + i_first, i_last, i_clb = code, code, clb + end + end + flush() + f:write("\n") + for i=1,#list do + local code = list[i] + local chr = data[code] + if chr and chr.range then + local lbc = chr.linebreak + if xtxclasses[lbc] then + f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n',code,chr.range,lbc)) + end + end + end + close(f) end end end end -banner = banner .. " | character tools " +logs.extendbanner("Character Tools 0.10") messages.help = [[ --stix convert stix table to math table ---utf generate enco-utf.tex (used by xetex) +--xtx generate xetx-*.tex (used by xetex) --pdf generate pdfr-def.tex (used by pdftex) ]] @@ -211,10 +313,10 @@ if environment.argument("stix") then local inname = environment.files[1] or "" local outname = environment.files[2] or "" scripts.chars.stixtomkiv(inname,outname) -elseif environment.argument("utf") then +elseif environment.argument("xtx") then scripts.chars.makeencoutf() elseif environment.argument("pdf") then scripts.chars.makepdfr() else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-check.lua b/scripts/context/lua/mtx-check.lua index 7c44fa855..6be7f2765 100644 --- a/scripts/context/lua/mtx-check.lua +++ b/scripts/context/lua/mtx-check.lua @@ -34,7 +34,7 @@ do end end - local P, S, V, C, CP, CC = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cc + local P, R, S, V, C, CP, CC = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cc local i_m, d_m = P("$"), P("$$") local l_s, r_s = P("["), P("]") @@ -49,22 +49,27 @@ do local line = newline / function() validator.n = validator.n + 1 end - local grammar = P { "tokens", - ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, - ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), - ["grouped"] = CP() * C(l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g) * CC("group") / progress, - ["setup"] = CP() * C(l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s) * CC("setup") / progress, - ["display"] = CP() * C(d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m) * CC("display") / progress, - ["inline"] = CP() * C(i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m) * CC("inline") / progress, - ["errors"] = (V("gerror") + V("serror") + V("derror") + V("ierror")) * true, - ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, - ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, - ["derror"] = CP() * d_m * CC("display math error") / message, - ["ierror"] = CP() * i_m * CC("inline math error") / message, - } + -- local grammar = P { "tokens", + -- ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + -- ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), + -- ["grouped"] = CP() * C(l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g) * CC("group") / progress, + -- ["setup"] = CP() * C(l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s) * CC("setup") / progress, + -- ["display"] = CP() * C(d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m) * CC("display") / progress, + -- ["inline"] = CP() * C(i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m) * CC("inline") / progress, + -- ["errors"] = (V("gerror") + V("serror") + V("derror") + V("ierror")) * true, + -- ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, + -- ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, + -- ["derror"] = CP() * d_m * CC("display math error") / message, + -- ["ierror"] = CP() * i_m * CC("inline math error") / message, + -- } + + local startluacode = P("\\startluacode") + local stopluacode = P("\\stopluacode") + + local somecode = startluacode * (1-stopluacode)^1 * stopluacode local grammar = P { "tokens", - ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + ["tokens"] = (V("ignore") + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, ["setup"] = l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, @@ -75,6 +80,7 @@ do ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, ["derror"] = CP() * d_m * CC("display math error") / message, ["ierror"] = CP() * i_m * CC("inline math error") / message, + ["ignore"] = somecode, } function validator.check(str) @@ -117,19 +123,16 @@ function scripts.checker.check(filename) end end - -banner = banner .. " | tex check tools " +logs.extendbanner("Syntax Checking 0.10",true) messages.help = [[ --convert check tex file for errors ]] -input.verbose = true - if environment.argument("check") then scripts.checker.check(environment.files[1]) elseif environment.argument("help") then - input.help(banner,messages.help) + logs.help(messages.help) elseif environment.files[1] then scripts.checker.check(environment.files[1]) end diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 7d5eb7e80..2bae51501 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -23,30 +23,6 @@ function io.copydata(fromfile,tofile) io.savedata(tofile,io.loaddata(fromfile) or "") end --- luat-inp - -function input.locate_format(name) -- move this to core / luat-xxx - local barename, fmtname = name:gsub("%.%a+$",""), "" - if input.usecache then - local path = file.join(caches.setpath("formats")) -- maybe platform - fmtname = file.join(path,barename..".fmt") or "" - end - if fmtname == "" then - fmtname = input.find_files(barename..".fmt")[1] or "" - end - fmtname = input.clean_path(fmtname) - if fmtname ~= "" then - barename = fmtname:gsub("%.%a+$","") - local luaname, lucname = barename .. ".lua", barename .. ".luc" - if lfs.isfile(lucname) then - return barename, luaname - elseif lfs.isfile(luaname) then - return barename, luaname - end - end - return nil, nil -end - -- ctx ctxrunner = { } @@ -135,15 +111,15 @@ do elseif ctxdata.ctxname then ctlname = file.replacesuffix(ctxdata.ctxname,'ctl') else - input.report("invalid ctl name: %s",ctlname or "?") + logs.simple("invalid ctl name: %s",ctlname or "?") return end end if table.is_empty(ctxdata.prepfiles) then - input.report("nothing prepared, no ctl file saved") + logs.simple("nothing prepared, no ctl file saved") os.remove(ctlname) else - input.report("saving logdata in: %s",ctlname) + logs.simple("saving logdata in: %s",ctlname) f = io.open(ctlname,'w') if f then f:write("\n\n") @@ -188,8 +164,8 @@ do ctxdata.jobname = file.addsuffix(ctxdata.jobname,'tex') ctxdata.ctxname = file.addsuffix(ctxdata.ctxname,'ctx') - input.report("jobname: %s",ctxdata.jobname) - input.report("ctxname: %s",ctxdata.ctxname) + logs.simple("jobname: %s",ctxdata.jobname) + logs.simple("ctxname: %s",ctxdata.ctxname) -- mtxrun should resolve kpse: and file: @@ -206,7 +182,7 @@ do end end - if not found and defaultname and defaultname ~= "" and file.exists(defaultname) then + if not found and defaultname and defaultname ~= "" and lfs.isfile(defaultname) then usedname, found = defaultname, true end @@ -236,7 +212,7 @@ do ctxdata.flags = ctxrunner.reflag(ctxdata.flags) for _, message in ipairs(ctxdata.messages) do - input.report("ctx comment: %s", xml.tostring(message)) + logs.simple("ctx comment: %s", xml.tostring(message)) end xml.each(ctxdata.xmldata,"ctx:value[@name='job']", function(ek,e,k) @@ -326,10 +302,10 @@ do -- potential optimization: when mtxrun run internal command = xml.text(command) command = ctxrunner.justtext(command) -- command is still xml element here - input.report("command: %s",command) + logs.simple("command: %s",command) local result = os.spawn(command) or 0 if result > 0 then - input.report("error, return code: %s",result) + logs.simple("error, return code: %s",result) end if ctxdata.runlocal then oldfile = file.basename(oldfile) @@ -340,11 +316,11 @@ do file.syncmtimes(oldfile,newfile) ctxdata.prepfiles[oldfile] = true else - input.report("error, check target location of new file: %s", newfile) + logs.simple("error, check target location of new file: %s", newfile) ctxdata.prepfiles[oldfile] = false end else - input.report("old file needs no preprocessing") + logs.simple("old file needs no preprocessing") ctxdata.prepfiles[oldfile] = lfs.isfile(newfile) end end @@ -362,7 +338,8 @@ end -- rest scripts.context.multipass = { - suffixes = { ".tuo", ".tuc" }, +-- suffixes = { ".tuo", ".tuc" }, + suffixes = { ".tuc" }, nofruns = 8, } @@ -394,6 +371,7 @@ scripts.context.backends = { function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,currentrun,finalrun) -- take jobname from ctx + jobname = file.removesuffix(jobname) local f = io.open(jobname..".top","w") if f then local function someflag(flag) @@ -433,74 +411,124 @@ function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,curr local function setalways(format,...) f:write(format:format(...),"\n") end + -- + setalways("%% runtime options files (command line driven)") + -- setalways("\\unprotect") - setvalue('output' , "\\setupoutput[%s]", scripts.context.backends, 'pdftex') - setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun or 0, currentrun or 0) - setalways( "\\setupsystem[\\c!type=%s]",os.platform) + -- + setalways("%% special commands, mostly for the ctx development team") + -- + if environment.argument("dumpdelta") then + setalways("\\tracersdumpdelta") + elseif environment.argument("dumphash") then + setalways("\\tracersdumphash") + end + setalways("%% feedback and basic job control") + if type(environment.argument("track")) == "string" then + setvalue ("track" , "\\enabletrackers[%s]") + end + setfixed ("timing" , "\\usemodule[timing]") setfixed ("batchmode" , "\\batchmode") setfixed ("nonstopmode" , "\\nonstopmode") setfixed ("tracefiles" , "\\tracefilestrue") + setfixed ("nostats" , "\\nomkivstatistics") setfixed ("paranoid" , "\\def\\maxreadlevel{1}") - setvalues("modefile" , "\\readlocfile{%s}{}{}") + -- + setalways("%% handy for special styles") + -- + setalways("\\startluacode") + setalways("document = document or { }") + setalways(table.serialize(environment.arguments, "document.arguments")) + setalways(table.serialize(environment.files, "document.files")) + setalways("\\stopluacode") + -- + setalways("%% process info") + -- + setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun or 0, currentrun or 0) + setalways( "\\setupsystem[\\c!type=%s]",os.platform) setvalue ("inputfile" , "\\setupsystem[inputfile=%s]") setvalue ("result" , "\\setupsystem[file=%s]") setvalues("path" , "\\usepath[%s]") + setvalue ("setuppath" , "\\setupsystem[\\c!directory={%s}]") + setvalue ("randomseed" , "\\setupsystem[\\c!random=%s]") + setvalue ("arguments" , "\\setupenv[%s]") + setalways("%% modes") + setvalues("modefile" , "\\readlocfile{%s}{}{}") + setvalues("mode" , "\\enablemode[%s]", true) + if ctxdata then + setvalues(ctxdata.modes, "\\enablemode[%s]") + end + -- + setalways("%% options (not that important)") + -- + setalways("\\startsetups *runtime:options") + setvalue ('output' , "\\setupoutput[%s]", scripts.context.backends, 'pdftex') setfixed ("color" , "\\setupcolors[\\c!state=\\v!start]") - -- setfixed ("nompmode" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics - -- setfixed ("nomprun" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics - -- setfixed ("automprun" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics - setfixed ("fast" , "\\fastmode\n") - setfixed ("silentmode" , "\\silentmode\n") - setfixed ("nostats" , "\\nomkivstatistics\n") setvalue ("separation" , "\\setupcolors[\\c!split=%s]") - setvalue ("setuppath" , "\\setupsystem[\\c!directory={%s}]") setfixed ("noarrange" , "\\setuparranging[\\v!disable]") if environment.argument('arrange') and not finalrun then setalways( "\\setuparranging[\\v!disable]") end - setvalue ("randomseed" , "\\setupsystem[\\c!random=%s]") - setvalue ("arguments" , "\\setupenv[%s]") - -- singular and plural - setvalues("mode" , "\\enablemode[%s]", true) + setalways("\\stopsetups") + -- + setalways("%% styles and modules") + -- + setalways("\\startsetups *runtime:modules") setvalues("filter" , "\\useXMLfilter[%s]", true) setvalues("usemodule" , "\\usemodule[%s]", true) setvalues("environment" , "\\environment %s ", true) - -- ctx stuff if ctxdata then - setvalues(ctxdata.modes, "\\enablemode[%s]") setvalues(ctxdata.modules, "\\usemodule[%s]") setvalues(ctxdata.environments, "\\environment %s ") end - -- done - setalways("\\protect") - setalways("\\endinput") + setalways("\\stopsetups") + -- + setalways("%% done") + -- + setalways("\\protect \\endinput") f:close() end end function scripts.context.multipass.copyluafile(jobname) - io.savedata(jobname..".tuc",io.loaddata(jobname..".tua") or "") -end - -function scripts.context.multipass.copytuifile(jobname) - local f, g = io.open(jobname..".tui"), io.open(jobname..".tuo",'w') - if f and g then - g:write("% traditional utility file, only commands written by mtxrun/context\n%\n") - for line in f:lines() do - if line:find("^c ") then - g:write((line:gsub("^c ","")),"%\n") - end - end - g:write("\\endinput\n") - f:close() - g:close() +-- io.savedata(jobname..".tuc",io.loaddata(jobname..".tua") or "") + local tuaname, tucname = jobname..".tua", jobname..".tuc" + if lfs.isfile(tuaname) then + os.remove(tucname) + os.rename(tuaname,tucname) end end +-- obsolete: +-- +-- function scripts.context.multipass.copytuifile(jobname) +-- local tuiname, tuoname = jobname .. ".tui", jobname .. ".tuo" +-- if lfs.isfile(tuiname) then +-- local f, g = io.open(tuiname), io.open(tuoname,'w') +-- if f and g then +-- g:write("% traditional utility file, only commands written by mtxrun/context\n%\n") +-- for line in f:lines() do +-- if line:find("^c ") then +-- g:write((line:gsub("^c ","")),"%\n") +-- end +-- end +-- g:write("\\endinput\n") +-- f:close() +-- g:close() +-- end +-- else +-- -- os.remove(tuoname) +-- end +-- end + scripts.context.xmlsuffixes = table.tohash { "xml", } +scripts.context.luasuffixes = table.tohash { + "lua", +} + scripts.context.beforesuffixes = { "tuo", "tuc" } @@ -518,12 +546,17 @@ scripts.context.interfaces = { it = "cont-it", ro = "cont-ro", pe = "cont-pe", + -- for taco and me + -- xp = "cont-xp", } scripts.context.defaultformats = { "cont-en", "cont-nl", +-- "cont-xp", "mptopdf", +-- "metatex", + "metafun", "plain" } @@ -532,7 +565,7 @@ local function analyze(filename) if f then local t = { } local line = f:read("*line") or "" - local preamble = line:match("^%% *(.*)$") + local preamble = line:match("[\254\255]*%%%s+(.+)$") -- there can be an utf bomb in front if preamble then for key, value in preamble:gmatch("(%S+)=(%S+)") do t[key] = value @@ -541,42 +574,53 @@ local function analyze(filename) elseif line:find("^ 0 then - input.identify_cnf() - input.load_cnf() - input.expand_variables() + -- local interface = environment.argument("interface") -- todo: environment.argument("interface","en") interface = (type(interface) == "string" and interface) or "en" -- local formatname = scripts.context.interfaces[interface] or "cont-en" - local formatfile, scriptfile = input.locate_format(formatname) + local formatfile, scriptfile = resolvers.locate_format(formatname) + -- this catches the command line + if not formatfile or not scriptfile then + logs.simple("warning: no format found, forcing remake (commandline driven)") + scripts.context.generate() + scripts.context.make(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) + end + -- if formatfile and scriptfile then for _, filename in ipairs(files) do local basename, pathname = file.basename(filename), file.dirname(filename) @@ -584,137 +628,275 @@ function scripts.context.run(ctxdata) if pathname == "" then filename = "./" .. filename end + -- look at the first line local a = analyze(filename) - if a then - if a.interface and a.interface ~= interface then + if a and (a.engine == 'pdftex' or a.engine == 'xetex' or environment.argument("pdftex") or environment.argument("xetex")) then + local texexec = resolvers.find_file("texexec.rb") or "" + if texexec ~= "" then + local command = string.format("ruby %s %s",texexec,environment.reconstruct_commandline(environment.arguments_after)) + os.exec(command) + end + else + if a and a.interface and a.interface ~= interface then formatname = scripts.context.interfaces[a.interface] or formatname - formatfile, scriptfile = input.locate_format(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) end - end - -- we default to mkiv xml ! - if scripts.context.xmlsuffixes[file.extname(filename) or "?"] or environment.argument("forcexml") then - if environment.argument("mkii") then - filename = makestub("\\processXMLfilegrouped{%s}",filename) - else - filename = makestub("\\xmlprocess{\\xmldocument}{%s}{}",filename) + -- this catches the command line + if not formatfile or not scriptfile then + logs.simple("warning: no format found, forcing remake (source driven)") + scripts.context.generate() + scripts.context.make(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) end - end - -- - -- todo: also other stubs - -- - local resultname, oldbase, newbase = environment.argument("result"), "", "" - if type(resultname) == "string" then - oldbase = file.removesuffix(jobname) - newbase = file.removesuffix(resultname) - if oldbase ~= newbase then - for _, suffix in pairs(scripts.context.beforesuffixes) do - local oldname = file.addsuffix(oldbase,suffix) - local newname = file.addsuffix(newbase,suffix) - local tmpname = "keep-"..oldname - os.remove(tmpname) - os.rename(oldname,tmpname) - os.remove(oldname) - os.rename(newname,oldname) + if formatfile and scriptfile then + -- we default to mkiv xml ! + local suffix = file.extname(filename) or "?" + if scripts.context.xmlsuffixes[suffix] or environment.argument("forcexml") then + if environment.argument("mkii") then + filename = makestub("\\processXMLfilegrouped{%s}",filename) + else + filename = makestub("\\xmlprocess{\\xmldocument}{%s}{}",filename) + end + elseif scripts.context.luasuffixes[suffix] then + filename = makestub("\\ctxlua{dofile('%s')}",filename) end - else - resultname = nil - end - else - resultname = nil - end - -- - if environment.argument("autopdf") then - os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) - if resultname then - os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) - end - end - -- - local flags = { } - if environment.argument("batchmode") then - flags[#flags+1] = "--interaction=batchmode" - end - flags[#flags+1] = "--fmt=" .. string.quote(formatfile) - flags[#flags+1] = "--lua=" .. string.quote(scriptfile) - local command = string.format("luatex %s %s", table.concat(flags," "), string.quote(filename)) - local oldhash, newhash = scripts.context.multipass.hashfiles(jobname), { } - local once = environment.argument("once") - local maxnofruns = (once and 1) or scripts.context.multipass.nofruns - for i=1,maxnofruns do - -- 1:first run, 2:successive run, 3:once, 4:last of maxruns - local kindofrun = (once and 3) or (i==1 and 1) or (i==maxnofruns and 4) or 2 - scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,i,false) -- kindofrun, currentrun, final - input.report("run %s: %s",i,command) - local returncode, errorstring = os.spawn(command) - if not returncode then - input.report("fatal error, message: %s",errorstring or "?") - os.exit(1) - break - elseif returncode > 0 then - input.report("fatal error, code: %s",returncode or "?") - os.exit(returncode) - break - else - scripts.context.multipass.copyluafile(jobname) - scripts.context.multipass.copytuifile(jobname) - newhash = scripts.context.multipass.hashfiles(jobname) - if scripts.context.multipass.changed(oldhash,newhash) then - oldhash = newhash + -- + -- todo: also other stubs + -- + local resultname, oldbase, newbase = environment.argument("result"), "", "" + if type(resultname) == "string" then + oldbase = file.removesuffix(jobname) + newbase = file.removesuffix(resultname) + if oldbase ~= newbase then + for _, suffix in pairs(scripts.context.beforesuffixes) do + local oldname = file.addsuffix(oldbase,suffix) + local newname = file.addsuffix(newbase,suffix) + local tmpname = "keep-"..oldname + os.remove(tmpname) + os.rename(oldname,tmpname) + os.remove(oldname) + os.rename(newname,oldname) + end + else + resultname = nil + end else - break + resultname = nil + end + -- + if environment.argument("autopdf") then + os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + if resultname then + os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) + end + end + -- + local okay = statistics.check_fmt_status(formatfile) + if okay ~= true then + logs.simple("warning: %s, forcing remake",tostring(okay)) + scripts.context.generate() + scripts.context.make(formatname) + end + -- + local flags = { } + if environment.argument("batchmode") then + flags[#flags+1] = "--interaction=batchmode" + end + flags[#flags+1] = "--fmt=" .. string.quote(formatfile) + flags[#flags+1] = "--lua=" .. string.quote(scriptfile) + flags[#flags+1] = "--backend=pdf" + local command = string.format("luatex %s %s", table.concat(flags," "), string.quote(filename)) + local oldhash, newhash = scripts.context.multipass.hashfiles(jobname), { } + local once = environment.argument("once") + local maxnofruns = (once and 1) or scripts.context.multipass.nofruns + local arrange = environment.argument("arrange") + for i=1,maxnofruns do + -- 1:first run, 2:successive run, 3:once, 4:last of maxruns + local kindofrun = (once and 3) or (i==1 and 1) or (i==maxnofruns and 4) or 2 + scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,i,false) -- kindofrun, currentrun, final + logs.simple("run %s: %s",i,command) + local returncode, errorstring = os.spawn(command) + --~ if returncode == 3 then + --~ scripts.context.generate() + --~ scripts.context.make(formatname) + --~ returncode, errorstring = os.spawn(command) + --~ if returncode == 3 then + --~ logs.simple("fatal error, return code 3, message: %s",errorstring or "?") + --~ os.exit(1) + --~ end + --~ end + if not returncode then + logs.simple("fatal error, no return code, message: %s",errorstring or "?") + os.exit(1) + break + elseif returncode > 0 then + logs.simple("fatal error, return code: %s",returncode or "?") + os.exit(returncode) + break + else + scripts.context.multipass.copyluafile(jobname) + -- scripts.context.multipass.copytuifile(jobname) + newhash = scripts.context.multipass.hashfiles(jobname) + if scripts.context.multipass.changed(oldhash,newhash) then + oldhash = newhash + else + break + end + end + end + -- + if arrange then + local kindofrun = 3 + scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,i,true) -- kindofrun, currentrun, final + logs.simple("arrange run: %s",command) + local returncode, errorstring = os.spawn(command) + if not returncode then + logs.simple("fatal error, no return code, message: %s",errorstring or "?") + os.exit(1) + elseif returncode > 0 then + logs.simple("fatal error, return code: %s",returncode or "?") + os.exit(returncode) + end + end + -- + if environment.argument("purge") then + scripts.context.purge_job(filename) + elseif environment.argument("purgeall") then + scripts.context.purge_job(filename,true) + end + -- + os.remove(jobname..".top") + -- + if resultname then + for _, suffix in pairs(scripts.context.aftersuffixes) do + local oldname = file.addsuffix(oldbase,suffix) + local newname = file.addsuffix(newbase,suffix) + local tmpname = "keep-"..oldname + os.remove(newname) + os.rename(oldname,newname) + os.rename(tmpname,oldname) + end + logs.simple("result renamed to: %s",newbase) + end + -- + if environment.argument("purge") then + scripts.context.purge_job(resultname) + elseif environment.argument("purgeall") then + scripts.context.purge_job(resultname,true) + end + -- + if environment.argument("autopdf") then + if resultname then + os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) + else + os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + end + end + -- + if environment.argument("timing") then + logs.line() + logs.simple("you can process (timing) statistics with:",jobname) + logs.line() + logs.simple("context --extra=timing '%s'",jobname) + logs.simple("mtxrun --script timing --xhtml [--launch --remove] '%s'",jobname) + logs.line() end - end - end - -- - -- todo: extra arrange run - -- - if environment.argument("purge") then - scripts.context.purge_job(filename) - elseif environment.argument("purgeall") then - scripts.context.purge_job(filename,true) - end - -- - if resultname then - for _, suffix in pairs(scripts.context.aftersuffixes) do - local oldname = file.addsuffix(oldbase,suffix) - local newname = file.addsuffix(newbase,suffix) - local tmpname = "keep-"..oldname - os.remove(newname) - os.rename(oldname,newname) - os.rename(tmpname,oldname) - end - input.report("result renamed to: %s",newbase) - end - if environment.argument("autopdf") then - if resultname then - os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) else - os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + if formatname then + logs.simple("error, no format found with name: %s, skipping",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end + break end end - -- end else - input.verbose = true - input.report("error, no format found with name: %s",formatname) + if formatname then + logs.simple("error, no format found with name: %s, aborting",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end end end end -function scripts.context.make() - local list = (environment.files[1] and environment.files) or scripts.context.defaultformats +function scripts.context.pipe() + -- context --pipe + -- context --pipe --purge --dummyfile=whatever.tmp + local interface = environment.argument("interface") + interface = (type(interface) == "string" and interface) or "en" + local formatname = scripts.context.interfaces[interface] or "cont-en" + local formatfile, scriptfile = resolvers.locate_format(formatname) + if not formatfile or not scriptfile then + logs.simple("warning: no format found, forcing remake (commandline driven)") + scripts.context.generate() + scripts.context.make(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) + end + if formatfile and scriptfile then + local okay = statistics.check_fmt_status(formatfile) + if okay ~= true then + logs.simple("warning: %s, forcing remake",tostring(okay)) + scripts.context.generate() + scripts.context.make(formatname) + end + local flags = { + "--interaction=scrollmode", + "--fmt=" .. string.quote(formatfile), + "--lua=" .. string.quote(scriptfile), + "--backend=pdf", + } + local filename = environment.argument("dummyfile") or "" + if filename == "" then + filename = "\\relax" + logs.simple("entering scrollmode, end job with \\end") + else + filename = file.addsuffix(filename,"tmp") + io.savedata(filename,"\\relax") + scripts.context.multipass.makeoptionfile(filename,{ flags = flags },3,1,false) -- kindofrun, currentrun, final + logs.simple("entering scrollmode using '%s' with optionfile, end job with \\end",filename) + end + local command = string.format("luatex %s %s", table.concat(flags," "), string.quote(filename)) + os.spawn(command) + if environment.argument("purge") then + scripts.context.purge_job(filename) + elseif environment.argument("purgeall") then + scripts.context.purge_job(filename,true) + os.remove(filename) + end + else + if formatname then + logs.simple("error, no format found with name: %s, aborting",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end + end +end + +function scripts.context.make(name) + local runners = { + "luatools --make --compile ", + (environment.argument("pdftex") and "mtxrun texexec.rb --make --pdftex ") or false, + (environment.argument("xetex") and "mtxrun texexec.rb --make --xetex " ) or false, + } + local list = (name and { name }) or (environment.files[1] and environment.files) or scripts.context.defaultformats for _, name in ipairs(list) do name = scripts.context.interfaces[name] or name - local command = "luatools --make --compile " .. name - input.report("running command: %s",command) - os.spawn(command) + for _, runner in ipairs(runners) do + if runner then + local command = runner .. name + logs.simple("running command: %s",command) + os.spawn(command) + end + end end end function scripts.context.generate() -- hack, should also be a shared function local command = "luatools --generate " - input.report("running command: %s",command) + logs.simple("running command: %s",command) os.spawn(command) end @@ -725,23 +907,81 @@ function scripts.context.ctx() scripts.context.run(ctxdata) end +function scripts.context.autoctx() + local ctxdata = nil + local files = (filename and { filename }) or environment.files + local firstfile = #files > 0 and files[1] + if firstfile and file.extname(firstfile) == "xml" then + local f = io.open(firstfile) + if f then + local chunk = f:read(512) or "" + f:close() + local ctxname = string.match(chunk,"<%?context%-directive%s+job%s+ctxfile%s+([^ ]-)%s*?>") + if ctxname then + ctxdata = ctxrunner.new() + ctxdata.jobname = firstfile + ctxrunner.manipulate(ctxdata,ctxname) + end + end + end + scripts.context.run(ctxdata) +end + +-- todo: quite after first image + +local template = [[ + \starttext + \startMPpage %% %s + input "%s" ; + \stopMPpage + \stoptext +]] + +local loaded = false + +function scripts.context.metapost() + local filename = environment.files[1] or "" +--~ local tempname = "mtx-context-metapost.tex" +--~ local tempdata = string.format(template,"metafun",filename) +--~ io.savedata(tempname,tempdata) +--~ environment.files[1] = tempname +--~ environment.setargument("result",file.removesuffix(filename)) +--~ environment.setargument("once",true) +--~ scripts.context.run() + if not loaded then + dofile(resolvers.find_file("mlib-run.lua")) + loaded = true + commands = commands or { } + commands.writestatus = logs.report + end + local formatname = environment.arguments("format") or "metafun" + if formatname == "" or type(format) == "boolean" then + formatname = "metafun" + end + if environment.arguments("svg") then + metapost.directrun(formatname,filename,"svg") + else + metapost.directrun(formatname,filename,"mps") + end +end + function scripts.context.version() - local name = input.find_file("context.tex") + local name = resolvers.find_file("context.tex") if name ~= "" then - input.report("main context file: %s",name) + logs.simple("main context file: %s",name) local data = io.loaddata(name) if data then local version = data:match("\\edef\\contextversion{(.-)}") if version then - input.report("current version: %s",version) + logs.simple("current version: %s",version) else - input.report("context version: unknown, no timestamp found") + logs.simple("context version: unknown, no timestamp found") end else - input.report("context version: unknown, load error") + logs.simple("context version: unknown, load error") end else - input.report("main context file: unknown, 'context.tex' not found") + logs.simple("main context file: unknown, 'context.tex' not found") end end @@ -780,6 +1020,7 @@ local function purge_file(dfile,cfile) end function scripts.context.purge_job(jobname,all) + jobname = file.basename(jobname) local filebase = file.removesuffix(jobname) local deleted = { } for _, suffix in ipairs(obsolete_results) do @@ -794,7 +1035,7 @@ function scripts.context.purge_job(jobname,all) end end if #deleted > 0 then - input.report("purged files: %s", table.join(deleted,", ")) + logs.simple("purged files: %s", table.join(deleted,", ")) end end @@ -815,56 +1056,260 @@ function scripts.context.purge(all) end end if #deleted > 0 then - input.report("purged files: %s", table.join(deleted,", ")) + logs.simple("purged files: %s", table.join(deleted,", ")) end end --~ purge_for_files("test",true) --~ purge_all_files() +local function touch(name,pattern) + local name = resolvers.find_file(name) + local olddata = io.loaddata(name) + if olddata then + local oldversion, newversion = "", os.date("%Y.%m.%d %H:%M") + local newdata, ok = olddata:gsub(pattern,function(pre,mid,post) + oldversion = mid + return pre .. newversion .. post + end) + if ok > 0 then + local backup = file.replacesuffix(name,"tmp") + os.remove(backup) + os.rename(name,backup) + io.savedata(name,newdata) + return true, oldversion, newversion, name + else + return false + end + end +end + function scripts.context.touch() if environment.argument("expert") then - local function touch(name,pattern) - local name = input.find_file(name) - local olddata = io.loaddata(name) - if olddata then - local oldversion, newversion = "", os.date("%Y.%M.%d %H:%m") - local newdata, ok = olddata:gsub(pattern,function(pre,mid,post) - oldversion = mid - return pre .. newversion .. post - end) - if ok > 0 then - local backup = file.replacesuffix(name,"tmp") - os.remove(backup) - os.rename(name,backup) - io.savedata(name,newdata) - return true, oldversion, newversion, name - else - return false - end - end - end local done, oldversion, newversion, foundname = touch("context.tex", "(\\edef\\contextversion{)(.-)(})") if done then - input.report("old version : %s", oldversion) - input.report("new version : %s", newversion) - input.report("touched file: %s", foundname) + logs.simple("old version : %s", oldversion) + logs.simple("new version : %s", newversion) + logs.simple("touched file: %s", foundname) local ok, _, _, foundname = touch("cont-new.tex", "(\\newcontextversion{)(.-)(})") if ok then - input.report("touched file: %s", foundname) + logs.simple("touched file: %s", foundname) + end + local ok, _, _, foundname = touch("cont-xp.tex", "(\\edef\\contextversion{)(.-)(})") + if ok then + logs.simple("touched file: %s", foundname) + end + end + end +end + +-- extras + +function scripts.context.extras(pattern) + local found = resolvers.find_file("context.tex") + if found == "" then + logs.simple("unknown extra: %s", extra) + else + pattern = file.join(dir.expand_name(file.dirname(found)),string.format("mtx-context-%s.tex",pattern or "*")) + local list = dir.glob(pattern) + if not extra or extra == "" then + logs.extendbanner("extras") + else + logs.extendbanner(extra) + end + for k,v in ipairs(list) do + local data = io.loaddata(v) or "" + data = string.match(data,"begin help(.-)end help") + if data then + local h = { string.format("extra: %s (%s)",string.gsub(v,"^.*mtx%-context%-(.-)%.tex$","%1"),v) } + for s in string.gmatch(data,"%% *(.-)[\n\r]") do + h[#h+1] = s + end + logs.help(table.concat(h,"\n"),"nomoreinfo") + end + end + end +end + +function scripts.context.extra() + local extra = environment.argument("extra") + if type(extra) == "string" then + if environment.argument("help") then + scripts.context.extras(extra) + else + local fullextra = extra + if not string.find(fullextra,"mtx%-context%-") then + fullextra = "mtx-context-" .. extra + end + local foundextra = resolvers.find_file(fullextra) + if foundextra == "" then + scripts.context.extras() + return + else + logs.simple("processing extra: %s", foundextra) end + environment.setargument("purgeall",true) + local result = environment.setargument("result") or "" + if result == "" then + environment.setargument("result","context-extra") + end + scripts.context.run(nil,foundextra) end + else + scripts.context.extras() end end +-- todo: we need to do a dummy run + +function scripts.context.track() + environment.files = { "m-track" } + scripts.context.multipass.nofruns = 1 + scripts.context.run() + -- maybe filter from log +end + function scripts.context.timed(action) - input.starttiming(scripts.context) - action() - input.stoptiming(scripts.context) - input.report("total runtime: %s",input.elapsedtime(scripts.context)) + statistics.timed(action) end -banner = banner .. " | context tools " +local zipname = "cont-tmf.zip" +local mainzip = "http://www.pragma-ade.com/context/latest/" .. zipname +local validtrees = { "texmf-local", "texmf-context" } + +function zip.loaddata(zipfile,filename) -- should be in zip lib + local f = zipfile:open(filename) + if f then + local data = f:read("*a") + f:close() + return data + end + return nil +end + +function scripts.context.update() + local force = environment.argument("force") + local socket = require("socket") + local http = require("socket.http") + local basepath = resolvers.find_file("context.tex") or "" + if basepath == "" then + logs.simple("quiting, no 'context.tex' found") + return + end + local basetree = basepath.match(basepath,"^(.-)tex/context/base/context.tex$") or "" + if basetree == "" then + logs.simple("quiting, no proper tds structure (%s)",basepath) + return + end + local function is_okay(basetree) + for _, tree in next, validtrees do + local pattern = string.gsub(tree,"%-","%%-") + if basetree:find(pattern) then + return tree + end + end + return false + end + local okay = is_okay(basetree) + if not okay then + logs.simple("quiting, tree '%s' is protected",okay) + return + else + logs.simple("updating tree '%s'",okay) + end + if not lfs.chdir(basetree) then + logs.simple("quiting, unable to change to '%s'",okay) + return + end + logs.simple("fetching '%s'",mainzip) + local latest = http.request(mainzip) + if not latest then + logs.simple("context tree '%s' can be updated, use --force",okay) + return + end + io.savedata("cont-tmf.zip",latest) + if false then + -- variant 1 + os.execute("mtxrun --script unzip cont-tmf.zip") + else + -- variant 2 + local zipfile = zip.open(zipname) + if not zipfile then + logs.simple("quiting, unable to open '%s'",zipname) + return + end + local newfile = zip.loaddata(zipfile,"tex/context/base/context.tex") + if not newfile then + logs.simple("quiting, unable to open '%s'","context.tex") + return + end + local oldfile = io.loaddata(resolvers.find_file("context.tex")) or "" + local function versiontonumber(what,str) + local version = str:match("\\edef\\contextversion{(.-)}") or "" + local year, month, day, hour, minute = str:match("\\edef\\contextversion{(%d+)%.(%d+)%.(%d+) *(%d+)%:(%d+)}") + if year and minute then + local time = os.time { year=year,month=month,day=day,hour=hour,minute=minute} + logs.simple("%s version: %s (%s)",what,version,time) + return time + else + logs.simple("%s version: %s (unknown)",what,version) + return nil + end + end + local oldversion = versiontonumber("old",oldfile) + local newversion = versiontonumber("new",newfile) + if not oldversion or not newversion then + logs.simple("quiting, version cannot be determined") + return + elseif oldversion == newversion then + logs.simple("quiting, your current version is up-to-date") + return + elseif oldversion > newversion then + logs.simple("quiting, your current version is newer") + return + end + for k in zipfile:files() do + local filename = k.filename + if filename:find("/$") then + lfs.mkdir(filename) + else + local data = zip.loaddata(zipfile,filename) + if data then + if force then + io.savedata(filename,data) + end + logs.simple(filename) + end + end + end + for _, scriptname in next, { "luatools.lua", "mtxrun.lua" } do + local oldscript = resolvers.find_file(scriptname) or "" + if oldscript ~= "" and is_okay(oldscript) then + local newscript = "./scripts/context/lua/" .. scriptname + local data = io.loaddata(newscript) or "" + if data ~= "" then + logs.simple("replacing script '%s' by '%s'",oldscript,newscript) + if force then + io.savedata(oldscript,data) + end + end + else + logs.simple("keeping script '%s'",oldscript) + end + end + if force then + os.execute("context --generate") + os.execute("context --make") + end + end + if force then + logs.simple("context tree '%s' has been updated",okay) + else + logs.simple("context tree '%s' can been updated (use --force)",okay) + end +end + +logs.extendbanner("ConTeXt Tools 0.51",true) messages.help = [[ --run process (one or more) files (default action) @@ -877,41 +1322,82 @@ messages.help = [[ --once only one run --purge(all) purge files (--pattern=...) --result=name rename result to given name +--arrange run extra arrange pass --expert expert options --interface use specified user interface ]] messages.expert = [[ -expert options: also provide --expert +expert options: + +--touch update context version number (remake needed afterwards, also provide --expert) +--update update context from website (not to be confused with contextgarden) +--profile profile job (use: mtxrun --script profile --analyse) +--track show/set tracker variables +--timing generate timing and statistics overview +--extra=name process extra (mtx-context- in distribution) +]] + +messages.private = [[ +private options: ---touch update context version (remake needed afterwards) +--dumphash dump hash table afterwards +--dumpdelta dump hash table afterwards (only new entries) ]] -input.verbose = true +messages.special = [[ +special options: + +--pdftex process file with texexec using pdftex +--xetex process file with texexec using xetex + +--pipe don't check for file and enter scroll mode (--dummyfile=whatever.tmp) +]] if environment.argument("once") then scripts.context.multipass.nofruns = 1 end +if environment.argument("profile") then + os.setenv("MTX_PROFILE_RUN","YES") +end + if environment.argument("run") then - scripts.context.timed(scripts.context.run) -elseif environment.argument("make") then - scripts.context.timed(scripts.context.make) -elseif environment.argument("generate") then - scripts.context.timed(scripts.context.generate) +-- scripts.context.timed(scripts.context.run) + scripts.context.timed(scripts.context.autoctx) +elseif environment.argument("make") or environment.argument("generate") then + scripts.context.timed(function() + if environment.argument("generate") then + scripts.context.generate() + end + if environment.argument("make") then + scripts.context.make() + end + end) elseif environment.argument("ctx") then scripts.context.timed(scripts.context.ctx) +elseif environment.argument("mp") or environment.argument("metapost") then + scripts.context.timed(scripts.context.metapost) elseif environment.argument("version") then scripts.context.version() elseif environment.argument("touch") then scripts.context.touch() +elseif environment.argument("update") then + scripts.context.update() elseif environment.argument("expert") then - input.help(banner,messages.expert) + logs.help(table.join({ messages.expert, messages.private, messages.special },"\n")) +elseif environment.argument("extra") then + scripts.context.extra() elseif environment.argument("help") then - input.help(banner,messages.help) + logs.help(messages.help) +elseif environment.argument("track") and type(environment.argument("track")) == "boolean" then + scripts.context.track() elseif environment.files[1] then - scripts.context.timed(scripts.context.run) +-- scripts.context.timed(scripts.context.run) + scripts.context.timed(scripts.context.autoctx) +elseif environment.argument("pipe") then + scripts.context.timed(scripts.context.pipe) elseif environment.argument("purge") then -- only when no filename given, supports --pattern scripts.context.purge() @@ -919,6 +1405,9 @@ elseif environment.argument("purgeall") then -- only when no filename given, supports --pattern scripts.context.purge(true) else - input.help(banner,messages.help) + logs.help(messages.help) end +if environment.argument("profile") then + os.setenv("MTX_PROFILE_RUN","NO") +end diff --git a/scripts/context/lua/mtx-convert.lua b/scripts/context/lua/mtx-convert.lua index eca050f29..cf1d640c5 100644 --- a/scripts/context/lua/mtx-convert.lua +++ b/scripts/context/lua/mtx-convert.lua @@ -51,7 +51,7 @@ do dir.mkdirs(outputpath) local tmpname = file.replacesuffix(newname,"tmp") local command = graphics.converters[suffix](oldname,tmpname) - input.report("command: %s",command) + logs.simple("command: %s",command) io.flush() os.spawn(command) os.remove(newname) @@ -88,7 +88,7 @@ function scripts.convert.convertall() end end -banner = banner .. " | graphic conversion tools " +logs.extendbanner("Graphic Conversion Tools 0.10",true) messages.help = [[ --convertall convert all graphics on path @@ -98,10 +98,8 @@ messages.help = [[ --delay time between sweeps ]] -input.verbose = true - if environment.argument("convertall") then scripts.convert.convertall() else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua index ef9e37258..befba924e 100644 --- a/scripts/context/lua/mtx-fonts.lua +++ b/scripts/context/lua/mtx-fonts.lua @@ -6,7 +6,11 @@ if not modules then modules = { } end modules ['mtx-fonts'] = { license = "see context related readme files" } -dofile(input.find_file("font-syn.lua")) +if not fontloader then fontloader = fontforge end + +dofile(resolvers.find_file("font-otp.lua","tex")) +dofile(resolvers.find_file("font-syn.lua","tex")) +dofile(resolvers.find_file("font-mis.lua","tex")) scripts = scripts or { } scripts.fonts = scripts.fonts or { } @@ -15,57 +19,55 @@ function scripts.fonts.reload(verbose) fonts.names.load(true,verbose) end +function scripts.fonts.names(name) + name = name or "luatex-fonts-names.lua" + fonts.names.identify(true) + local data = fonts.names.data + if data then + data.fallback_mapping = nil + logs.report("fontnames","saving names in '%s'",name) + io.savedata(name,table.serialize(data,true)) + elseif lfs.isfile(name) then + os.remove(name) + end +end + local function showfeatures(v,n,f,s,t) - local iv = input.verbose - input.verbose = true - input.report("fontname: %s",v) - input.report("fullname: %s",n) - input.report("filename: %s",f) - if t == "otf" or t == "ttf" then - local filename = input.find_file(f,t) or "" - if filename ~= "" then - local ff = fontforge.open(filename) - if ff then - local data = fontforge.to_table(ff) - fontforge.close(ff) - local features = { } - local function collect(what) - if data[what] then - for _, d in ipairs(data[what]) do - if d.features then - for _, df in ipairs(d.features) do - features[df.tag] = features[df.tag] or { } - for _, ds in ipairs(df.scripts) do - features[df.tag][ds.script] = features[df.tag][ds.script] or { } - for _, lang in ipairs(ds.langs) do - features[df.tag][ds.script][lang] = true - end - end - end - end + logs.simple("fontname: %s",v) + logs.simple("fullname: %s",n) + logs.simple("filename: %s",f) + local features = fonts.get_features(f,t) + if features then + for what, v in table.sortedpairs(features) do + local data = features[what] + if data and next(data) then + logs.simple() + logs.simple("%s features:",what) + logs.simple() + logs.simple("feature script languages") + logs.simple() + for f,ff in table.sortedpairs(data) do + local done = false + for s, ss in table.sortedpairs(ff) do + if s == "*" then s = "all" end + if ss ["*"] then ss["*"] = nil ss.all = true end + if done then + f = "" + else + done = true end - end - end - collect('gsub') - collect('gpos') - input.report("") - for _, f in ipairs(table.sortedkeys(features)) do - local ff = features[f] - for _, s in ipairs(table.sortedkeys(ff)) do - local ss = ff[s] - input.report("feature: %s, script: %s, language: %s",f:lower(),s:lower(),(table.concat(table.sortedkeys(ss), " ")):lower()) + logs.simple("% -8s % -8s % -8s",f,s,table.concat(table.sortedkeys(ss), " ")) end end end end end - input.report("") - input.verbose = iv + logs.reportline() end function scripts.fonts.list(pattern,reload,all,info) if reload then - input.report("fontnames, reloading font database") + logs.simple("fontnames, reloading font database") end -- make a function for this pattern = pattern:lower() @@ -81,7 +83,7 @@ function scripts.fonts.list(pattern,reload,all,info) -- local t = fonts.names.list(pattern,reload) if reload then - input.report("fontnames, done\n\n") + logs.simple("fontnames, done\n\n") end if t then local s, w = table.sortedkeys(t), { 0, 0, 0 } @@ -112,51 +114,52 @@ function scripts.fonts.save(name,sub) local function save(savename,fontblob) if fontblob then savename = savename:lower() .. ".lua" - input.report("fontsave, saving data in %s",savename) - table.tofile(savename,fontforge.to_table(fontblob),"return") - fontforge.close(fontblob) + logs.simple("fontsave, saving data in %s",savename) + table.tofile(savename,fontloader.to_table(fontblob),"return") + fontloader.close(fontblob) end end if name and name ~= "" then - local filename = input.find_file(name) -- maybe also search for opentype + local filename = resolvers.find_file(name) -- maybe also search for opentype if filename and filename ~= "" then local suffix = file.extname(filename) if suffix == 'ttf' or suffix == 'otf' or suffix == 'ttc' then - local fontinfo = fontforge.info(filename) + local fontinfo = fontloader.info(filename) if fontinfo then + logs.simple("font: %s located as %s",name,filename) if fontinfo[1] then for _, v in ipairs(fontinfo) do - save(v.fontname,fontforge.open(filename,v.fullname)) + save(v.fontname,fontloader.open(filename,v.fullname)) end else - save(fontinfo.fullname,fontforge.open(filename)) + save(fontinfo.fullname,fontloader.open(filename)) end end else - input.verbose = true - input.report("font: %s not saved",filename) + logs.simple("font: %s not saved",filename) end else - input.verbose = true - input.report("font: %s not found",name) + logs.simple("font: %s not found",name) end end end -banner = banner .. " | font tools " +logs.extendbanner("Font Tools 0.20",true) messages.help = [[ --reload generate new font database --list [--info] list installed fonts (show info) --save save open type font in raw table +--names generate 'luatex-fonts-names.lua' (not for context!) --pattern=str filter files --all provide alternatives ]] if environment.argument("reload") then - local verbose = environment.argument("verbose") - scripts.fonts.reload(verbose) + scripts.fonts.reload(true) +elseif environment.argument("names") then + scripts.fonts.names() elseif environment.argument("list") then local pattern = environment.argument("pattern") or environment.files[1] or "" local all = environment.argument("all") @@ -168,5 +171,5 @@ elseif environment.argument("save") then local sub = environment.files[2] or "" scripts.fonts.save(name,sub) else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-grep.lua b/scripts/context/lua/mtx-grep.lua index 18e36d2ea..82a80314a 100644 --- a/scripts/context/lua/mtx-grep.lua +++ b/scripts/context/lua/mtx-grep.lua @@ -9,54 +9,85 @@ if not modules then modules = { } end modules ['mtx-babel'] = { scripts = scripts or { } scripts.grep = scripts.grep or { } -banner = banner .. " | simple grepper " +logs.extendbanner("Simple Grepper 0.10",true) + +local find, format = string.find, string.format + +local cr = lpeg.P("\r") +local lf = lpeg.P("\n") +local crlf = cr * lf +local newline = crlf + cr + lf +local content = lpeg.C((1-newline)^0) * newline + +local write_nl = texio.write_nl function scripts.grep.find(pattern, files, offset) if pattern and pattern ~= "" then - local format = string.format - input.starttiming(scripts.grep) - local count, nofmatches, noffiles, nofmatchedfiles = environment.argument("count"), 0, 0, 0 - local function grep(name) - local data = io.loaddata(name) - if data then - noffiles = noffiles + 1 - local n, m = 0, 0 - for line in data:gmatch("[^\n]+") do -- faster than loop over lines + statistics.starttiming(scripts.grep) + local nofmatches, noffiles, nofmatchedfiles = 0, 0, 0 + local n, m, name, check = 0, 0, "", nil + local count, nocomment = environment.argument("count"), environment.argument("nocomment") + if nocomment then + if count then + check = function(line) n = n + 1 - if line:find(pattern) then + if find(line,"^[%%#]") then + -- skip + elseif find(line,pattern) then m = m + 1 - if not count then - input.log(format("%s %s: %s",name,n,line)) - io.flush() - end end end - if count and m > 0 then - nofmatches = nofmatches + m - nofmatchedfiles = nofmatchedfiles + 1 - input.log(format("%s: %s",name,m)) - io.flush() + else + check = function(line) + n = n + 1 + if find(line,"^[%%#]") then + -- skip + elseif find(line,pattern) then + m = m + 1 + write_nl(format("%s %s: %s",name,n,line)) + io.flush() + end + end + end + else + if count then + check = function(line) + n = n + 1 + if find(line,pattern) then + m = m + 1 + end + end + else + check = function(line) + n = n + 1 + if find(line,pattern) then + m = m + 1 + write_nl(format("%s %s: %s",name,n,line)) + io.flush() + end end end end ---~ for i=offset or 1, #files do ---~ local filename = files[i] ---~ if filename:find("%*") then ---~ for _, name in ipairs(dir.glob(filename)) do ---~ grep(name) ---~ end ---~ else ---~ grep(filename) ---~ end ---~ end + local capture = (content/check)^0 for i=offset or 1, #files do - for _, name in ipairs(dir.glob(files[i])) do - grep(name) + for _, nam in ipairs(dir.glob(files[i])) do + name = nam + local data = io.loaddata(name) + if data then + n, m, noffiles = 0, 0, noffiles + 1 + capture:match(data) + if count and m > 0 then + nofmatches = nofmatches + m + nofmatchedfiles = nofmatchedfiles + 1 + write_nl(format("%s: %s",name,m)) + io.flush() + end + end end end - input.stoptiming(scripts.grep) + statistics.stoptiming(scripts.grep) if count and nofmatches > 0 then - input.log(format("\nfiles: %s, matches: %s, matched files: %s, runtime: %0.3f seconds",noffiles,nofmatches,nofmatchedfiles,input.loadtime(scripts.grep))) + write_nl(format("\nfiles: %s, matches: %s, matched files: %s, runtime: %0.3f seconds",noffiles,nofmatches,nofmatchedfiles,statistics.elapsedtime(scripts.grep))) end end end @@ -64,9 +95,10 @@ end messages.help = [[ --pattern search for pattern (optional) --count count matches only -]] +--nocomment skip lines that start with %% or # -input.verbose = true +patterns are lua patterns and need to be escaped accordingly +]] local pattern = environment.argument("pattern") local files = environment.files and #environment.files > 0 and environment.files @@ -76,5 +108,5 @@ if pattern and files then elseif files then scripts.grep.find(files[1], files, 2) else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-interface.lua b/scripts/context/lua/mtx-interface.lua index b57617846..264a2dbe4 100644 --- a/scripts/context/lua/mtx-interface.lua +++ b/scripts/context/lua/mtx-interface.lua @@ -11,7 +11,9 @@ local format = string.format scripts = scripts or { } scripts.interface = scripts.interface or { } -local flushers = { } +local flushers = { } +local userinterfaces = { 'en','cs','de','it','nl','ro','fr','pe' } +local messageinterfaces = { 'en','cs','de','it','nl','ro','fr','pe','no' } function flushers.scite(interface,collection) local result, i = {}, 0 @@ -59,25 +61,23 @@ end function flushers.raw(interface,collection) for _, command in ipairs(collection) do - input.report(command) + logs.simple(command) end end function scripts.interface.editor(editor) local interfaces= environment.files if #interfaces == 0 then - interfaces= { 'en','cs','de','it','nl','ro','fr' } + interfaces= userinterfaces end - local xmlfile = input.find_file("cont-en.xml") or "" + local xmlfile = resolvers.find_file("cont-en.xml") or "" if xmlfile == "" then - input.verbose = true - input.report("unable to locate cont-en.xml") + logs.simple("unable to locate cont-en.xml") end for _, interface in ipairs(interfaces) do - local keyfile = input.find_file(format("keys-%s.xml",interface)) or "" + local keyfile = resolvers.find_file(format("keys-%s.xml",interface)) or "" if keyfile == "" then - input.verbose = true - input.report("unable to locate keys-*.xml") + logs.simple("unable to locate keys-*.xml") else local collection = { } local mappings = { } @@ -112,7 +112,7 @@ function scripts.interface.editor(editor) end function scripts.interface.check() - local xmlfile = input.find_file("cont-en.xml") or "" + local xmlfile = resolvers.find_file("cont-en.xml") or "" if xmlfile ~= "" then local f = io.open("cont-en-check.tex","w") if f then @@ -138,14 +138,12 @@ function scripts.interface.check() end function scripts.interface.context() - local verbose = input.verbose - input.verbose = true - local filename = input.find_file("mult-def.lua") or "" + local filename = resolvers.find_file("mult-def.lua") or "" if filename ~= "" then local interface = dofile(filename) if interface and next(interface) then local variables, constants, commands, elements = interface.variables, interface.constants, interface.commands, interface.elements - local filename = input.find_file("cont-en.xml") or "" + local filename = resolvers.find_file("cont-en.xml") or "" local xmldata = filename ~= "" and (io.loaddata(filename) or "") local function flush(texresult,xmlresult,language,what,tag) local t = interface[what] @@ -156,7 +154,7 @@ function scripts.interface.context() local v = t[key] local value = v[language] or v["en"] if not value then - input.report(format("warning, no value for key '%s' for language '%s'",key,language)) + logs.simple(format("warning, no value for key '%s' for language '%s'",key,language)) else local value = t[key][language] or t[key].en texresult[#texresult+1] = format("\\setinterface%s{%s}{%s}",tag,key,value) @@ -194,9 +192,9 @@ function scripts.interface.context() local texfilename = format("mult-%s.tex",language) local xmlfilename = format("keys-%s.xml",language) io.savedata(texfilename,table.concat(texresult,"\n")) - input.report(format("saving interface definitions '%s'",texfilename)) + logs.simple(format("saving interface definitions '%s'",texfilename)) io.savedata(xmlfilename,table.concat(xmlresult,"\n")) - input.report(format("saving interface translations '%s'",xmlfilename)) + logs.simple(format("saving interface translations '%s'",xmlfilename)) if language ~= "en" and xmldata ~= "" then local newdata = xmldata:gsub("( 0 then - input.report("number of converted files: %i", #report) - input.report("") + logs.simple("number of converted files: %i", #report) + logs.simple("") for _, r in ipairs(report) do - input.report("%s => %s", r[1], r[2]) + logs.simple("%s => %s", r[1], r[2]) end else - input.report("no input files match %s", table.concat(files,' ')) + logs.simple("no input files match %s", table.concat(files,' ')) end else - input.report("no files match %s", table.concat(environment.files,' ')) + logs.simple("no files match %s", table.concat(environment.files,' ')) end end -banner = banner .. " | mptopdf converter " +logs.extendbanner("MetaPost to PDF Converter 0.51",true) messages.help = [[ --rawmp raw metapost run @@ -125,14 +125,12 @@ messages.help = [[ --latex force --tex=latex ]] -input.verbose = true - if environment.files[1] then scripts.mptopdf.convertall() else if not environment.arguments.help then - input.report("provide MP output file (or pattern)") - input.report("") + logs.simple("provide MP output file (or pattern)") + logs.simple("") end - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-package.lua b/scripts/context/lua/mtx-package.lua new file mode 100644 index 000000000..06c89907a --- /dev/null +++ b/scripts/context/lua/mtx-package.lua @@ -0,0 +1,68 @@ +if not modules then modules = { } end modules ['mtx-package'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, gsub, gmatch = string.format, string.gsub, string.gmatch + +scripts = scripts or { } +messages = messages or { } +scripts.package = scripts.package or { } + +function scripts.package.merge_luatex_files(name,strip) + local oldname = resolvers.find_file(name) or "" + oldname = file.replacesuffix(oldname,"lua") + if oldname == "" then + logs.simple("missing '%s'",name) + else + local newname = file.removesuffix(oldname) .. "-merged.lua" + local data = io.loaddata(oldname) or "" + if data == "" then + logs.simple("missing '%s'",newname) + else + logs.simple("loading '%s'",oldname) + local collected = { } + collected[#collected+1] = format("-- merged file : %s\n",newname) + collected[#collected+1] = format("-- parent file : %s\n",oldname) + collected[#collected+1] = format("-- merge date : %s\n",os.date()) + -- loadmodule can have extra arguments + for lib in gmatch(data,"loadmodule *%([\'\"](.-)[\'\"]") do + if file.basename(lib) ~= file.basename(newname) then + local fullname = resolvers.find_file(lib) or "" + if fullname == "" then + logs.simple("missing '%s'",lib) + else + logs.simple("fetching '%s'",fullname) + local data = io.loaddata(fullname) + if strip then + data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-%ldx%]%]%-%-[\n\r]*","") + data = gsub(data,"%-%-%~[^\n\r]*[\n\r]*","\n") + data = gsub(data,"%s+%-%-[^\n\r]*[\n\r]*","\n") + data = gsub(data,"[\n\r]+","\n") + end + collected[#collected+1] = "\ndo -- begin closure to overcome local limits and interference\n\n" + collected[#collected+1] = data + collected[#collected+1] = "\nend -- closure\n" + end + end + end + logs.simple("saving '%s'",newname) + io.savedata(newname,table.concat(collected)) + end + end +end + +logs.extendbanner("Package Tools 0.1",true) + +messages.help = [[ +--merge merge 'loadmodule' into merge file +]] + +if environment.argument("merge") then + scripts.package.merge_luatex_files(environment.files[1] or "") +else + logs.help(messages.help) +end diff --git a/scripts/context/lua/mtx-patterns.lua b/scripts/context/lua/mtx-patterns.lua index 7fda6900c..7f130465b 100644 --- a/scripts/context/lua/mtx-patterns.lua +++ b/scripts/context/lua/mtx-patterns.lua @@ -19,8 +19,8 @@ scripts.patterns.list = { { "cs", "hyph-cs.tex", "czech" }, { "??", "hyph-cy.tex", "welsh" }, { "da", "hyph-da.tex", "danish" }, - { "de", "hyph-de-1901.tex", "german, old spelling" }, - { "deo", "hyph-de-1996.tex", "german, new spelling" }, + { "deo", "hyph-de-1901.tex", "german, old spelling" }, + { "de", "hyph-de-1996.tex", "german, new spelling" }, --~ { "??", "hyph-el-monoton.tex", "" }, --~ { "??", "hyph-el-polyton.tex", "" }, --~ { "agr", "hyph-grc", "ancient greek" }, @@ -60,7 +60,7 @@ scripts.patterns.list = { { "??", "hyph-sr-cyrl.tex", "serbian" }, { "sv", "hyph-sv.tex", "swedish" }, { "tr", "hyph-tr.tex", "turkish" }, - { "??", "hyph-uk.tex", "ukrainian" }, + { "uk", "hyph-uk.tex", "ukrainian" }, { "??", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" }, } @@ -91,6 +91,7 @@ local permitted_characters = table.tohash { 0x0009, -- tab 0x0027, -- apostrofe 0x002D, -- hyphen + 0x200C, -- } function scripts.patterns.load(path,name,mnemonic,fullcheck) @@ -104,9 +105,9 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) local subdata = io.loaddata(subfull) or "" if subdata == "" then if mnemonic then - input.report("no subfile %s for language %s",subname,mnemonic) + logs.simple("no subfile %s for language %s",subname,mnemonic) else - input.report("no subfile %s",name) + logs.simple("no subfile %s",name) end end return previous .. subdata @@ -122,15 +123,15 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) line = line:gsub("%%","%%%%") if fullcheck then if mnemonic then - input.report("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line) + logs.simple("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line) else - input.report("invalid utf in file %s, line %s: %s",name,n,line) + logs.simple("invalid utf in file %s, line %s: %s",name,n,line) end else if mnemonic then - input.report("file %s for %s contains invalid utf",name,mnemonic) + logs.simple("file %s for %s contains invalid utf",name,mnemonic) else - input.report("file %s contains invalid utf",name) + logs.simple("file %s contains invalid utf",name) end break end @@ -154,17 +155,17 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) for k, v in pairs(h) do if not permitted_commands[k] then okay = false end if mnemonic then - input.report("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) + logs.simple("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) else - input.report("command \\%s found in file %s, n=%s",k,name,v) + logs.simple("command \\%s found in file %s, n=%s",k,name,v) end end if not environment.argument("fast") then for k, v in pairs(c) do if mnemonic then - input.report("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) + logs.simple("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) else - input.report("command \\%s found in comment of file %s, n=%s",k,name,v) + logs.simple("command \\%s found in comment of file %s, n=%s",k,name,v) end end end @@ -222,9 +223,9 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) local stripped = { } for k, v in pairs(p) do if mnemonic then - input.report("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) + logs.simple("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else - input.report("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v) + logs.simple("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v) end if not permitted_characters[k] then okay = false @@ -234,9 +235,9 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end for k, v in pairs(h) do if mnemonic then - input.report("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) + logs.simple("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else - input.report("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v) + logs.simple("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v) end if not permitted_characters[k] then okay = false @@ -246,15 +247,15 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end local stripset = "" for k, v in pairs(stripped) do - input.report("entries that contain character %s will be omitted",char(k)) + logs.simple("entries that contain character %s will be omitted",char(k)) stripset = stripset .. "%" .. char(k) end return okay, pats, hyps, comment, stripset, pused, hused else if mnemonic then - input.report("no file %s for language %s",fullname,mnemonic) + logs.simple("no file %s for language %s",fullname,mnemonic) else - input.report("no file %s",fullname) + logs.simple("no file %s",fullname) end return false, { }, { }, "", "", { }, { } end @@ -265,14 +266,14 @@ function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,commen local nofhyphenations = #hyphenations local pu = table.concat(table.sortedkeys(pused), " ") local hu = table.concat(table.sortedkeys(hused), " ") - input.report("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations) + logs.simple("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations) if mnemonic ~= "??" then local rmefile = file.join(destination,"lang-"..mnemonic..".rme") local patfile = file.join(destination,"lang-"..mnemonic..".pat") local hypfile = file.join(destination,"lang-"..mnemonic..".hyp") local topline = "% generated by mtxrun --script pattern --convert" local banner = "% for comment and copyright, see " .. rmefile - input.report("saving language data for %s",mnemonic) + logs.simple("saving language data for %s",mnemonic) if not comment or comment == "" then comment = "% no comment" end if not type(destination) == "string" then destination = "." end os.remove(rmefile) @@ -285,60 +286,58 @@ function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,commen end function scripts.patterns.prepare() - dofile(input.find_file("char-def.lua")) + dofile(resolvers.find_file("char-def.lua")) end function scripts.patterns.check() local path = environment.argument("path") or "." local found = false - local verbose = input.verbose - input.verbose = true if #environment.files > 0 then for _, name in ipairs(environment.files) do - input.report("checking language file %s", name) + logs.simple("checking language file %s", name) local okay = scripts.patterns.load(path,name,nil,not environment.argument("fast")) if #environment.files > 1 then - input.report("") + logs.simple("") end end else for k, v in pairs(scripts.patterns.list) do local mnemonic, name = v[1], v[2] - input.report("checking language %s, file %s", mnemonic, name) + logs.simple("checking language %s, file %s", mnemonic, name) local okay = scripts.patterns.load(path,name,mnemonic,not environment.argument("fast")) if not okay then - input.report("there are errors that need to be fixed") + logs.simple("there are errors that need to be fixed") end - input.report("") + logs.simple("") end end - input.verbose = verbose end function scripts.patterns.convert() local path = environment.argument("path") or "." - local destination = environment.argument("destination") or "." - if path == destination then - input.report("source path and destination path should differ (use --path and/or --destination)") + if path == "" then + logs.simple("provide sourcepath using --path ") else - local verbose = input.verbose - input.verbose = true - for k, v in pairs(scripts.patterns.list) do - local mnemonic, name = v[1], v[2] - input.report("converting language %s, file %s", mnemonic, name) - local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) - if okay then - scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) - else - input.report("convertion aborted due to error(s)") + local destination = environment.argument("destination") or "." + if path == destination then + logs.simple("source path and destination path should differ (use --path and/or --destination)") + else + for k, v in pairs(scripts.patterns.list) do + local mnemonic, name = v[1], v[2] + logs.simple("converting language %s, file %s", mnemonic, name) + local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) + if okay then + scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) + else + logs.simple("convertion aborted due to error(s)") + end + logs.simple("") end - input.report("") end end - input.verbose = verbose end -banner = banner .. " | pattern tools " +logs.extendbanner("Pattern Tools 0.20",true) messages.help = [[ --convert generate context language files (mnemonic driven, if not given then all) @@ -354,10 +353,10 @@ elseif environment.argument("convert") then scripts.patterns.prepare() scripts.patterns.convert() else - input.help(banner,messages.help) + logs.help(messages.help) end -- mtxrun --script pattern --check hyph-*.tex --- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination e:/tmp/patterns +-- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns +-- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination=e:/tmp/patterns diff --git a/scripts/context/lua/mtx-profile.lua b/scripts/context/lua/mtx-profile.lua new file mode 100644 index 000000000..d99f7e926 --- /dev/null +++ b/scripts/context/lua/mtx-profile.lua @@ -0,0 +1,164 @@ +if not modules then modules = { } end modules ['mtx-profile'] = { + version = 1.000, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- todo: also line number +-- todo: sort runtime as option + +local match, format, find = string.match, string.format, string.find + +scripts = scripts or { } +scripts.profiler = scripts.profiler or { } + +local timethreshold = 0 +local callthreshold = 2500 +local countthreshold = 2500 + +local functiontemplate = "%12s %03.4f %9i %s" +local calltemplate = "%9i %s" +local totaltemplate = "%i internal calls, %i function calls taking %3.4f seconds" +local thresholdtemplate = "thresholds: %i internal calls, %i function calls, %i seconds" + +function scripts.profiler.analyse(filename) + local f = io.open(filename) + if f then + local times, counts, calls = { }, { }, { } + local totalruntime, totalcount, totalcalls = 0, 0, 0 + while true do + local line = f:read() + if line then + local stacklevel, filename, functionname, linenumber, currentline, localtime, totaltime = line:match("^(%d+)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)") + if not filename then + -- next + elseif filename == "=[C]" then + if not functionname:find("^%(") then + calls[functionname] = (calls[functionname] or 0) + 1 + end + else + local filename = filename:match("^@(.*)$") + if filename then + local fi = times[filename] + if not fi then fi = { } times[filename] = fi end + fi[functionname] = (fi[functionname] or 0) + tonumber(localtime) + counts[functionname] = (counts[functionname] or 0) + 1 + end + end + else + break + end + end + f:close() + print("") + local loaded = { } + for _, filename in ipairs(table.sortedkeys(times)) do + local functions = times[filename] + for _, functionname in ipairs(table.sortedkeys(functions)) do + local totaltime = functions[functionname] + local count = counts[functionname] + totalcount = totalcount + count + if totaltime > timethreshold or count > countthreshold then + totalruntime = totalruntime + totaltime + local functionfile, somenumber = functionname:match("^@(.+):(.-)$") + if functionfile then + local number = tonumber(somenumber) + if number then + if not loaded[functionfile] then + loaded[functionfile] = string.splitlines(io.loaddata(functionfile) or "") + end + functionname = loaded[functionfile][number] or functionname + functionname = functionname:gsub("^%s*","") + functionname = functionname:gsub("%s*%-%-.*$","") + functionname = number .. ": " .. functionname + end + end + filename = file.basename(filename) + print(functiontemplate:format(filename,totaltime,count,functionname)) + end + end + end + print("") + for _, call in ipairs(table.sortedkeys(calls)) do + local n = calls[call] + totalcalls = totalcalls + n + if n > callthreshold then + print(calltemplate:format(n,call)) + end + end + print("") + print(totaltemplate:format(totalcalls,totalcount,totalruntime)) + print("") + print(thresholdtemplate:format(callthreshold,countthreshold,timethreshold)) + end +end + +function scripts.profiler.analyse(filename) + local f = io.open(filename) + local calls = { } + local lines = 0 + if f then + while true do + local line = f:read() + if line then + lines = lines + 1 + local c = match(line,"\\([a-zA-Z%!%?@]+) *%->") + if c then + local cc = calls[c] + if not cc then + calls[c] = 1 + else + calls[c] = cc + 1 + end + end + else + break + end + end + f:close() + local noc = 0 +local criterium = 100 + for name, n in next, calls do + if n > criterium then + if find(name,"^@@[a-z][a-z]") then + -- parameter + elseif find(name,"^[cvserft]%!") then + -- variables and constants + elseif find(name,"^%?%?[a-z][a-z]$") then + -- prefix + elseif find(name,"^%!%!") then + -- reserved + elseif find(name,"^@.+@$") then + -- weird + else + noc = noc + n + print(format("%6i: %s",n,name)) + end + end + end + print("") + print(format("number of lines: %s",lines)) + print(format("number of calls: %s",noc)) + print(format("criterium calls: %s",criterium)) + end +end + +--~ scripts.profiler.analyse("t:/manuals/mk/mk-fonts-profile.lua") +--~ scripts.profiler.analyse("t:/manuals/mk/mk-introduction-profile.lua") + +logs.extendbanner("LuaTeX Profiler 1.00",true) + +messages.help = [[ +--analyse analyse lua calls +--trace analyse tex calls +]] + +if environment.argument("analyse") then + scripts.profiler.analyse(environment.files[1] or "luatex-profile.log") +elseif environment.argument("trace") then + scripts.profiler.analyse(environment.files[1] or "temp.log") +else + logs.help(messages.help) +end diff --git a/scripts/context/lua/mtx-server-ctx-fonttest.lua b/scripts/context/lua/mtx-server-ctx-fonttest.lua new file mode 100644 index 000000000..efaae66e3 --- /dev/null +++ b/scripts/context/lua/mtx-server-ctx-fonttest.lua @@ -0,0 +1,681 @@ +if not modules then modules = { } end modules ['mtx-server-ctx-fonttest'] = { + version = 1.001, + comment = "Font Feature Tester", + author = "Hans Hagen", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +dofile(resolvers.find_file("l-aux.lua","tex")) +dofile(resolvers.find_file("trac-lmx.lua","tex")) +dofile(resolvers.find_file("font-ott.lua","tex")) +dofile(resolvers.find_file("font-syn.lua","tex")) +dofile(resolvers.find_file("font-mis.lua","tex")) +dofile(resolvers.find_file("font-otp.lua","tex")) + +local format, gsub, concat, match, find = string.format, string.gsub, table.concat, string.match, string.find + +local sample_line = "This is a sample line!" +local tempname = "mtx-server-ctx-fonttest-temp" +local temppath = caches.setpath("temp","mtx-server-ctx-fonttest") +local basename = "mtx-server-ctx-fonttest-data.lua" +local basepath = temppath + +for _, suffix in ipairs { "tex", "pdf", "log" } do + os.remove(file.join(temppath,file.addsuffix(tempname,suffix))) +end + +local process_templates = { } + +process_templates.default = [[ +\starttext + \setcharactermirroring[1] + \definefontfeature[sample][%s] + \definedfont[name:%s*sample] + \startTEXpage[offset=3pt] + \detokenize{%s} + \stopTEXpage +\stoptext +]] + +process_templates.cache = [[ +\starttext + \definedfont[name:%s] + \startTEXpage[offset=3pt] + cached: \detokenize{%s} + \stopTEXpage +\stoptext +]] + +process_templates.trace = [[ +\usemodule[fnt-20] + +\definefontfeature[sample][%s] + +\setupcolors[state=start] + +\setcharactermirroring[1] + +\setvariables + [otftracker] + [title=Test Run, + font=name:%s, + direction=0, + features=sample, + sample={‍\detokenize{%s}}] +]] + +local javascripts = [[ +function selected_radio(name) { + var form = document.forms["main-form"] ; + var script = form.elements[name] ; + if (script) { + var n = script.length ; + if (n) { + for (var i=0; i%s%s%s%s%s",id,id,name,file,sub,type) + end + end + listoffonts[#listoffonts+1] = "" + return concat(listoffonts,"\n") + end + end + return "no fonts" +end + +local edit_template = [[ + +

name:    title:  +

scripts: %s +

languages: %s +

features: %s +

options: %s +]] + +local result_template = [[ +

+ +

results: + tex file + pdf file +

+]] + +scripts.webserver.registerpath(temppath) + +local function edit_font(currentfont,detail,tempname) + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local htmldata = showfeatures(fontfile) + if htmldata then + local features, languages, scripts, options = { }, { }, { }, { } + for k,v in ipairs(table.sortedkeys(htmldata.scripts)) do + local s = fonts.otf.tables.scripts[v] or v + if detail and v == detail.script then + scripts[#scripts+1] = format(" %s",s,v,v,v,v) + else + scripts[#scripts+1] = format(" %s",s,v,v,v,v) + end + end + for k,v in ipairs(table.sortedkeys(htmldata.languages)) do + local l = fonts.otf.tables.languages[v] or v + if detail and v == detail.language then + languages[#languages+1] = format(" %s",l,v,v,v,v) + else + languages[#languages+1] = format(" %s",l,v,v,v,v) + end + end + for k,v in ipairs(table.sortedkeys(htmldata.features)) do + local f = fonts.otf.tables.features[v] or v + if detail and detail["f-"..v] then + features[#features+1] = format(" %s",f,v,v,v,v) + else + features[#features+1] = format(" %s",f,v,v,v,v) + end + end + for k, v in ipairs { "trace", "basemode" } do + if detail and detail["o-"..v] then + options[#options+1] = format(" %s",v,v,v) + else + options[#options+1] = format(" %s",v,v,v) + end + end + local e = format(edit_template, + (detail and detail.sampletext) or sample_line,(detail and detail.name) or "no name",(detail and detail.title) or "", + concat(scripts," "),concat(languages," "),concat(features," "),concat(options," ")) + if tempname then + local pdffile, texfile = file.addsuffix(tempname,"pdf"), file.addsuffix(tempname,"tex") + local r = format(result_template,pdffile,texfile,pdffile) + return e .. r, htmldata.javascript or "" + else + return e, htmldata.javascript or "" + end + else + return "error, nothing set up yet" + end +end + +local function process_font(currentfont,detail) -- maybe just fontname + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local features = { + "mode=node", + format("language=%s",detail.language or "dflt"), + format("script=%s",detail.script or "dflt"), + } + for k,v in pairs(detail) do + local f = match(k,"^f%-(.*)$") + if f then + features[#features+1] = format("%s=yes",f) + end + end + local variant = process_templates.default + if detail["o-trace"] then + variant = process_templates.trace + end + local sample = string.strip(detail.sampletext or "") + if sample == "" then sample = sample_line end + logs.simple("sample text: %s",sample) + io.savedata(file.join(temppath,file.addsuffix(tempname,"tex")),format(variant,concat(features,","),currentfont,sample)) + os.execute(format("mtxrun --path=%s --script context --once --batchmode --mode=*nofonts %s",temppath,tempname)) + return edit_font(currentfont,detail,tempname) +end + +local tex_template = [[ +

+%s
+
+]] + +local function show_source(currentfont,detail) + if tempname and tempname ~= "" then + return format(tex_template,io.loaddata(file.join(temppath,file.addsuffix(tempname,"tex"))) or "no source yet") + else + return "no source file" + end +end + +local function show_log(currentfont,detail) + if tempname and tempname ~= "" then + local data = io.loaddata(file.join(temppath,file.addsuffix(tempname,'log'))) or "no log file yet" + data = gsub(data,"[%s%%]*begin of optionfile.-end of optionfile[%s%%]*","\n") + return format(tex_template,data) + else + return "no log file" + end +end + +local function show_font(currentfont,detail) + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local features = fonts.get_features(fontfile) + local result = { } + result[#result+1] = format("

names

",what) + result[#result+1] = "" + result[#result+1] = format("",currentfont) + result[#result+1] = format("",fontname) + result[#result+1] = format("",fontfile) + result[#result+1] = "
fontname:%s
fullname:%s
filename:%s
" + if features then + for what, v in table.sortedpairs(features) do + local data = features[what] + if data and next(data) then + result[#result+1] = format("

%s features

",what) + result[#result+1] = "" + result[#result+1] = "" + for f,ff in table.sortedpairs(data) do + local done = false + for s, ss in table.sortedpairs(ff) do + if s == "*" then s = "all" end + if ss ["*"] then ss["*"] = nil ss.all = true end + if done then + f = "" + else + done = true + end + local title = fonts.otf.tables.features[f] or "" + result[#result+1] = format("",title,f,s,concat(table.sortedkeys(ss)," ")) + end + end + result[#result+1] = "
featuretag script languages 
%s  %s  %s  %s  
" + end + end + else + result[#result+1] = "

This font has no features." + end + return concat(result,"\n") +end + + +local info_template = [[ +

+version   : %s
+comment   : %s
+author    : %s
+copyright : %s
+
+maillist  : ntg-context at ntg.nl
+webpage   : www.pragma-ade.nl
+wiki      : contextgarden.net
+
+]] + +local function info_about() + local m = modules ['mtx-server-ctx-fonttest'] + return format(info_template,m.version,m.comment,m.author,m.copyright) +end + +local save_template = [[ + the current setup has been saved: +

+ + + + + + + + + +
name  %s
title  %s
font  %s
script  %s
language  %s
features  %s
options  %s
sampletext %s
+]] + +local function loadbase() + local datafile = file.join(basepath,basename) + local storage = io.loaddata(datafile) or "" + if storage == "" then + storage = { } + else + logs.simple("loading '%s'",datafile) + storage = loadstring(storage) + storage = (storage and storage()) or { } + end + return storage +end + +local function loadstored(detail,currentfont,name) + local storage = loadbase() + storage = storage and storage[name] + if storage then + currentfont = storage.font + detail.script = storage.script or detail.script + detail.language = storage.language or detail.language + detail.title = storage.title or detail.title + detail.sampletext = storage.text or detail.sampletext + detail.name = name or "no name" + for k,v in pairs(storage.features) do + detail["f-"..k] = v + end + for k,v in pairs(storage.options) do + detail["o-"..k] = v + end + end + detail.loadname = nil + return detail, currentfont +end + +local function savebase(storage,name) + local datafile = file.join(basepath,basename) + logs.simple("saving '%s' in '%s'",name or "data",datafile) + io.savedata(datafile,table.serialize(storage,true)) +end + +local function deletestored(detail,currentfont,name) + local storage = loadbase() + if storage and name and storage[name] then + logs.simple("deleting '%s' from base",name) + storage[name] = nil + savebase(storage) + end + detail.deletename = nil + return detail, "" +end + +local function save_font(currentfont,detail) + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local name, title, script, language, features, options, text = currentfont, "", "dflt", "dflt", { }, { }, "" + if detail then + local htmldata = showfeatures(fontfile) + script = detail.script or script + language = detail.language or language + text = string.strip(detail.sampletext or text) + name = string.strip(detail.name or name) + title = string.strip(detail.title or title) + for k,v in pairs(htmldata.features) do + if detail["f-"..k] then features[k] = true end + end + for k,v in ipairs { "trace", "basemode" } do + if detail["o-"..v] then options[k] = true end + end + end + if name == "" then + name = "no name" + end + local storage = loadbase() + storage[name] = { + font = currentfont, title = title, script = script, language = language, features = features, options = options, text = text, + } + savebase(storage,name) + return format(save_template,name,title,currentfont,script,language,concat(table.sortedkeys(features)," "),concat(table.sortedkeys(options)," "),text) +end + +local function load_font(currentfont) + local datafile = file.join(basepath,basename) + local storage = loadbase(datafile) + local result = {} + result[#result+1] = format("del name font fontname script language features title sampletext ") + for k,v in table.sortedpairs(storage) do + local fontname, fontfile, issub = fonts.names.specification(v.font or "") + result[#result+1] = format("x %s %s %s %s %s %s %s %s ", + k,k,k,v.font,fontname,v.script,v.language,concat(table.sortedkeys(v.features)," "),v.title or "no title",v.text or "") + end + if #result == 1 then + return "nothing saved yet" + else + return format("%s
",concat(result,"\n")) + end +end + +local function reset_font(currentfont) + return edit_font(currentfont) +end + +local extras_template = [[ + remake font database (take some time)

+]] + +local function do_extras(detail,currentfont,extra) + return extras_template +end + +local extras = { } + +local function do_extra(detail,currentfont,extra) + local e = extras[extra] + if e then e(detail,currentfont,extra) end + return do_extras(detail,currentfont,extra) +end + +function extras.reload() + local command = "mtxrun --script font --reload" + logs.simple("run command: %s",command) + os.execute(command) + return do_extras() +end + + +local status_template = [[ + +]] + +function doit(configuration,filename,hashed) + + local start = os.clock() + + local detail = url.query(hashed.query or "") + + local currentfont = detail.currentfont + local action = detail.action + local selection = detail.selection + + local loadname = detail.loadname + local deletename = detail.deletename + local extra = detail.extra + + if loadname and loadname ~= "" then + detail, currentfont = loadstored(detail,currentfont,loadname) + action = "process" + elseif deletename and deletename ~= "" then + detail, currentfont = deletestored(detail,currentfont,deletename) + action = "load" + elseif selection and selection ~= "" then + currentfont = selection + elseif extra and extra ~= "" then + do_extra(detail,currentfont,extra) + action = "extras" + end + + lmx.restore() + + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + + if fontfile then + lmx.variables['title-default'] = format('ConTeXt Font Tester: %s (%s)',fontname,fontfile) + else + lmx.variables['title-default'] = 'ConTeXt Font Tester' + end + + lmx.variables['color-background-green'] = '#4F6F6F' + lmx.variables['color-background-blue'] = '#6F6F8F' + lmx.variables['color-background-yellow'] = '#8F8F6F' + lmx.variables['color-background-purple'] = '#8F6F8F' + + lmx.variables['color-background-body'] = '#808080' + lmx.variables['color-background-main'] = '#3F3F3F' + lmx.variables['color-background-one'] = lmx.variables['color-background-green'] + lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] + + lmx.variables['title'] = lmx.variables['title-default'] + + lmx.set('title', lmx.get('title')) + lmx.set('color-background-one', lmx.get('color-background-green')) + lmx.set('color-background-two', lmx.get('color-background-blue')) + + -- lua table and adapt + + lmx.set('formaction', "mtx-server-ctx-fonttest.lua") + + local menu = { } + for k, v in ipairs { 'process', 'select', 'save', 'load', 'edit', 'reset', 'features', 'source', 'log', 'info', 'extras'} do + menu[#menu+1] = format("",v,v) + end + lmx.set('menu', concat(menu," ")) + + logs.simple("action: %s",action or "no action") + + lmx.set("status",format(status_template,currentfont or "")) + + local result + + if action == "select" then + lmx.set('maintext',select_font()) + elseif action == "info" then + lmx.set('maintext',info_about()) + elseif action == "extras" then + lmx.set('maintext',do_extras()) + elseif currentfont and currentfont ~= "" then + if action == "save" then + lmx.set('maintext',save_font(currentfont,detail)) + elseif action == "load" then + lmx.set('maintext',load_font(currentfont,detail)) + elseif action == "source" then + lmx.set('maintext',show_source(currentfont,detail)) + elseif action == "log" then + lmx.set('maintext',show_log(currentfont,detail)) + elseif action == "features" then + lmx.set('maintext',show_font(currentfont,detail)) + else + local e, s + if action == "process" then + e, s = process_font(currentfont,detail) + elseif action == "reset" then + e, s = reset_font(currentfont) + elseif action == "edit" then + e, s = edit_font(currentfont,detail) + else + e, s = process_font(currentfont,detail) + end + lmx.set('maintext',e) + lmx.set('javascriptdata',s) + lmx.set('javascripts',javascripts) + lmx.set('javascriptinit', "check_form()") + end + else + lmx.set('maintext',select_font()) + end + + result = { content = lmx.convert('context-fonttest.lmx') } + + logs.simple("time spent on page: %0.03f seconds",os.clock()-start) + + return result + +end + +return doit, true + +--~ make_lmx_page("test") diff --git a/scripts/context/lua/mtx-server-ctx-help.lua b/scripts/context/lua/mtx-server-ctx-help.lua new file mode 100644 index 000000000..c53d9f6e0 --- /dev/null +++ b/scripts/context/lua/mtx-server-ctx-help.lua @@ -0,0 +1,648 @@ +if not modules then modules = { } end modules ['mtx-server-ctx-help'] = { + version = 1.001, + comment = "Basic Definition Browser", + author = "Hans Hagen", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--~ dofile(resolvers.find_file("l-xml.lua","tex")) +dofile(resolvers.find_file("l-aux.lua","tex")) +dofile(resolvers.find_file("trac-lmx.lua","tex")) + +-- problem ... serialize parent stack + +local format = string.format +local concat = table.concat + +-- -- -- make this a module: cont-xx.lua + +document = document or { } +document.setups = document.setups or { } + +document.setups.div = { + pe = "
%s
" +} + +document.setups.span = { + pe = "%s" +} + +document.setups.translations = document.setups.translations or { + + nl = { + ["title"] = "setup", + ["formula"] = "formule", + ["number"] = "getal", + ["list"] = "lijst", + ["dimension"] = "maat", + ["mark"] = "markering", + ["reference"] = "verwijzing", + ["command"] = "commando", + ["file"] = "file", + ["name"] = "naam", + ["identifier"] = "naam", + ["text"] = "tekst", + ["section"] = "sectie", + ["singular"] = "naam enkelvoud", + ["plural"] = "naam meervoud", + ["matrix"] = "n*m", + ["see"] = "zie", + ["inherits"] = "erft van", + ["optional"] = "optioneel", + ["displaymath"] = "formule", + ["index"] = "ingang", + ["math"] = "formule", + ["nothing"] = "leeg", + ["file"] = "file", + ["position"] = "positie", + ["reference"] = "verwijzing", + ["csname"] = "naam", + ["destination"] = "bestemming", + ["triplet"] = "triplet", + ["word"] = "woord", + ["content"] = "tekst", + }, + + en = { + ["title"] = "setup", + ["formula"] = "formula", + ["number"] = "number", + ["list"] = "list", + ["dimension"] = "dimension", + ["mark"] = "mark", + ["reference"] = "reference", + ["command"] = "command", + ["file"] = "file", + ["name"] = "name", + ["identifier"] = "identifier", + ["text"] = "text", + ["section"] = "section", + ["singular"] = "singular name", + ["plural"] = "plural name", + ["matrix"] = "n*m", + ["see"] = "see", + ["inherits"] = "inherits from", + ["optional"] = "optional", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + + ["noargument"] = "\\cs", + ["oneargument"] = "\\cs#1{..}", + ["twoarguments"] = "\\cs#1#2{..}{..}", + ["threearguments"] = "\\cs#1#2#3{..}{..}{..}", + + }, + + de = { + ["title"] = "Setup", + ["formula"] = "Formel", + ["number"] = "Nummer", + ["list"] = "Liste", + ["dimension"] = "Dimension", + ["mark"] = "Beschriftung", + ["reference"] = "Referenz", + ["command"] = "Befehl", + ["file"] = "Datei", + ["name"] = "Name", + ["identifier"] = "Name", + ["text"] = "Text", + ["section"] = "Abschnitt", + ["singular"] = "singular", + ["plural"] = "plural", + ["matrix"] = "n*m", + ["see"] = "siehe", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + cz = { + ["title"] = "setup", + ["formula"] = "rovnice", + ["number"] = "cislo", + ["list"] = "seznam", + ["dimension"] = "dimenze", + ["mark"] = "znacka", + ["reference"] = "reference", + ["command"] = "prikaz", + ["file"] = "soubor", + ["name"] = "jmeno", + ["identifier"] = "jmeno", + ["text"] = "text", + ["section"] = "sekce", + ["singular"] = "jmeno v singularu", + ["plural"] = "jmeno v pluralu", + ["matrix"] = "n*m", + ["see"] = "viz", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + it = { + ["title"] = "setup", + ["formula"] = "formula", + ["number"] = "number", + ["list"] = "list", + ["dimension"] = "dimension", + ["mark"] = "mark", + ["reference"] = "reference", + ["command"] = "command", + ["file"] = "file", + ["name"] = "name", + ["identifier"] = "name", + ["text"] = "text", + ["section"] = "section", + ["singular"] = "singular name", + ["plural"] = "plural name", + ["matrix"] = "n*m", + ["see"] = "see", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + ro = { + ["title"] = "setari", + ["formula"] = "formula", + ["number"] = "numar", + ["list"] = "lista", + ["dimension"] = "dimensiune", + ["mark"] = "marcaj", + ["reference"] = "referinta", + ["command"] = "comanda", + ["file"] = "fisier", + ["name"] = "nume", + ["identifier"] = "nume", + ["text"] = "text", + ["section"] = "sectiune", + ["singular"] = "nume singular", + ["plural"] = "nume pluram", + ["matrix"] = "n*m", + ["see"] = "vezi", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + fr = { + ["title"] = "réglage", + ["formula"] = "formule", + ["number"] = "numéro", + ["list"] = "liste", + ["dimension"] = "dimension", + ["mark"] = "marquage", + ["reference"] = "reference", + ["command"] = "commande", + ["file"] = "fichier", + ["name"] = "nom", + ["identifier"] = "identificateur", + ["text"] = "texte", + ["section"] = "section", + ["singular"] = "nom singulier", + ["plural"] = "nom pluriel", + ["matrix"] = "n*m", + ["see"] = "vois", + ["inherits"] = "herite de", + ["optional"] = "optionel", + ["displaymath"] = "formule", + ["index"] = "entrée", + ["math"] = "formule", + ["nothing"] = "vide", + ["file"] = "fichier", + ["position"] = "position", + ["reference"] = "réference", + ["csname"] = "nom", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "mot", + ["content"] = "texte", + } + +} + +document.setups.formats = { + interface = [[%s]], + href = [[%s]], + source = [[%s]], + optional_single = "[optional string %s]", + optional_list = "[optional list %s]", + mandate_single = "[mandate string %s]", + mandate_list = "[mandate list %s]", + parameter = [[%s%s%s]], + parameters = [[%s
]], + listing = [[
%s]],
+    special = "%s",
+    default = "%s",
+}
+
+local function translate(tag,int,noformat)
+    local t = document.setups.translations
+    local te = t["en"]
+    local ti = t[int] or te
+    if noformat then
+        return ti[tag] or te[tag] or tag
+    else
+        return document.setups.formats.special:format(ti[tag] or te[tag] or tag)
+    end
+end
+
+local function translated(e,int)
+    local attributes = e.at
+    local s = attributes.type or "?"
+    local tag = s:match("^cd:(.*)$")
+    if attributes.default == "yes" then
+        return document.setups.formats.default:format(tag)
+    elseif tag then
+        return translate(tag,int)
+    else
+        return s
+    end
+end
+
+document.setups.loaded = document.setups.loaded or { }
+
+document.setups.current = { }
+document.setups.showsources = false
+
+function document.setups.load(filename)
+    filename = resolvers.find_file(filename) or ""
+    if filename ~= "" then
+        local current = document.setups.loaded[filename]
+        if not current then
+            local loaded = xml.load(filename)
+            if loaded then
+                -- xml.inject(document.setups.root,"/",loaded)
+                current = {
+                    file = filename,
+                    root = loaded,
+                    names = { },
+                    used = { },
+                }
+                document.setups.loaded[filename] = current
+            end
+        end
+        document.setups.current = current or { }
+    end
+end
+
+function document.setups.name(ek)
+    local at = ek.at
+    local name = at.name
+    if at.type == 'environment' then
+        name = "start" .. name
+    end
+    if at.variant then
+        name = name .. ":" .. at.variant
+    end
+    if at.generated == "yes" then
+        name = name .. "*"
+    end
+    return name:lower()
+end
+
+function document.setups.csname(ek,int)
+    local cs = ""
+    local at = ek.at
+    if at.type == 'environment' then
+        cs = translate("start",int,true) .. cs
+    end
+    for r, d, k in xml.elements(ek,'cd:sequence/(cd:string|variable)') do
+        local dk = d[k]
+        if dk.tg == "string" then
+            cs = cs .. dk.at.value
+        else
+            cs = cs .. dk.at.value -- to be translated
+        end
+    end
+    return cs
+end
+
+function document.setups.names()
+    local current = document.setups.current
+    local names = current.names
+    if not names or #names == 0 then
+        names = { }
+        local name = document.setups.name
+        local csname = document.setups.csname
+        for r, d, k in xml.elements(current.root,'cd:command') do
+            local dk = d[k]
+            names[#names+1] = { dk.at.name, csname(dk,int) }
+        end
+        table.sort(names, function(a,b) return a[2]:lower() < b[2]:lower() end)
+        current.names = names
+    end
+    return names
+end
+
+function document.setups.show(name)
+    local current = document.setups.current
+    if current.root then
+        local name = name:gsub("[<>]","")
+        local setup = xml.first(current.root,"cd:command[@name='" .. name .. "']")
+        current.used[#current.used+1] = setup
+        xml.sprint(setup)
+    end
+end
+
+function document.setups.showused()
+    local current = document.setups.current
+    if current.root and next(current.used) then
+        for k,v in ipairs(table.sortedkeys(current.used)) do
+            xml.sprint(current.used[v])
+        end
+    end
+end
+function document.setups.showall()
+    local current = document.setups.current
+    if current.root then
+        local list = { }
+        xml.each_element(current.root,"cd:command", function(r,d,t)
+            local ek = d[t]
+            list[document.setups.name(ek)] = ek
+        end )
+        for k,v in ipairs(table.sortedkeys(list)) do
+            xml.sprint(list[v])
+        end
+    end
+end
+function document.setups.resolve(name)
+    local current = document.setups.current
+    if current.root then
+        local e = xml.filter(current.root,format("cd:define[@name='%s']/text()",name))
+        if e then
+            xml.sprint(e)
+        end
+    end
+end
+
+function document.setups.collect(name,int)
+    local current = document.setups.current
+    local formats = document.setups.formats
+    local command = xml.filter(current.root,format("cd:command[@name='%s']",name))
+    if command then
+        local attributes = command.at
+        local data = {
+            command = command,
+            category = attributes.category or "",
+        }
+        if document.setups.showsources then
+            data.source = (attributes.file and formats.source:format(attributes.file,attributes.file)) or ""
+        else
+            data.source = attributes.file or ""
+        end
+        local sequence, n = { "\\" .. document.setups.csname(command,int) }, 0
+        local arguments = { }
+        for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do
+            n = n + 1
+            local attributes = d[k].at
+            if attributes.optional == 'yes' then
+                if attributes.list == 'yes' then
+                    sequence[#sequence+1] = formats.optional_list:format(n)
+                else
+                    sequence[#sequence+1] = formats.optional_single:format(n)
+                end
+            else
+                if attributes.list == 'yes' then
+                    sequence[#sequence+1] = formats.mandate_list:format(n)
+                else
+                    sequence[#sequence+1] = formats.mandate_single:format(n)
+                end
+            end
+        end
+        data.sequence = concat(sequence, " ")
+        local parameters, n = { }, 0
+        for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do
+            n = n + 1
+            if d[k].tg == "keywords" then
+                local left = sequence[n+1]
+                local right = { }
+                for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do
+                    local tag = d[k].tg
+                    if tag == "resolve" then
+                        local name = d[k].at.name or ""
+                        if name ~= "" then
+                            local resolved = xml.filter(current.root,format("cd:define[@name='%s']",name))
+                            for r, d, k in xml.elements(resolved,"cd:constant") do
+                                right[#right+1] = translated(d[k],int)
+                            end
+                        end
+                    else
+                        right[#right+1] = translated(d[k],int)
+                    end
+                end
+                parameters[#parameters+1] = formats.parameter:format(left,"",concat(right, ", "))
+            else
+                local what = sequence[n+1]
+                for r, d, k in xml.elements(d[k],"(cd:parameter|cd:inherit)") do
+                    local tag = d[k].tg
+                    local left, right = d[k].at.name or "?", { }
+                    if tag == "inherit" then
+                        local name = d[k].at.name or "?"
+                        local goto = document.setups.formats.href:format(name,"\\"..name)
+                        if #parameters > 0 and not parameters[#parameters]:find("
") then
+                            parameters[#parameters+1] = formats.parameter:format("
","","")
+                        end
+                        parameters[#parameters+1] = formats.parameter:format(what,formats.special:format(translate("inherits",int)),goto)
+                    else
+                        for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do
+                            local tag = d[k].tg
+                            if tag == "resolve" then
+                                local name = d[k].at.name or ""
+                                if name ~= "" then
+                                    local resolved = xml.filter(current.root,format("cd:define[@name='%s']",name))
+                                    for r, d, k in xml.elements(resolved,"cd:constant") do
+                                        right[#right+1] = translated(d[k],int)
+                                    end
+                                end
+                            else
+                                right[#right+1] = translated(d[k],int)
+                            end
+                        end
+                        parameters[#parameters+1] = formats.parameter:format(what,left,concat(right, ", "))
+                    end
+                    what = ""
+                end
+            end
+            parameters[#parameters+1] = formats.parameter:format("
","","")
+        end
+        data.parameters = parameters
+        return data
+    else
+        return nil
+    end
+end
+
+-- -- --
+
+tex = tex or { }
+
+lmx.variables['color-background-green']      = '#4F6F6F'
+lmx.variables['color-background-blue']       = '#6F6F8F'
+lmx.variables['color-background-yellow']     = '#8F8F6F'
+lmx.variables['color-background-purple']     = '#8F6F8F'
+
+lmx.variables['color-background-body']       = '#808080'
+lmx.variables['color-background-main']       = '#3F3F3F'
+lmx.variables['color-background-main-left']  = '#3F3F3F'
+lmx.variables['color-background-main-right'] = '#5F5F5F'
+lmx.variables['color-background-one']        = lmx.variables['color-background-green']
+lmx.variables['color-background-two']        = lmx.variables['color-background-blue']
+
+lmx.variables['title-default']               = 'ConTeXt Help Information'
+lmx.variables['title']                       = lmx.variables['title-default']
+
+function lmx.loadedfile(filename)
+    return io.loaddata(resolvers.find_file(filename)) -- return resolvers.texdatablob(filename)
+end
+
+-- -- --
+
+local interfaces = {
+    czech    = 'cz',
+    dutch    = 'nl',
+    english  = 'en',
+    french   = 'fr',
+    german   = 'de',
+    italian  = 'it',
+    persian  = 'pe',
+    romanian = 'ro',
+}
+
+local lastinterface, lastcommand, lastsource = "en", "", ""
+
+local function doit(configuration,filename,hashed)
+
+    local formats = document.setups.formats
+
+    local start = os.clock()
+
+    local detail = aux.settings_to_hash(hashed.query or "")
+
+    lastinterface, lastcommand, lastsource = detail.interface or lastinterface, detail.command or lastcommand, detail.source or lastsource
+
+    if lastinterface then
+        logs.simple("checking interface: %s",lastinterface)
+        document.setups.load(format("cont-%s.xml",lastinterface))
+    end
+
+    local div = document.setups.div[lastinterface]
+    local span = document.setups.span[lastinterface]
+
+    local result = { content = "error" }
+
+    local names, refs, ints = document.setups.names(lastinterface), { }, { }
+    for k,v in ipairs(names) do
+        refs[k] = document.setups.formats.href:format(v[1],v[2])
+    end
+    for k,v in ipairs(table.sortedkeys(interfaces)) do
+        ints[k] = document.setups.formats.interface:format(interfaces[v],v)
+    end
+
+    lmx.restore()
+    lmx.set('title', 'ConTeXt Help Information')
+    lmx.set('color-background-one', lmx.get('color-background-green'))
+    lmx.set('color-background-two', lmx.get('color-background-blue'))
+
+    local n = concat(refs,"
")
+    local i = concat(ints,"

")
+
+    if div then
+        lmx.set('names',div:format(n))
+        lmx.set('interfaces',div:format(i))
+    else
+        lmx.set('names', n)
+        lmx.set('interfaces', i)
+    end
+
+    -- first we need to add information about mkii/mkiv
+
+    if document.setups.showsources and lastsource and lastsource ~= "" then
+        -- todo: mkii, mkiv, tex (can be different)
+        local data = io.loaddata(resolvers.find_file(lastsource))
+        lmx.set('maintitle', lastsource)
+        lmx.set('maintext', formats.listing:format(data))
+        lastsource = ""
+    elseif lastcommand and lastcommand ~= "" then
+        local data = document.setups.collect(lastcommand,lastinterface)
+        if data then
+            lmx.set('maintitle', data.sequence)
+            local extra = { }
+            for k, v in ipairs { "environment", "category", "source" } do
+                if data[v] and data[v] ~= "" then
+                    lmx.set(v, data[v])
+                    extra[#extra+1] = v .. ": " .. data[v]
+                end
+            end
+            lmx.set('extra', concat(extra,", "))
+            lmx.set('maintext', formats.parameters:format(concat(data.parameters)))
+        else
+            lmx.set('maintext', "select command")
+        end
+    else
+        lmx.set('maintext', "no definition")
+    end
+
+    local content = lmx.convert('context-help.lmx')
+
+    logs.simple("time spent on page: %0.03f seconds",os.clock()-start)
+
+    return { content = content }
+end
+
+return doit, true
diff --git a/scripts/context/lua/mtx-server-ctx-startup.lua b/scripts/context/lua/mtx-server-ctx-startup.lua
new file mode 100644
index 000000000..fcb757b3e
--- /dev/null
+++ b/scripts/context/lua/mtx-server-ctx-startup.lua
@@ -0,0 +1,53 @@
+if not modules then modules = { } end modules ['mtx-server-ctx-startup'] = {
+    version   = 1.001,
+    comment   = "Overview Of Goodies",
+    author    = "Hans Hagen",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+dofile(resolvers.find_file("trac-lmx.lua","tex"))
+
+function doit(configuration,filename,hashed)
+
+    lmx.restore()
+
+    lmx.variables['color-background-green']  = '#4F6F6F'
+    lmx.variables['color-background-blue']   = '#6F6F8F'
+    lmx.variables['color-background-yellow'] = '#8F8F6F'
+    lmx.variables['color-background-purple'] = '#8F6F8F'
+
+    lmx.variables['color-background-body']   = '#808080'
+    lmx.variables['color-background-main']   = '#3F3F3F'
+    lmx.variables['color-background-one']    = lmx.variables['color-background-green']
+    lmx.variables['color-background-two']    = lmx.variables['color-background-blue']
+
+    lmx.variables['title']                   = "Overview Of Goodies"
+
+    lmx.set('title',                lmx.get('title'))
+    lmx.set('color-background-one', lmx.get('color-background-green'))
+    lmx.set('color-background-two', lmx.get('color-background-blue'))
+
+
+    local list = { }
+    local root = file.dirname(resolvers.find_file("mtx-server.lua") or ".")
+    if root == "" then root = "." end
+    local pattern = root .. "/mtx-server-ctx-*.lua"
+    local files = dir.glob(pattern)
+    for i=1,#files do
+        local filename = file.basename(files[i])
+        local name = string.match(filename,"mtx%-server%-ctx%-(.-)%.lua$")
+        if name and name ~= "startup" then
+            list[#list+1] = string.format("%s

",filename,name,name)
+        end
+    end
+
+    lmx.set('maintext',table.concat(list,"\n"))
+
+    result = { content = lmx.convert('context-base.lmx') }
+
+    return result
+
+end
+
+return doit, true
diff --git a/scripts/context/lua/mtx-server.lua b/scripts/context/lua/mtx-server.lua
index d9eb355f6..74f0ed924 100644
--- a/scripts/context/lua/mtx-server.lua
+++ b/scripts/context/lua/mtx-server.lua
@@ -9,9 +9,10 @@ if not modules then modules = { } end modules ['mtx-server'] = {
 scripts           = scripts           or { }
 scripts.webserver = scripts.webserver or { }
 
-dofile(input.find_file("l-url.lua"))
+dofile(resolvers.find_file("l-url.lua","tex"))
+dofile(resolvers.find_file("luat-soc.lua","tex"))
 
-local socket = require("socket")
+local socket = socket or require("socket") -- redundant in future version
 local format = string.format
 
 -- The following two lists are taken from webrick (ruby) and
@@ -126,13 +127,39 @@ local handlers = { }
 
 local function errormessage(client,configuration,n)
     local data = format("%s %s
%s %s
",n,messages[n],n,messages[n])
-    input.report("handling error %s: %s",n,messages[n])
+    logs.simple("handling error %s: %s",n,messages[n])
     handlers.generic(client,configuration,data,nil,true)
 end
 
+local validpaths, registered = { }, { }
+
+function scripts.webserver.registerpath(name)
+    if not registered[name] then
+        local cleanname = string.gsub(name,"%.%.","deleted-parent")
+        logs.simple("registering path '%s'",cleanname)
+        validpaths[#validpaths+1] = cleanname
+        registered[name] = true
+    end
+end
+
 function handlers.generic(client,configuration,data,suffix,iscontent)
     if not iscontent then
-        data = io.loaddata(file.join(configuration.root,data))
+        local name = data
+        logs.simple("requested file '%s'",name)
+        local fullname = file.join(configuration.root,name)
+        data = io.loaddata(fullname) or ""
+        if data == "" then
+            for n=1,#validpaths do
+                local fullname = file.join(validpaths[n],name)
+                data = io.loaddata(fullname) or ""
+                if data ~= "" then
+                    logs.simple("sending generic file '%s'",fullname)
+                    break
+                end
+            end
+        else
+            logs.simple("sending generic file '%s'",fullname)
+        end
     end
     if data and data ~= "" then
         client:send("HTTP/1.1 200 OK\r\n")
@@ -155,21 +182,37 @@ end
 --~     return { content = filename }
 --~ end
 
+local loaded = { }
+
 function handlers.lua(client,configuration,filename,suffix,iscontent,hashed) -- filename will disappear, and become hashed.filename
     local filename = file.join(configuration.scripts,filename)
-    if not input.aux.qualified_path(filename) then
+    if not file.is_qualified_path(filename) then
         filename = file.join(configuration.root,filename)
     end
     -- todo: split url in components, see l-url; rather trivial
-    input.report("locating script: %s",filename)
-    if lfs.isfile(filename) then
-        local result = loadfile(filename)
-        input.report("return type: %s",type(result))
-        if result and type(result) == "function" then
-         -- result() should return a table { [type=,] [length=,] content= }, function or string
-            result = result()
+    local result, keep = loaded[filename], false
+    if result then
+        logs.simple("reusing script: %s",filename)
+    else
+        logs.simple("locating script: %s",filename)
+        if lfs.isfile(filename) then
+            logs.simple("loading script: %s",filename)
+            result = loadfile(filename)
+            logs.simple("return type: %s",type(result))
+            if result and type(result) == "function" then
+             -- result() should return a table { [type=,] [length=,] content= }, function or string
+                result, keep = result()
+                if keep then
+                    logs.simple("saving script: %s",type(result))
+                    loaded[filename] = result
+                end
+            end
+        else
+            errormessage(client,configuration,404)
         end
-        if result and type(result) == "function" then
+    end
+    if result then
+        if type(result) == "function" then
             result = result(configuration,filename,hashed) -- second argument will become query
         end
         if result and type(result) == "string" then
@@ -181,9 +224,9 @@ function handlers.lua(client,configuration,filename,suffix,iscontent,hashed) --
                 local action = handlers[suffix] or handlers.generic
                 action(client,configuration,result.content,suffix,true) -- content
             elseif result.filename then
-                local suffix = file.extname(filename) or "text/html"
+                local suffix = file.extname(result.filename) or "text/html"
                 local action = handlers[suffix] or handlers.generic
-                action(client,configuration,filename,suffix,false) -- filename
+                action(client,configuration,result.filename,suffix,false) -- filename
             else
                 errormessage(client,configuration,404)
             end
@@ -198,18 +241,19 @@ end
 handlers.luc  = handlers.lua
 handlers.html = handlers.htm
 
-local indices = { "index.htm", "index.html" }
+local indices    = { "index.htm", "index.html" }
+local portnumber = 31415 -- pi suits tex
 
 function scripts.webserver.run(configuration)
     -- check configuration
-    configuration.port = tonumber(configuration.port or os.getenv("MTX_SERVER_PORT") or 8080) or 8080
+    configuration.port = tonumber(configuration.port or os.getenv("MTX_SERVER_PORT") or portnumber) or portnumber
     if not configuration.root or not lfs.isdir(configuration.root) then
         configuration.root = os.getenv("MTX_SERVER_ROOT") or "."
     end
     -- locate root and index file in tex tree
     if not lfs.isdir(configuration.root) then
         for _, name in ipairs(indices) do
-            local root = input.resolve("path:" .. name) or ""
+            local root = resolvers.resolve("path:" .. name) or ""
             if root ~= "" then
                 configuration.root = root
                 configuration.index = configuration.index or name
@@ -217,6 +261,7 @@ function scripts.webserver.run(configuration)
             end
         end
     end
+    configuration.root = dir.expand_name(configuration.root)
     if not configuration.index then
         for _, name in ipairs(indices) do
             if lfs.isfile(file.join(configuration.root,name)) then
@@ -226,14 +271,17 @@ function scripts.webserver.run(configuration)
         end
         configuration.index = configuration.index or "unknown"
     end
-    configuration.scripts = configuration.scripts or "cgi"
+    if not configuration.scripts or configuration.scripts == "" then
+        configuration.scripts = dir.expand_name(file.join(configuration.root or ".",configuration.scripts or "."))
+    end
     -- so far for checks
-    input.report("running at port: %s",configuration.port)
-    input.report("document root: %s",configuration.root)
-    input.report("main index file: %s",configuration.index)
-    input.report("scripts subpath: %s",configuration.scripts)
+    logs.simple("running at port: %s",configuration.port)
+    logs.simple("document root: %s",configuration.root or resolvers.ownpath)
+    logs.simple("main index file: %s",configuration.index)
+    logs.simple("scripts subpath: %s",configuration.scripts)
     local server = assert(socket.bind("*", configuration.port))
     while true do -- no multiple clients
+        local start = os.clock()
         local client = server:accept()
         client:settimeout(configuration.timeout or 60)
         local request, e = client:receive()
@@ -241,26 +289,26 @@ function scripts.webserver.run(configuration)
             errormessage(client,configuration,404)
         else
             local from = client:getpeername()
-            input.report("request from: %s",tostring(from))
+            logs.simple("request from: %s",tostring(from))
             local fullurl = request:match("GET (.+) HTTP/.*$") -- todo: more clever
-fullurl = socket.url.unescape(fullurl)
-local hashed = url.hashed(fullurl)
-local query = url.query(hashed.query)
-filename = hashed.path
+            fullurl = socket.url.unescape(fullurl)
+            local hashed = url.hashed(fullurl)
+            local query = url.query(hashed.query)
+            local filename = hashed.path
             if filename then
                 filename = socket.url.unescape(filename)
-                input.report("requested action: %s",filename)
+                logs.simple("requested action: %s",filename)
                 if filename:find("%.%.") then
                     filename = nil -- invalid path
                 end
                 if filename == nil or filename == "" or filename == "/" then
                     filename = configuration.index
-                    input.report("invalid filename, forcing: %s",filename)
+                    logs.simple("invalid filename, forcing: %s",filename)
                 end
                 local suffix = file.extname(filename)
                 local action = handlers[suffix] or handlers.generic
                 if action then
-                    input.report("performing action: %s",filename)
+                    logs.simple("performing action: %s",filename)
                     action(client,configuration,filename,suffix,false,hashed) -- filename and no content
                 else
                     errormessage(client,configuration,404)
@@ -270,10 +318,11 @@ filename = hashed.path
             end
         end
         client:close()
+        logs.simple("time spent with client: %0.03f seconds",os.clock()-start)
     end
 end
 
-banner = banner .. " | webserver "
+logs.extendbanner("Simple Webserver 0.10")
 
 messages.help = [[
 --start               start server
@@ -281,15 +330,26 @@ messages.help = [[
 --root                server root
 --scripts             scripts sub path
 --index               index file
+--auto                start on own path
 ]]
 
-if environment.argument("start") then
+if environment.argument("auto") then
+    local path = resolvers.find_file("mtx-server.lua") or "."
     scripts.webserver.run {
         port    = environment.argument("port"),
-        root    = environment.argument("root"),           -- "e:/websites/www.pragma-ade.com",
+        root    = environment.argument("root") or file.dirname(path) or ".",
+        scripts = environment.argument("scripts") or file.dirname(path) or ".",
+    }
+elseif environment.argument("start") then
+    scripts.webserver.run {
+        port    = environment.argument("port"),
+        root    = environment.argument("root") or ".",           -- "e:/websites/www.pragma-ade.com",
         index   = environment.argument("index"),
-        scripts = environment.argument("scripts") or "cgi",
+        scripts = environment.argument("scripts"),
     }
 else
-    input.help(banner,messages.help)
+    logs.help(messages.help)
 end
+
+
+-- mtxrun --script server --start => http://localhost:8080/mtx-server-ctx-help.lua
diff --git a/scripts/context/lua/mtx-timing.lua b/scripts/context/lua/mtx-timing.lua
new file mode 100644
index 000000000..1dcb9aa0e
--- /dev/null
+++ b/scripts/context/lua/mtx-timing.lua
@@ -0,0 +1,201 @@
+if not modules then modules = { } end modules ['mtx-timing'] = {
+    version   = 1.002,
+    comment   = "companion to mtxrun.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, gsub, concat = string.format, string.gsub, table.concat
+
+dofile(resolvers.find_file("trac-tim.lua","tex"))
+dofile(resolvers.find_file("trac-lmx.lua","tex"))
+
+local meta = [[
+    beginfig(%s) ;
+        begingroup ;
+            save p, q, b, h, w ;
+            path p, q, b ; numeric h, w ;
+            linecap := butt ;
+            h := 100 ;
+            w := 800pt ;
+            p := %s ;
+            q := %s ;
+            p := p shifted -llcorner p ;
+            q := q shifted -llcorner q ;
+            q := q xstretched w ;
+            p := p xstretched w ;
+            b := boundingbox (llcorner p -- llcorner p shifted (w,h)) ;
+            draw b withcolor white withpen pencircle scaled 4pt ;
+            draw p withcolor red   withpen pencircle scaled 4pt ;
+            draw q withcolor blue  withpen pencircle scaled 2pt ;
+        endgroup ;
+    endfig ;
+]]
+
+local html_graphic = [[
+    
%s (red) %s (blue)

+    
+        
+            
+            
+        
+    
%s
+                  min: %s

+                  max: %s

+                  pages: %s

+                  average: %s

+            

+    

+]]
+
+local html_menu = [[
+    %s
+]]
+
+local directrun = true
+
+function goodies.progress.make_svg(filename,other)
+    local metadata, menudata, c = { }, { }, 0
+    metadata[#metadata+1] = 'outputformat := "svg" ;'
+    for _, kind in pairs { "parameters", "nodes" } do
+        local mdk = { }
+        menudata[kind] = mdk
+        for n, name in pairs(goodies.progress[kind](filename)) do
+            local first = goodies.progress.path(filename,name)
+            local second = goodies.progress.path(filename,other)
+            c = c + 1
+            metadata[#metadata+1] = format(meta,c,first,second)
+            mdk[#mdk+1] = { name, c }
+        end
+    end
+    metadata[#metadata+1] = "end ."
+    metadata = concat(metadata,"\n\n")
+    if directrun then
+        dofile(resolvers.find_file("mlib-run.lua","tex"))
+        commands = commands or { }
+        commands.writestatus = logs.report
+        local result = metapost.directrun("metafun","timing data","svg",true,metadata)
+        return menudata, result
+    else
+        local mpname = file.replacesuffix(filename,"mp")
+        io.savedata(mpname,metadata)
+        os.execute(format("mpost --progname=context --mem=metafun.mem %s",mpname))
+        os.remove(mpname)
+        os.remove(file.removesuffix(filename).."-mpgraph.mpo") -- brr
+        os.remove(file.removesuffix(filename)..".log") -- brr
+        return menudata
+    end
+end
+
+function goodies.progress.makehtml(filename,other,menudata,metadata)
+    local graphics = { }
+    local result = { graphics = graphics }
+    for _, kind in pairs { "parameters", "nodes" } do
+        local md = menudata[kind]
+        local menu = { }
+        result[kind] = menu
+        for k, v in ipairs(md) do
+            local name, number = v[1], v[2]
+            local min     = goodies.progress.bot(filename,name)
+            local max     = goodies.progress.top(filename,name)
+            local pages   = goodies.progress.pages(filename)
+            local average = math.round(max/pages)
+            if directrun then
+                local data = metadata[number]
+                menu[#menu+1] = format(html_menu,name,name)
+                graphics[#graphics+1] = format(html_graphic,name,name,other,data,min,max,pages,average)
+            else
+                local mpname = file.replacesuffix(filename,number)
+                local data = io.loaddata(mpname) or ""
+            --  data = gsub(data,"[\n\r]*","")
+                data = gsub(data,"<%?xml.->","")
+                menu[#menu+1] = format(html_menu,name,name)
+                graphics[#graphics+1] = format(html_graphic,name,name,other,data,min,max,pages,average)
+                os.remove(mpname)
+            end
+        end
+    end
+    return result
+end
+
+function goodies.progress.valid_file(name)
+    return name and name ~= "" and lfs.isfile(name .. "-luatex-progress.lut")
+end
+
+function goodies.progress.make_lmx_page(name,launch,remove)
+    local filename = name .. "-luatex-progress"
+    local other    = "elapsed_time"
+    local template = 'context-timing.lmx'
+
+    lmx.variables['color-background-green']  = '#4F6F6F'
+    lmx.variables['color-background-blue']   = '#6F6F8F'
+    lmx.variables['color-background-yellow'] = '#8F8F6F'
+    lmx.variables['color-background-purple'] = '#8F6F8F'
+
+    lmx.variables['color-background-body']   = '#808080'
+    lmx.variables['color-background-main']   = '#3F3F3F'
+    lmx.variables['color-background-one']    = lmx.variables['color-background-green']
+    lmx.variables['color-background-two']    = lmx.variables['color-background-blue']
+
+    lmx.variables['title-default']           = 'ConTeXt Timing Information'
+    lmx.variables['title']                   = lmx.variables['title-default']
+
+    lmx.htmfile = function(name) return name .. "-timing.xhtml" end
+    lmx.lmxfile = function(name) return resolvers.find_file(name,'tex') end
+
+    lmx.set('title', format('ConTeXt Timing Information: %s',file.basename(name)))
+    lmx.set('color-background-one', lmx.get('color-background-green'))
+    lmx.set('color-background-two', lmx.get('color-background-blue'))
+
+    goodies.progress.convert(filename)
+
+    local menudata, metadata = goodies.progress.make_svg(filename,other)
+    local htmldata = goodies.progress.makehtml(filename,other,menudata,metadata)
+
+    lmx.set('parametersmenu', concat(htmldata.parameters, "  "))
+    lmx.set('nodesmenu',      concat(htmldata.nodes,      "  "))
+    lmx.set('graphics',       concat(htmldata.graphics,   "\n\n"))
+
+    if launch then
+        local htmfile = lmx.show(template)
+        if remove then
+            os.sleep(1) -- give time to launch
+            os.remove(htmfile)
+        end
+    else
+        lmx.make(template)
+    end
+
+    lmx.restore()
+end
+
+scripts         = scripts         or { }
+scripts.timings = scripts.timings or { }
+
+function scripts.timings.xhtml(filename)
+    if filename == "" then
+        logs.simple("provide filename")
+    elseif not goodies.progress.valid_file(filename) then
+        logs.simple("first run context again with the --timing option")
+    else
+        local basename = file.removesuffix(filename)
+        local launch   = environment.argument("launch")
+        local remove   = environment.argument("remove")
+        goodies.progress.make_lmx_page(basename,launch,remove)
+    end
+end
+
+logs.extendbanner("ConTeXt Timing Tools 0.1",true)
+
+messages.help = [[
+--xhtml               make xhtml file
+--launch              launch after conversion
+--remove              remove after launching
+]]
+
+if environment.argument("xhtml") then
+    scripts.timings.xhtml(environment.files[1] or "")
+else
+    logs.help(messages.help)
+end
diff --git a/scripts/context/lua/mtx-unzip.lua b/scripts/context/lua/mtx-unzip.lua
new file mode 100644
index 000000000..f990f4210
--- /dev/null
+++ b/scripts/context/lua/mtx-unzip.lua
@@ -0,0 +1,101 @@
+-- maybe --pattern
+
+logs.extendbanner("Simple Unzipper 0.10")
+
+messages.help = [[
+--list                list files in archive
+--junk                flatten unzipped directory structure
+--extract             extract files
+]]
+
+scripts          = scripts          or { }
+scripts.unzipper = scripts.unzipper or { }
+
+function scripts.unzipper.help()
+    logs.help(messages.help)
+end
+
+function scripts.unzipper.opened()
+    local filename = environment.files[1]
+    if filename and filename ~= "" then
+        filename = file.addsuffix(filename,'zip')
+        local zipfile = zip.open(filename)
+        if zipfile then
+            return zipfile
+        end
+    end
+    logs.report("unzip", "no zip file: " .. filename)
+    return false
+end
+
+function scripts.unzipper.list()
+    local zipfile = scripts.unzipper.opened()
+    if zipfile then
+        local n = 0
+        for k in zipfile:files() do
+            if #k.filename > n then n = #k.filename end
+        end
+        local files, paths, compressed, uncompressed = 0, 0, 0, 0
+        for k in zipfile:files() do
+            if k.filename:find("/$") then
+                paths = paths + 1
+                print(string.format("%s", k.filename:rpadd(n," ")))
+            else
+                files = files + 1
+                local cs, us = k.compressed_size, k.uncompressed_size
+                if cs > compressed then
+                    compressed = cs
+                end
+                if us > uncompressed then
+                    uncompressed = us
+                end
+                print(string.format("%s  % 9i  % 9i", k.filename:rpadd(n," "),cs,us))
+            end
+        end
+        print(string.format("\n%s  % 9i  % 9i", (files .. " files, " .. paths .. " directories"):rpadd(n," "),compressed,uncompressed))
+    end
+end
+
+function zip.loaddata(zipfile,filename)
+    local f = zipfile:open(filename)
+    if f then
+        local data = f:read("*a")
+        f:close()
+        return data
+    end
+    return nil
+end
+
+function scripts.unzipper.extract()
+    local zipfile = scripts.unzipper.opened()
+    if zipfile then
+        local junk = environment.arguments["j"] or environment.arguments["junk"]
+        for k in zipfile:files() do
+            local filename = k.filename
+            if filename:find("/$") then
+                if not junk then
+                    lfs.mkdir(filename)
+                end
+            else
+                local data = zip.loaddata(zipfile,filename)
+                if data then
+                    if junk then
+                        filename = file.basename(filename)
+                    end
+                    io.savedata(filename,data)
+                    print(filename)
+                end
+            end
+        end
+    end
+end
+
+if environment.arguments["h"] or environment.arguments["help"] then
+    scripts.unzipper.help()
+elseif environment.arguments["l"] or environment.arguments["list"] then
+    scripts.unzipper.list(zipfile)
+elseif environment.files[1] then -- implicit --extract
+    scripts.unzipper.extract(zipfile)
+else
+    scripts.unzipper.help()
+end
diff --git a/scripts/context/lua/mtx-update.lua b/scripts/context/lua/mtx-update.lua
index b56780c68..66f6898d3 100644
--- a/scripts/context/lua/mtx-update.lua
+++ b/scripts/context/lua/mtx-update.lua
@@ -1,5 +1,5 @@
 if not modules then modules = { } end modules ['mtx-update'] = {
-    version   = 1.001,
+    version   = 1.002,
     comment   = "companion to mtxrun.lua",
     author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
     copyright = "PRAGMA ADE / ConTeXt Development Team",
@@ -11,15 +11,20 @@ if not modules then modules = { } end modules ['mtx-update'] = {
 -- Together with Arthur Reutenauer she made sure that it worked well on all
 -- platforms that matter.
 
+local format, concat, gmatch = string.format, table.concat, string.gmatch
+
 scripts         = scripts         or { }
 scripts.update  = scripts.update  or { }
 
 minimals        = minimals        or { }
 minimals.config = minimals.config or { }
 
+-- this is needed under windows
+-- else rsync fails to set the right chmod flags to files
+
 os.setenv("CYGWIN","nontsec")
 
-scripts.update.allformats = {
+scripts.update.texformats = {
     "cont-en",
     "cont-nl",
     "cont-cz",
@@ -28,125 +33,105 @@ scripts.update.allformats = {
     "cont-it",
     "cont-ro",
     "cont-uk",
-    "metafun",
+    "cont-pe",
+    "cont-xp",
     "mptopdf",
     "plain"
 }
 
-scripts.update.fewformats = {
-    "cont-en",
-    "cont-nl",
+scripts.update.mpformats = {
     "metafun",
-    "mptopdf",
-    "plain"
+    "mpost",
 }
 
+-- experimental is not functional at the moment
+
 scripts.update.repositories = {
     "current",
     "experimental"
 }
 
+-- more options than just these two are available (no idea why this is here)
+
 scripts.update.versions = {
     "current",
     "latest"
 }
 
+-- list of basic folders that are needed to make a functional distribution
+
+scripts.update.base = {
+    { "base/tex/",                "texmf" },
+    { "base/metapost/",           "texmf" },
+    { "fonts/common/",            "texmf" },
+    { "fonts/other/",             "texmf" }, -- not *really* needed, but helpful
+    { "context//",       "texmf-context" },
+    { "context/img/",             "texmf-context" },
+    { "misc/setuptex/",           "." },
+    { "misc/web2c",               "texmf" },
+    { "bin/common//",   "texmf-" },
+    { "bin/context//",  "texmf-" },
+    { "bin/metapost//", "texmf-" },
+    { "bin/man/",                 "texmf-" },
+}
+
+-- binaries and font-related files
+-- for pdftex we don't need OpenType fonts, for LuaTeX/XeTeX we don't need TFM files
+
 scripts.update.engines = {
     ["luatex"] = {
-        { "base/tex/",                "texmf" },
-        { "base/metapost/",           "texmf" },
         { "fonts/new/",               "texmf" },
-        { "fonts/common/",            "texmf" },
-        { "fonts/other/",             "texmf" },
-        { "context//",       "texmf-context" },
-        { "context/img/",             "texmf-context" },
-        { "context/config/",          "texmf-context" },
-        { "misc/setuptex/",           "." },
-        { "misc/web2c",               "texmf" },
-        { "bin/common//",   "texmf-" },
-        { "bin/context//",  "texmf-" },
-        { "bin/metapost//", "texmf-" },
         { "bin/luatex//",   "texmf-" },
-        { "bin/man/",                 "texmf-" }
     },
     ["xetex"] = {
-        { "base/tex/",                "texmf" },
-        { "base/metapost/",           "texmf" },
         { "base/xetex/",              "texmf" },
         { "fonts/new/",               "texmf" },
-        { "fonts/common/",            "texmf" },
-        { "fonts/other/",             "texmf" },
-        { "context//",       "texmf-context" },
-        { "context/img/",             "texmf-context" },
-        { "context/config/",          "texmf-context" },
-        { "misc/setuptex/",           "." },
-        { "misc/web2c",               "texmf" },
-        { "bin/common//",   "texmf-" },
-        { "bin/context//",  "texmf-" },
-        { "bin/metapost//", "texmf-" },
         { "bin/xetex//",    "texmf-" },
-        { "bin/man/",                 "texmf-" }
     },
     ["pdftex"] = {
-        { "base/tex/",                "texmf" },
-        { "base/metapost/",           "texmf" },
         { "fonts/old/",               "texmf" },
-        { "fonts/common/",            "texmf" },
-        { "fonts/other/",             "texmf" },
-        { "context//",       "texmf-context" },
-        { "context/img/",             "texmf-context" },
-        { "context/config/",          "texmf-context" },
-        { "misc/setuptex/",           "." },
-        { "misc/web2c",               "texmf" },
-        { "bin/common//",   "texmf-" },
-        { "bin/context//",  "texmf-" },
-        { "bin/metapost//", "texmf-" },
         { "bin/pdftex//",   "texmf-" },
-        { "bin/man/",                 "texmf-" }
     },
     ["all"] = {
-        { "base/tex/",                "texmf" },
-        { "base/metapost/",           "texmf" },
-        { "base/xetex/",              "texmf" },
-        { "fonts/old/",               "texmf" },
         { "fonts/new/",               "texmf" },
-        { "fonts/common/",            "texmf" },
-        { "fonts/other/",             "texmf" },
-        { "context//",       "texmf-context" },
-        { "context/img/",             "texmf-context" },
-        { "context/config/",          "texmf-context" },
-        { "misc/setuptex/",           "." },
-        { "misc/web2c",               "texmf" },
-        { "bin/common//",   "texmf-" },
-        { "bin/context//",  "texmf-" },
-        { "bin/metapost//", "texmf-" },
+        { "fonts/old/",               "texmf" },
+        { "base/xetex/",              "texmf" },
         { "bin/luatex//",   "texmf-" },
         { "bin/xetex//",    "texmf-" },
         { "bin/pdftex//",   "texmf-" },
-        { "bin/man/",                 "texmf-" }
     },
 }
 
 scripts.update.platforms = {
-    ["mswin"]       = "mswin",
-    ["windows"]     = "mswin",
-    ["win32"]       = "mswin",
-    ["win"]         = "mswin",
-    ["linux"]       = "linux",
-    ["freebsd"]     = "freebsd",
-    ["linux-32"]    = "linux",
-    ["linux-64"]    = "linux-64",
-    ["linux32"]     = "linux",
-    ["linux64"]     = "linux-64",
-    ["linux-ppc"]   = "linux-ppc",
-    ["ppc"]         = "linux-ppc",
-    ["osx"]         = "osx-intel",
-    ["osx-intel"]   = "osx-intel",
-    ["osx-ppc"]     = "osx-ppc",
-    ["osx-powerpc"] = "osx-ppc",
-    ["osxintel"]    = "osx-intel",
-    ["osxppc"]      = "osx-ppc",
-    ["osxpowerpc"]  = "osx-ppc",
+    ["mswin"]         = "mswin",
+    ["windows"]       = "mswin",
+    ["win32"]         = "mswin",
+    ["win"]           = "mswin",
+    ["linux"]         = "linux",
+    ["freebsd"]       = "freebsd",
+    ["freebsd-amd64"] = "freebsd-amd64",
+    ["linux-32"]      = "linux",
+    ["linux-64"]      = "linux-64",
+    ["linux32"]       = "linux",
+    ["linux64"]       = "linux-64",
+    ["linux-ppc"]     = "linux-ppc",
+    ["ppc"]           = "linux-ppc",
+    ["osx"]           = "osx-intel",
+    ["macosx"]        = "osx-intel",
+    ["osx-intel"]     = "osx-intel",
+    ["osx-ppc"]       = "osx-ppc",
+    ["osx-powerpc"]   = "osx-ppc",
+    ["osxintel"]      = "osx-intel",
+    ["osxppc"]        = "osx-ppc",
+    ["osxpowerpc"]    = "osx-ppc",
+    ["solaris-intel"] = "solaris-intel",
+    ["solaris-sparc"] = "solaris-sparc",
+    ["solaris"]       = "solaris-sparc",
+}
+
+-- the list is filled up later (when we know what modules to download)
+
+scripts.update.modules = {
 }
 
 function scripts.update.run(str)
@@ -160,7 +145,7 @@ function scripts.update.run(str)
 end
 
 function scripts.update.fullpath(path)
-    if input.aux.rootbased_path(path) then
+    if file.is_rootbased_path(path) then
         return path
     else
         return lfs.currentdir() .. "/" .. path
@@ -168,47 +153,138 @@ function scripts.update.fullpath(path)
 end
 
 function scripts.update.synchronize()
+
     logs.report("update","start")
+
     local texroot = scripts.update.fullpath(states.get("paths.root"))
-    local engines = states.get('engines')
-    local platforms = states.get('platforms')
-    local repositories = states.get('repositories')
-    local bin = states.get("rsync.program")
-    local url = states.get("rsync.server")
-    local version = states.get("context.version")
+    local engines = states.get('engines') or { }
+    local platforms = states.get('platforms') or { }
+    local repositories = states.get('repositories') -- minimals
+    local bin = states.get("rsync.program")         -- rsync
+    local url = states.get("rsync.server")          -- contextgarden.net
+    local version = states.get("context.version")   -- current (or beta)
+    local extras = states.get("extras")             -- extra goodies (like modules)
     local force = environment.argument("force")
+
+    bin = string.gsub(bin,"\\","/")
+
     if not url:find("::$") then url = url .. "::" end
     local ok = lfs.attributes(texroot,"mode") == "directory"
     if not ok and force then
         dir.mkdirs(texroot)
         ok = lfs.attributes(texroot,"mode") == "directory"
     end
+
+    if force then
+        dir.mkdirs(format("%s/%s", texroot, "texmf-cache"))
+        dir.mkdirs(format("%s/%s", texroot, "texmf-local"))
+    end
+
     if ok or not force then
-        if force then
-            dir.mkdirs(string.format("%s/%s", texroot, "texmf-cache"))
+
+        local fetched, individual, osplatform = { }, { }, os.currentplatform()
+
+        -- takes a collection as argument and returns a list of folders
+
+        local function collection_to_list_of_folders(collection, platform)
+            local archives = {}
+            for _, c in ipairs(collection) do
+                local archive = c[1]
+                archive = archive:gsub("", platform)
+                archive = archive:gsub("", version)
+                archives[#archives+1] = archive
+            end
+            return archives
         end
-        local fetched, individual = { }, { }
-        for engine, _ in pairs(engines) do
-            local collections = scripts.update.engines[engine]
-            if collections then
-                for _, collection in ipairs(collections) do
-                    for platform, _ in pairs(platforms) do
-                        platform = scripts.update.platforms[platform]
-                        if platform then
-                            local archive = collection[1]:gsub("", platform)
-                            local destination = string.format("%s/%s", texroot, collection[2]:gsub("", platform))
-                            destination = destination:gsub("\\","/")
-                            archive = archive:gsub("",version)
---~                             if platform == "windows" or platform == "mswin" then
-                            if os.currentplatform() == "windows" or os.currentplatform() == "mswin" then
-                                destination = destination:gsub("([a-zA-Z]):/", "/cygdrive/%1/")
-                            end
-                            individual[#individual+1] = { archive, destination }
+
+        -- takes a list of folders as argument and returns a string for rsync
+        -- sample input:
+        --     {'bin/common', 'bin/context'}
+        -- output:
+        --     'minimals/current/bin/common minimals/current/bin/context'
+
+        local function list_of_folders_to_rsync_string(list_of_folders)
+            local repository  = 'current'
+            local prefix = format("%s/%s/", states.get('rsync.module'), repository) -- minimals/current/
+
+            return prefix .. concat(list_of_folders, format(" %s", prefix))
+        end
+
+        -- example of usage: print(list_of_folders_to_rsync_string(collection_to_list_of_folders(scripts.update.base, os.currentplatform)))
+
+        -- rename function and add some more functionality:
+        --   * recursive/non-recursive (default: non-recursive)
+        --   * filter folders or regular files only (default: no filter)
+        --   * grep for size of included files (with --stats switch)
+
+        local function get_list_of_files_from_rsync(list_of_folders)
+            -- temporary file to store the output of rsync (could be a more random name; watch for overwrites)
+            local temp_file = "rsync.tmp.txt"
+            -- a set of folders
+            local folders = {}
+            local command = format("%s %s'%s' > %s", bin, url, list_of_folders_to_rsync_string(list_of_folders), temp_file)
+            os.execute(command)
+            -- read output of rsync
+            local data = io.loaddata(temp_file) or ""
+            -- for every line extract the filename
+            for chmod, s in data:gmatch("([d%-][rwx%-]+).-(%S+)[\n\r]") do
+                -- skip "current" folder
+                if s ~= '.' and chmod:len() == 10 then
+                    folders[#folders+1] = s
+                end
+            end
+            -- delete the file to which we have put output of rsync
+            os.remove(temp_file)
+            return folders
+        end
+
+        -- rsync://contextgarden.net/minimals/current/modules/
+
+        if extras and type(extras) == "table" then
+            -- fetch the list of available modules from rsync server
+            local available_modules = get_list_of_files_from_rsync({"modules/"})
+            -- hash of requested modules
+            -- local h = table.tohash(extras:split(","))
+            for _, s in ipairs(available_modules) do
+            --  if extras == "all" or h[s] then
+                if extras.all or extras[s] then
+                    scripts.update.modules[#scripts.update.modules+1] = { format("modules/%s/",s), "texmf-context" }
+                end
+            end
+            -- TODO: check if every module from the list has been added and issue warning otherwise
+            -- one idea to do it: remove every value from h once added and then check if anything is left in h
+        end
+
+        local function add_collection(collection,platform)
+            if collection and platform then
+                platform = scripts.update.platforms[platform]
+                if platform then
+                    for _, c in ipairs(collection) do
+                        local archive = c[1]:gsub("", platform)
+                        local destination = format("%s/%s", texroot, c[2]:gsub("", platform))
+                        destination = destination:gsub("\\","/")
+                        archive = archive:gsub("",version)
+                        if osplatform == "windows" or osplatform == "mswin" then
+                            destination = destination:gsub("([a-zA-Z]):/", "/cygdrive/%1/")
                         end
+                        individual[#individual+1] = { archive, destination }
                     end
                 end
             end
         end
+
+        for platform, _ in pairs(platforms) do
+            add_collection(scripts.update.base,platform)
+        end
+        for platform, _ in pairs(platforms) do
+            add_collection(scripts.update.modules,platform)
+        end
+        for engine, _ in pairs(engines) do
+            for platform, _ in pairs(platforms) do
+                add_collection(scripts.update.engines[engine],platform)
+            end
+        end
+
         local combined = { }
         for _, repository in ipairs(scripts.update.repositories) do
             if repositories[repository] then
@@ -219,11 +295,11 @@ function scripts.update.synchronize()
                         cd = { }
                         combined[destination] = cd
                     end
-                    cd[#cd+1] = string.format("%s/%s/%s",states.get('rsync.module'),repository,archive)
+                    cd[#cd+1] = format("%s/%s/%s",states.get('rsync.module'),repository,archive)
                 end
             end
         end
-        if input.verbose then
+        if logs.verbose then
             for k, v in pairs(combined) do
                 logs.report("update", k)
                 for k,v in ipairs(v) do
@@ -232,24 +308,53 @@ function scripts.update.synchronize()
             end
         end
         for destination, archive in pairs(combined) do
-            local archives, command = table.concat(archive," "), ""
-            local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete")
-            if true then -- environment.argument("keep") or destination:find("%.$") then
-                command = string.format("%s %s    %s'%s' '%s'", bin, normalflags,              url, archives, destination)
-            else
-                command = string.format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination)
+            local archives, command = concat(archive," "), ""
+        --  local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete")
+        --    if environment.argument("keep") or destination:find("%.$") then
+        --       command = format("%s %s    %s'%s' '%s'", bin, normalflags,              url, archives, destination)
+        --    else
+        --        command = format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination)
+        --    end
+            local normalflags, deleteflags = states.get("rsync.flags.normal"), ""
+            if (destination:find("texmf$") or destination:find("texmf%-context$")) and (not environment.argument("keep")) then
+                deleteflags = states.get("rsync.flags.delete")
             end
-            logs.report("mtx update", string.format("running command: %s",command))
+            command = format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination)
+            logs.report("mtx update", format("running command: %s",command))
             if not fetched[command] then
                 scripts.update.run(command)
                 fetched[command] = command
             end
         end
+
+        local function update_script(script, platform)
+            local bin = bin:gsub("\\","/")
+            local texroot = texroot:gsub("\\","/")
+            platform = scripts.update.platforms[platform]
+            if platform then
+                local command
+                if platform == 'mswin' then
+                    bin = bin:gsub("([a-zA-Z]):/", "/cygdrive/%1/")
+                    texroot = texroot:gsub("([a-zA-Z]):/", "/cygdrive/%1/")
+                    command = string.format("%s -t %s/texmf-context/scripts/context/lua/%s.lua %s/texmf-mswin/bin/", bin, texroot, script, texroot)
+                else
+                    command = string.format("%s -tgo --chmod=a+x %s/texmf-context/scripts/context/lua/%s.lua %s/texmf-%s/bin/%s", bin, texroot, script, texroot, platform, script)
+                end
+                logs.report("mtx update", format("updating %s for %s: %s", script, platform, command))
+                scripts.update.run(command)
+            end
+        end
+
+        for platform, _ in pairs(platforms) do
+            update_script('luatools',platform)
+            update_script('mtxrun',platform)
+        end
+
     else
-        logs.report("mtx update", string.format("no valid texroot: %s",texroot))
+        logs.report("mtx update", format("no valid texroot: %s",texroot))
     end
     if not force then
-        logs.report("update", "use --force to really update")
+        logs.report("update", "use --force to really update files")
     end
     logs.report("update","done")
 end
@@ -262,32 +367,59 @@ function table.fromhash(t)
     return h
 end
 
-
+-- make the ConTeXt formats
 function scripts.update.make()
+
     logs.report("make","start")
+
     local force = environment.argument("force")
     local texroot = scripts.update.fullpath(states.get("paths.root"))
-    local engines = states.get('engines')
+    local engines= states.get('engines')
     local platforms = states.get('platforms')
     local formats = states.get('formats')
-    input.load_tree(texroot)
+
+    resolvers.load_tree(texroot)
+    -- update filename database for pdftex/xetex
     scripts.update.run("mktexlsr")
+    -- update filename database for luatex
     scripts.update.run("luatools --generate")
-    local formatlist = table.concat(table.fromhash(formats), " ")
+    local askedformats = formats
+    local texformats = table.tohash(scripts.update.texformats)
+    local mpformats = table.tohash(scripts.update.mpformats)
+    for k,v in pairs(texformats) do
+        if not askedformats[k] then
+            texformats[k] = nil
+        end
+    end
+    for k,v in pairs(mpformats) do
+        if not askedformats[k] then
+            mpformats[k] = nil
+        end
+    end
+    local formatlist = concat(table.fromhash(texformats), " ")
     if formatlist ~= "" then
         for engine in pairs(engines) do
-            -- todo: just handle make here or in mtxrun --script context --make
---~             os.execute("set")
-            scripts.update.run(string.format("texexec --make --all --fast --%s %s",engine,formatlist))
+            if engine == "luatex" then
+                scripts.update.run(format("context --make")) -- maybe also formatlist
+            else
+                -- todo: just handle make here or in mtxrun --script context --make
+                scripts.update.run(format("texexec --make --all --fast --%s %s",engine,formatlist))
+            end
         end
     end
+    local formatlist = concat(table.fromhash(mpformats), " ")
+    if formatlist ~= "" then
+        scripts.update.run(format("texexec --make --all --fast %s",formatlist))
+    end
     if not force then
-        logs.report("make", "use --force to really make")
+        logs.report("make", "use --force to really make formats")
     end
+    scripts.update.run("mktexlsr")
+    scripts.update.run("luatools --generate")
     logs.report("make","done")
 end
 
-banner = banner .. " | download tools "
+logs.extendbanner("Download Tools 0.20",true)
 
 messages.help = [[
 --platform=string     platform (windows, linux, linux-64, osx-intel, osx-ppc, linux-ppc)
@@ -296,8 +428,9 @@ messages.help = [[
 --repository=string   specify version (current, experimental)
 --context=string      specify version (current, latest, yyyy.mm.dd)
 --rsync=string        rsync binary (rsync)
---texroot             installation directory (not guessed for the moment)
---engine              tex engine (luatex, pdftex, xetex)
+--texroot=string      installation directory (not guessed for the moment)
+--engine=string       tex engine (luatex, pdftex, xetex)
+--extras=string       extra modules (can be list or 'all')
 --force               instead of a dryrun, do the real thing
 --update              update minimal tree
 --make                also make formats and generate file databases
@@ -305,8 +438,6 @@ messages.help = [[
 --state               update tree using saved state
 ]]
 
-input.verbose = true
-
 scripts.savestate = true
 
 if scripts.savestate then
@@ -315,7 +446,7 @@ if scripts.savestate then
 
     -- tag, value, default, persistent
 
-    input.starttiming(states)
+    statistics.starttiming(states)
 
     states.set("info.version",0.1) -- ok
     states.set("info.count",(states.get("info.count") or 0) + 1,1,false) -- ok
@@ -333,11 +464,11 @@ if scripts.savestate then
     states.set("context.version", environment.argument("context"), "current", true) -- ok
 
     local valid = table.tohash(scripts.update.repositories)
-    for r in string.gmatch(environment.argument("repository") or "current","([^, ]+)") do
+    for r in gmatch(environment.argument("repository") or "current","([^, ]+)") do
         if valid[r] then states.set("repositories." .. r, true) end
     end
     local valid = scripts.update.engines
-    for r in string.gmatch(environment.argument("engine") or "all","([^, ]+)") do
+    for r in gmatch(environment.argument("engine") or "all","([^, ]+)") do
         if r == "all" then
             for k, v in pairs(valid) do
                 if k ~= "all" then
@@ -349,12 +480,16 @@ if scripts.savestate then
         end
     end
     local valid = scripts.update.platforms
-    for r in string.gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do
+    for r in gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do
         if valid[r] then states.set("platforms." .. r, true) end
     end
 
-    local valid = table.tohash(scripts.update.allformats)
-    for r in string.gmatch(environment.argument("formats") or "","([^, ]+)") do
+    local valid = table.tohash(scripts.update.texformats)
+    for r in gmatch(environment.argument("formats") or "","([^, ]+)") do
+        if valid[r] then states.set("formats." .. r, true) end
+    end
+    local valid = table.tohash(scripts.update.mpformats)
+    for r in gmatch(environment.argument("formats") or "","([^, ]+)") do
         if valid[r] then states.set("formats." .. r, true) end
     end
 
@@ -362,7 +497,9 @@ if scripts.savestate then
     states.set("formats.cont-nl", true)
     states.set("formats.metafun", true)
 
-    -- modules
+    for r in gmatch(environment.argument("extras") or "","([^, ]+)") do
+        states.set("extras." .. r, true)
+    end
 
     logs.report("state","loaded")
 
@@ -382,12 +519,12 @@ if environment.argument("update") then
 elseif environment.argument("make") then
     scripts.update.make()
 else
-    input.help(banner,messages.help)
+    logs.help(messages.help)
 end
 
 if scripts.savestate then
-    input.stoptiming(states)
-    states.set("info.runtime",tonumber(input.elapsedtime(states)))
+    statistics.stoptiming(states)
+    states.set("info.runtime",tonumber(statistics.elapsedtime(states)))
     if environment.argument("force") then
         states.save()
         logs.report("state","saved")
diff --git a/scripts/context/lua/mtx-watch.lua b/scripts/context/lua/mtx-watch.lua
index b7b6fb77b..2e4dcf6ef 100644
--- a/scripts/context/lua/mtx-watch.lua
+++ b/scripts/context/lua/mtx-watch.lua
@@ -58,6 +58,20 @@ do
                     end
                 end
             end
+            local function toset(t)
+                if type(t) == "table" then
+                    return table.concat(t,",")
+                else
+                    return t
+                end
+            end
+            local function noset(t)
+                if type(t) == "table" then
+                    return t[1]
+                else
+                    return t
+                end
+            end
             local function process()
                 local done = false
                 for _, path in ipairs(environment.files) do
@@ -77,8 +91,8 @@ do
                                 local command = joblog.command
                                 if command then
                                     local replacements = {
-                                        inputpath  = (joblog.paths and joblog.paths.input ) or ".",
-                                        outputpath = (joblog.paths and joblog.paths.output) or ".",
+                                        inputpath  = toset((joblog.paths and joblog.paths.input ) or "."),
+                                        outputpath = noset((joblog.paths and joblog.paths.output) or "."),
                                         filename   = joblog.filename or "",
                                     }
                                     command = command:gsub("%%(.-)%%", replacements)
@@ -225,7 +239,7 @@ function scripts.watch.show_logs(path) -- removes duplicates
     end
 end
 
-banner = banner .. " | watchdog"
+logs.extendbanner("Watchdog 1.00",true)
 
 messages.help = [[
 --logpath             optional path for log files
@@ -236,8 +250,6 @@ messages.help = [[
 --showlog             show log data
 ]]
 
-input.verbose = true
-
 if environment.argument("watch") then
     scripts.watch.watch()
 elseif environment.argument("collect") then
@@ -245,5 +257,5 @@ elseif environment.argument("collect") then
 elseif environment.argument("showlog") then
     scripts.watch.show_logs()
 else
-    input.help(banner,messages.help)
+    logs.help(messages.help)
 end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 6c68ec51a..0af429bf1 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -8,6 +8,7 @@ if not modules then modules = { } end modules ['mtxrun'] = {
     license   = "see context related readme files"
 }
 
+
 -- one can make a stub:
 --
 -- #!/bin/sh
@@ -26,7 +27,7 @@ if not modules then modules = { } end modules ['mtxrun'] = {
 -- one. Interesting is that using a scripting language instead of c does
 -- not have a speed penalty. Actually the lua variant is more efficient,
 -- especially when multiple calls to kpsewhich are involved. The lua
--- library also gives way more ocntrol.
+-- library also gives way more control.
 
 -- to be done / considered
 --
@@ -37,144 +38,42 @@ if not modules then modules = { } end modules ['mtxrun'] = {
 -- remember for subruns: _CTX_K_S_#{original}_
 -- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb]
 
-banner = "version 1.1.2 - 2007+ - PRAGMA ADE / CONTEXT" -- not local
 texlua = true
 
 -- begin library merge
--- filename : l-string.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
 
-if not versions then versions = { } end versions['l-string'] = 1.001
 
---~ function string.split(str, pat) -- taken from the lua wiki
---~     local t = {n = 0} -- so this table has a length field, traverse with ipairs then!
---~     local fpat = "(.-)"..pat
---~     local last_end = 1
---~     local s, e, cap = string.find(str, fpat, 1)
---~     while s ~= nil do
---~         if s~=1 or cap~="" then
---~             table.insert(t,cap)
---~         end
---~         last_end = e+1
---~         s, e, cap = string.find(str, fpat, last_end)
---~     end
---~     if last_end<=string.len(str) then
---~         table.insert(t,(string.sub(str,last_end)))
---~     end
---~     return t
---~ end
 
---~ function string:split(pat) -- taken from the lua wiki but adapted
---~     local t = { }          -- self and colon usage (faster)
---~     local fpat = "(.-)"..pat
---~     local last_end = 1
---~     local s, e, cap = self:find(fpat, 1)
---~     while s ~= nil do
---~         if s~=1 or cap~="" then
---~             t[#t+1] = cap
---~         end
---~         last_end = e+1
---~         s, e, cap = self:find(fpat, last_end)
---~     end
---~     if last_end <= #self then
---~         t[#t+1] = self:sub(last_end)
---~     end
---~     return t
---~ end
+do -- create closure to overcome 200 locals limit
 
---~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed
---~
---~ function string:splitter(pat)
---~    local st, g = 1, self:gmatch("()"..pat.."()")
---~    local function splitter(self)
---~      if st then
---~        local s, f = g()
---~        local rv = self:sub(st, (s or 0)-1)
---~        st = f
---~        return rv
---~      end
---~    end
---~    return splitter, self
---~ end
+if not modules then modules = { } end modules ['l-string'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-function string:splitter(pat)
-    -- by Rici Lake (posted on lua list) -- only names changed
-    -- p 79 ref man: () returns position of match
-    local st, g = 1, self:gmatch("()("..pat..")")
-    local function strgetter(self, segs, seps, sep, cap1, ...)
-        st = sep and seps + #sep
-        return self:sub(segs, (seps or 0) - 1), cap1 or sep, ...
-    end
-    local function strsplitter(self)
-        if st then return strgetter(self, st, g()) end
-    end
-    return strsplitter, self
-end
+local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep
 
-function string:split(separator)
-    local t = {}
-    for k in self:splitter(separator) do t[#t+1] = k end
-    return t
-end
+if not string.split then
 
--- faster than a string:split:
+    -- this will be overloaded by a faster lpeg variant
 
-function string:splitchr(chr)
-    if #self > 0 then
-        local t = { }
-        for s in (self..chr):gmatch("(.-)"..chr) do
-            t[#t+1] = s
+    function string:split(pattern)
+        if #self > 0 then
+            local t = { }
+            for s in gmatch(self..pattern,"(.-)"..pattern) do
+                t[#t+1] = s
+            end
+            return t
+        else
+            return { }
         end
-        return t
-    else
-        return { }
     end
-end
 
-function string.piecewise(str, pat, fnc) -- variant of split
-    for k in string.splitter(str,pat) do fnc(k) end
 end
 
---~ function string.piecewise(str, pat, fnc) -- variant of split
---~     for k in str:splitter(pat) do fnc(k) end
---~ end
-
---~ do if lpeg then
-
---~     -- this alternative is 30% faster esp when we cache them
---~     -- problem: no expressions
-
---~     splitters = { }
-
---~     function string:split(separator)
---~         if #self > 0 then
---~             local split = splitters[separator]
---~             if not split then
---~                 -- based on code by Roberto
---~                 local p = lpeg.P(separator)
---~                 local c = lpeg.C((1-p)^0)
---~                 split = lpeg.Ct(c*(p*c)^0)
---~                 splitters[separator] = split
---~             end
---~             return split:match(self)
---~         else
---~             return { }
---~         end
---~     end
-
---~     string.splitchr = string.split
-
---~     function string:piecewise(separator,fnc)
---~         for _,v in pairs(self:split(separator)) do
---~             fnc(v)
---~         end
---~     end
-
---~ end end
-
 local chr_to_esc = {
     ["%"] = "%%",
     ["."] = "%.",
@@ -188,20 +87,20 @@ local chr_to_esc = {
 string.chr_to_esc = chr_to_esc
 
 function string:esc() -- variant 2
-    return (self:gsub("(.)",chr_to_esc))
+    return (gsub(self,"(.)",chr_to_esc))
 end
 
 function string:unquote()
-    return (self:gsub("^([\"\'])(.*)%1$","%2"))
+    return (gsub(self,"^([\"\'])(.*)%1$","%2"))
 end
 
-function string:quote()
+function string:quote() -- we could use format("%q")
     return '"' .. self:unquote() .. '"'
 end
 
 function string:count(pattern) -- variant 3
     local n = 0
-    for _ in self:gmatch(pattern) do
+    for _ in gmatch(self,pattern) do
         n = n + 1
     end
     return n
@@ -210,29 +109,25 @@ end
 function string:limit(n,sentinel)
     if #self > n then
         sentinel = sentinel or " ..."
-        return self:sub(1,(n-#sentinel)) .. sentinel
+        return sub(self,1,(n-#sentinel)) .. sentinel
     else
         return self
     end
 end
 
 function string:strip()
-    return (self:gsub("^%s*(.-)%s*$", "%1"))
+    return (gsub(self,"^%s*(.-)%s*$", "%1"))
 end
 
---~ function string.strip(str) -- slightly different
---~     return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," "))
---~ end
-
 function string:is_empty()
-    return not self:find("%S")
+    return not find(find,"%S")
 end
 
 function string:enhance(pattern,action)
     local ok, n = true, 0
     while ok do
         ok = false
-        self = self:gsub(pattern, function(...)
+        self = gsub(self,pattern, function(...)
             ok, n = true, n + 1
             return action(...)
         end)
@@ -240,59 +135,19 @@ function string:enhance(pattern,action)
     return self, n
 end
 
---~ function string:enhance(pattern,action)
---~     local ok, n = 0, 0
---~     repeat
---~         self, ok = self:gsub(pattern, function(...)
---~             n = n + 1
---~             return action(...)
---~         end)
---~     until ok == 0
---~     return self, n
---~ end
-
---~     function string:to_hex()
---~         if self then
---~             return (self:gsub("(.)",function(c)
---~                 return string.format("%02X",c:byte())
---~             end))
---~         else
---~             return ""
---~         end
---~     end
-
---~     function string:from_hex()
---~         if self then
---~             return (self:gsub("(..)",function(c)
---~                 return string.char(tonumber(c,16))
---~             end))
---~         else
---~             return ""
---~         end
---~     end
-
-string.chr_to_hex = { }
-string.hex_to_chr = { }
+local chr_to_hex, hex_to_chr = { }, { }
 
 for i=0,255 do
-    local c, h = string.char(i), string.format("%02X",i)
-    string.chr_to_hex[c], string.hex_to_chr[h] = h, c
+    local c, h = char(i), format("%02X",i)
+    chr_to_hex[c], hex_to_chr[h] = h, c
 end
 
---~     function string:to_hex()
---~         if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end
---~     end
-
---~     function string:from_hex()
---~         if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end
---~     end
-
 function string:to_hex()
-    return ((self or ""):gsub("(.)",string.chr_to_hex))
+    return (gsub(self or "","(.)",chr_to_hex))
 end
 
 function string:from_hex()
-    return ((self or ""):gsub("(..)",string.hex_to_chr))
+    return (gsub(self or "","(..)",hex_to_chr))
 end
 
 if not string.characters then
@@ -306,7 +161,7 @@ if not string.characters then
     end
     local function nextbyte(str, index)
         index = index + 1
-        return (index <= #str) and index or nil, string.byte(str:sub(index,index))
+        return (index <= #str) and index or nil, byte(str:sub(index,index))
     end
     function string:bytes()
         return nextbyte, self, 0
@@ -314,9 +169,7 @@ if not string.characters then
 
 end
 
---~ function string:padd(n,chr)
---~     return self .. self.rep(chr or " ",n-#self)
---~ end
+-- we can use format for this (neg n)
 
 function string:rpadd(n,chr)
     local m = n-#self
@@ -338,8 +191,8 @@ end
 
 string.padd = string.rpadd
 
-function is_number(str)
-    return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1
+function is_number(str) -- tonumber
+    return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1
 end
 
 --~ print(is_number("1"))
@@ -351,9 +204,9 @@ end
 --~ print(is_number("+.1"))
 
 function string:split_settings() -- no {} handling, see l-aux for lpeg variant
-    if self:find("=") then
+    if find(self,"=") then
         local t = { }
-        for k,v in self:gmatch("(%a+)=([^%,]*)") do
+        for k,v in gmatch(self,"(%a+)=([^%,]*)") do
             t[k] = v
         end
         return t
@@ -375,24 +228,67 @@ local patterns_escapes = {
 }
 
 function string:pattesc()
-    return (self:gsub(".",patterns_escapes))
+    return (gsub(self,".",patterns_escapes))
 end
 
 function string:tohash()
     local t = { }
-    for s in self:gmatch("([^, ]+)") do -- lpeg
+    for s in gmatch(self,"([^, ]+)") do -- lpeg
         t[s] = true
     end
     return t
 end
 
+local pattern = lpeg.Ct(lpeg.C(1)^0)
 
--- filename : l-lpeg.lua
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+function string:totable()
+    return pattern:match(self)
+end
+
+--~ for _, str in ipairs {
+--~     "1234567123456712345671234567",
+--~     "a\tb\tc",
+--~     "aa\tbb\tcc",
+--~     "aaa\tbbb\tccc",
+--~     "aaaa\tbbbb\tcccc",
+--~     "aaaaa\tbbbbb\tccccc",
+--~     "aaaaaa\tbbbbbb\tcccccc",
+--~ } do print(string.tabtospace(str)) end
+
+function string.tabtospace(str,tab)
+    -- we don't handle embedded newlines
+    while true do
+        local s = find(str,"\t")
+        if s then
+            if not tab then tab = 7 end -- only when found
+            local d = tab-(s-1)%tab
+            if d > 0 then
+                str = gsub(str,"\t",rep(" ",d),1)
+            else
+                str = gsub(str,"\t","",1)
+            end
+        else
+            break
+        end
+    end
+    return str
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-lpeg'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-if not versions then versions = { } end versions['l-lpeg'] = 1.001
+local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
 
@@ -416,36 +312,40 @@ if not versions then versions = { } end versions['l-lpeg'] = 1.001
 local hash = { }
 
 function lpeg.anywhere(pattern) --slightly adapted from website
-    return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) }
+    return P { P(pattern) + 1 * lpeg.V(1) }
 end
 
 function lpeg.startswith(pattern) --slightly adapted
-    return lpeg.P(pattern)
+    return P(pattern)
 end
 
---~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2
-
 function lpeg.splitter(pattern, action)
-    return (((1-lpeg.P(pattern))^1)/action+1)^0
+    return (((1-P(pattern))^1)/action+1)^0
 end
 
-local crlf     = lpeg.P("\r\n")
-local cr       = lpeg.P("\r")
-local lf       = lpeg.P("\n")
-local space    = lpeg.S(" \t\f\v")
+-- variant:
+
+--~ local parser = lpeg.Ct(lpeg.splitat(newline))
+
+local crlf     = P("\r\n")
+local cr       = P("\r")
+local lf       = P("\n")
+local space    = S(" \t\f\v")  -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto)
 local newline  = crlf + cr + lf
 local spacing  = space^0 * newline
 
-local empty    = spacing * lpeg.Cc("")
-local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1
+local empty    = spacing * Cc("")
+local nonempty = Cs((1-spacing)^1) * spacing^-1
 local content  = (empty + nonempty)^1
 
-local capture = lpeg.Ct(content^0)
+local capture = Ct(content^0)
 
 function string:splitlines()
     return capture:match(self)
 end
 
+lpeg.linebyline = content -- better make a sublibrary
+
 --~ local p = lpeg.splitat("->",false)  print(p:match("oeps->what->more"))  -- oeps what more
 --~ local p = lpeg.splitat("->",true)   print(p:match("oeps->what->more"))  -- oeps what->more
 --~ local p = lpeg.splitat("->",false)  print(p:match("oeps"))              -- oeps
@@ -453,16 +353,16 @@ end
 
 local splitters_s, splitters_m = { }, { }
 
-function lpeg.splitat(separator,single)
+local function splitat(separator,single)
     local splitter = (single and splitters_s[separator]) or splitters_m[separator]
     if not splitter then
-        separator = lpeg.P(separator)
+        separator = P(separator)
         if single then
-            local other, any = lpeg.C((1 - separator)^0), lpeg.P(1)
-            splitter = other * (separator * lpeg.C(any^0) + "")
+            local other, any = C((1 - separator)^0), P(1)
+            splitter = other * (separator * C(any^0) + "")
             splitters_s[separator] = splitter
         else
-            local other = lpeg.C((1 - separator)^0)
+            local other = C((1 - separator)^0)
             splitter = other * (separator * other)^0
             splitters_m[separator] = splitter
         end
@@ -470,26 +370,43 @@ function lpeg.splitat(separator,single)
     return splitter
 end
 
+lpeg.splitat = splitat
+
+local cache = { }
+
+function string:split(separator)
+    local c = cache[separator]
+    if not c then
+        c = Ct(splitat(separator))
+        cache[separator] = c
+    end
+    return c:match(self)
+end
 
--- filename : l-table.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
 
-if not versions then versions = { } end versions['l-table'] = 1.001
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-table'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
 table.join = table.concat
 
 local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove
-local format = string.format
+local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump
 local getmetatable, setmetatable = getmetatable, setmetatable
-local pairs, ipairs, type, next, tostring = pairs, ipairs, type, next, tostring
+local type, next, tostring, ipairs = type, next, tostring, ipairs
 
 function table.strip(tab)
     local lst = { }
     for i=1,#tab do
-        local s = tab[i]:gsub("^%s*(.-)%s*$","%1")
+        local s = gsub(tab[i],"^%s*(.-)%s*$","%1")
         if s == "" then
             -- skip this one
         else
@@ -501,7 +418,7 @@ end
 
 local function sortedkeys(tab)
     local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed
-    for key,_ in pairs(tab) do
+    for key,_ in next, tab do
         srt[#srt+1] = key
         if kind == 3 then
             -- no further check
@@ -528,7 +445,7 @@ end
 
 local function sortedhashkeys(tab) -- fast one
     local srt = { }
-    for key,_ in pairs(tab) do
+    for key,_ in next, tab do
         srt[#srt+1] = key
     end
     sort(srt)
@@ -538,14 +455,25 @@ end
 table.sortedkeys     = sortedkeys
 table.sortedhashkeys = sortedhashkeys
 
+function table.sortedpairs(t)
+    local s = sortedhashkeys(t) -- maybe just sortedkeys
+    local n = 0
+    local function kv(s)
+        n = n + 1
+        local k = s[n]
+        return k, t[k]
+    end
+    return kv, s
+end
+
 function table.append(t, list)
-    for _,v in pairs(list) do
+    for _,v in next, list do
         insert(t,v)
     end
 end
 
 function table.prepend(t, list)
-    for k,v in pairs(list) do
+    for k,v in next, list do
         insert(t,k,v)
     end
 end
@@ -554,7 +482,7 @@ function table.merge(t, ...) -- first one is target
     t = t or {}
     local lst = {...}
     for i=1,#lst do
-        for k, v in pairs(lst[i]) do
+        for k, v in next, lst[i] do
             t[k] = v
         end
     end
@@ -564,7 +492,7 @@ end
 function table.merged(...)
     local tmp, lst = { }, {...}
     for i=1,#lst do
-        for k, v in pairs(lst[i]) do
+        for k, v in next, lst[i] do
             tmp[k] = v
         end
     end
@@ -596,13 +524,14 @@ end
 local function fastcopy(old) -- fast one
     if old then
         local new = { }
-        for k,v in pairs(old) do
+        for k,v in next, old do
             if type(v) == "table" then
                 new[k] = fastcopy(v) -- was just table.copy
             else
                 new[k] = v
             end
         end
+        -- optional second arg
         local mt = getmetatable(old)
         if mt then
             setmetatable(new,mt)
@@ -619,7 +548,7 @@ local function copy(t, tables) -- taken from lua wiki, slightly adapted
     if not tables[t] then
         tables[t] = tcopy
     end
-    for i,v in pairs(t) do -- brrr, what happens with sparse indexed
+    for i,v in next, t do -- brrr, what happens with sparse indexed
         if type(i) == "table" then
             if tables[i] then
                 i = tables[i]
@@ -652,7 +581,7 @@ function table.sub(t,i,j)
 end
 
 function table.replace(a,b)
-    for k,v in pairs(b) do
+    for k,v in next, b do
         a[k] = v
     end
 end
@@ -674,16 +603,18 @@ end
 
 function table.tohash(t,value)
     local h = { }
-    if value == nil then value = true end
-    for _, v in pairs(t) do -- no ipairs here
-        h[v] = value
+    if t then
+        if value == nil then value = true end
+        for _, v in next, t do -- no ipairs here
+            h[v] = value
+        end
     end
     return h
 end
 
 function table.fromhash(t)
     local h = { }
-    for k, v in pairs(t) do -- no ipairs here
+    for k, v in next, t do -- no ipairs here
         if v then h[#h+1] = k end
     end
     return h
@@ -707,24 +638,10 @@ local reserved = table.tohash { -- intercept a language flaw, no reserved words
     'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
 }
 
-local function key(k)
-    if type(k) == "number" then -- or k:find("^%d+$") then
-        if hexify then
-            return ("[0x%04X]"):format(k)
-        else
-            return "["..k.."]"
-        end
-    elseif noquotes and not reserved[k] and k:find("^%a[%a%d%_]*$") then
-        return k
-    else
-        return '["'..k..'"]'
-    end
-end
-
 local function simple_table(t)
     if #t > 0 then
         local n = 0
-        for _,v in pairs(t) do
+        for _,v in next, t do
             n = n + 1
         end
         if n == #t then
@@ -734,14 +651,14 @@ local function simple_table(t)
                 local tv = type(v)
                 if tv == "number" then
                     if hexify then
-                        tt[#tt+1] = ("0x%04X"):format(v)
+                        tt[#tt+1] = format("0x%04X",v)
                     else
-                        tt[#tt+1] = tostring(v)
+                        tt[#tt+1] = tostring(v) -- tostring not needed
                     end
                 elseif tv == "boolean" then
                     tt[#tt+1] = tostring(v)
                 elseif tv == "string" then
-                    tt[#tt+1] = ("%q"):format(v)
+                    tt[#tt+1] = format("%q",v)
                 else
                     tt = nil
                     break
@@ -753,51 +670,72 @@ local function simple_table(t)
     return nil
 end
 
+-- Because this is a core function of mkiv I moved some function calls
+-- inline.
+--
+-- twice as fast in a test:
+--
+-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) )
+
 local function do_serialize(root,name,depth,level,indexed)
     if level > 0 then
         depth = depth .. " "
         if indexed then
-            handle(("%s{"):format(depth))
+            handle(format("%s{",depth))
         elseif name then
-            handle(("%s%s={"):format(depth,key(name)))
+        --~ handle(format("%s%s={",depth,key(name)))
+            if type(name) == "number" then -- or find(k,"^%d+$") then
+                if hexify then
+                    handle(format("%s[0x%04X]={",depth,name))
+                else
+                    handle(format("%s[%s]={",depth,name))
+                end
+            elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then
+                handle(format("%s%s={",depth,name))
+            else
+                handle(format("%s[%q]={",depth,name))
+            end
         else
-            handle(("%s{"):format(depth))
+            handle(format("%s{",depth))
         end
     end
     if root and next(root) then
         local first, last = nil, 0 -- #root cannot be trusted here
         if compact then
-          for k,v in ipairs(root) do -- NOT: for k=1,#root do (we need to quit at nil)
+            -- NOT: for k=1,#root do (we need to quit at nil)
+            for k,v in ipairs(root) do -- can we use next?
                 if not first then first = k end
                 last = last + 1
             end
         end
-    --~ for _,k in pairs(sortedkeys(root)) do -- 1% faster:
         local sk = sortedkeys(root)
         for i=1,#sk do
             local k = sk[i]
             local v = root[k]
+            --~ if v == root then
+                -- circular
+            --~ else
             local t = type(v)
             if compact and first and type(k) == "number" and k >= first and k <= last then
                 if t == "number" then
                     if hexify then
-                        handle(("%s 0x%04X,"):format(depth,v))
+                        handle(format("%s 0x%04X,",depth,v))
                     else
-                        handle(("%s %s,"):format(depth,v))
+                        handle(format("%s %s,",depth,v))
                     end
                 elseif t == "string" then
-                    if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then
-                        handle(("%s %s,"):format(depth,v))
+                    if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                        handle(format("%s %s,",depth,v))
                     else
-                        handle(("%s %q,"):format(depth,v))
+                        handle(format("%s %q,",depth,v))
                     end
                 elseif t == "table" then
                     if not next(v) then
-                        handle(("%s {},"):format(depth))
-                    elseif inline then
+                        handle(format("%s {},",depth))
+                    elseif inline then -- and #t > 0
                         local st = simple_table(v)
                         if st then
-                            handle(("%s { %s },"):format(depth,concat(st,", ")))
+                            handle(format("%s { %s },",depth,concat(st,", ")))
                         else
                             do_serialize(v,k,depth,level+1,true)
                         end
@@ -805,39 +743,102 @@ local function do_serialize(root,name,depth,level,indexed)
                         do_serialize(v,k,depth,level+1,true)
                     end
                 elseif t == "boolean" then
-                    handle(("%s %s,"):format(depth,tostring(v)))
+                    handle(format("%s %s,",depth,tostring(v)))
                 elseif t == "function" then
                     if functions then
-                        handle(('%s loadstring(%q),'):format(depth,v:dump()))
+                        handle(format('%s loadstring(%q),',depth,dump(v)))
                     else
-                        handle(('%s "function",'):format(depth))
+                        handle(format('%s "function",',depth))
                     end
                 else
-                    handle(("%s %q,"):format(depth,tostring(v)))
+                    handle(format("%s %q,",depth,tostring(v)))
                 end
             elseif k == "__p__" then -- parent
                 if false then
-                    handle(("%s __p__=nil,"):format(depth))
+                    handle(format("%s __p__=nil,",depth))
                 end
             elseif t == "number" then
-                if hexify then
-                    handle(("%s %s=0x%04X,"):format(depth,key(k),v))
+            --~ if hexify then
+            --~     handle(format("%s %s=0x%04X,",depth,key(k),v))
+            --~ else
+            --~     handle(format("%s %s=%s,",depth,key(k),v))
+            --~ end
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,v))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    if hexify then
+                        handle(format("%s %s=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s %s=%s,",depth,k,v))
+                    end
                 else
-                    handle(("%s %s=%s,"):format(depth,key(k),v))
+                    if hexify then
+                        handle(format("%s [%q]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
                 end
             elseif t == "string" then
-                if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then
-                    handle(("%s %s=%s,"):format(depth,key(k),v))
+                if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                --~ handle(format("%s %s=%s,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%s,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%s,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%s,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
                 else
-                    handle(("%s %s=%q,"):format(depth,key(k),v))
+                --~ handle(format("%s %s=%q,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%q,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%q,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%q,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%q,",depth,k,v))
+                    end
                 end
             elseif t == "table" then
                 if not next(v) then
-                    handle(("%s %s={},"):format(depth,key(k)))
+                    --~ handle(format("%s %s={},",depth,key(k)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]={},",depth,k))
+                        else
+                            handle(format("%s [%s]={},",depth,k))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s={},",depth,k))
+                    else
+                        handle(format("%s [%q]={},",depth,k))
+                    end
                 elseif inline then
                     local st = simple_table(v)
                     if st then
-                        handle(("%s %s={ %s },"):format(depth,key(k),concat(st,", ")))
+                    --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", ")))
+                        if type(k) == "number" then -- or find(k,"^%d+$") then
+                            if hexify then
+                                handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", ")))
+                            else
+                                handle(format("%s [%s]={ %s },",depth,k,concat(st,", ")))
+                            end
+                        elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                            handle(format("%s %s={ %s },",depth,k,concat(st,", ")))
+                        else
+                            handle(format("%s [%q]={ %s },",depth,k,concat(st,", ")))
+                        end
                     else
                         do_serialize(v,k,depth,level+1)
                     end
@@ -845,24 +846,58 @@ local function do_serialize(root,name,depth,level,indexed)
                     do_serialize(v,k,depth,level+1)
                 end
             elseif t == "boolean" then
-                handle(("%s %s=%s,"):format(depth,key(k),tostring(v)))
+            --~ handle(format("%s %s=%s,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%s,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%s,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%s,",depth,k,tostring(v)))
+                end
             elseif t == "function" then
                 if functions then
-                    handle(('%s %s=loadstring(%q),'):format(depth,key(k),v:dump()))
-                else
-                    handle(('%s %s="function",'):format(depth,key(k)))
+                    --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v)))
+                        else
+                            handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v)))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=loadstring(%q),",depth,k,dump(v)))
+                    else
+                        handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v)))
+                    end
                 end
             else
-                handle(("%s %s=%q,"):format(depth,key(k),tostring(v)))
-            --  handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end)))
+                --~ handle(format("%s %s=%q,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%q,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%q,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%q,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%q,",depth,k,tostring(v)))
+                end
             end
+            --~ end
         end
     end
    if level > 0 then
-        handle(("%s},"):format(depth))
+        handle(format("%s},",depth))
     end
 end
 
+-- replacing handle by a direct t[#t+1] = ... (plus test) is not much
+-- faster (0.03 on 1.00 for zapfino.tma)
+
 local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
     noquotes = _noquotes
     hexify = _hexify
@@ -880,7 +915,7 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
         end
     elseif tname == "number" then
         if hexify then
-            handle(("[0x%04X]={"):format(name))
+            handle(format("[0x%04X]={",name))
         else
             handle("[" .. name .. "]={")
         end
@@ -1031,14 +1066,18 @@ function table.insert_after_value(t,value,str)
     end
 end
 
-function table.are_equal(a,b,n,m)
+local function are_equal(a,b,n,m) -- indexed
     if #a == #b then
         n = n or 1
         m = m or #a
         for i=n,m do
             local ai, bi = a[i], b[i]
-            if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then
-                -- continue
+            if ai==bi then
+                -- same
+            elseif type(ai)=="table" and type(bi)=="table" then
+                if not are_equal(ai,bi) then
+                    return false
+                end
             else
                 return false
             end
@@ -1049,9 +1088,30 @@ function table.are_equal(a,b,n,m)
     end
 end
 
+local function identical(a,b) -- assumes same structure
+    for ka, va in next, a do
+        local vb = b[k]
+        if va == vb then
+            -- same
+        elseif type(va) == "table" and  type(vb) == "table" then
+            if not identical(va,vb) then
+                return false
+            end
+        else
+            return false
+        end
+    end
+    return true
+end
+
+table.are_equal = are_equal
+table.identical = identical
+
+-- maybe also make a combined one
+
 function table.compact(t)
     if t then
-        for k,v in pairs(t) do
+        for k,v in next, t do
             if not next(v) then
                 t[k] = nil
             end
@@ -1080,7 +1140,7 @@ end
 
 function table.swapped(t)
     local s = { }
-    for k, v in pairs(t) do
+    for k, v in next, t do
         s[v] = k
     end
     return s
@@ -1102,14 +1162,14 @@ end
 
 function table.hexed(t,seperator)
     local tt = { }
-    for i=1,#t do tt[i] = ("0x%04X"):format(t[i]) end
+    for i=1,#t do tt[i] = format("0x%04X",t[i]) end
     return concat(tt,seperator or " ")
 end
 
 function table.reverse_hash(h)
     local r = { }
-    for k,v in pairs(h) do
-        r[v] = (k:gsub(" ","")):lower()
+    for k,v in next, h do
+        r[v] = lower(gsub(k," ",""))
     end
     return r
 end
@@ -1124,14 +1184,36 @@ function table.reverse(t)
     return tt
 end
 
+--~ function table.keys(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return k
+--~ end
 
--- filename : l-io.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+--~ function table.keys_as_string(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return concat(k,"")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-io'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-if not versions then versions = { } end versions['l-io'] = 1.001
+local byte = string.byte
 
 if string.find(os.getenv("PATH"),";") then
     io.fileseparator, io.pathseparator = "\\", ";"
@@ -1139,8 +1221,8 @@ else
     io.fileseparator, io.pathseparator = "/" , ":"
 end
 
-function io.loaddata(filename)
-    local f = io.open(filename,'rb')
+function io.loaddata(filename,textmode)
+    local f = io.open(filename,(textmode and 'r') or 'rb')
     if f then
         local data = f:read('*all')
     --  garbagecollector.check(data)
@@ -1198,146 +1280,83 @@ function io.noflines(f)
     return n
 end
 
-do
-
-    local sb = string.byte
-
-    local nextchar = {
-        [ 4] = function(f)
-            return f:read(1,1,1,1)
-        end,
-        [ 2] = function(f)
-            return f:read(1,1)
-        end,
-        [ 1] = function(f)
-            return f:read(1)
-        end,
-        [-2] = function(f)
-            local a, b = f:read(1,1)
-            return b, a
-        end,
-        [-4] = function(f)
-            local a, b, c, d = f:read(1,1,1,1)
-            return d, c, b, a
-        end
-    }
-
-    function io.characters(f,n)
-        if f then
-            return nextchar[n or 1], f
-        else
-            return nil, nil
-        end
+local nextchar = {
+    [ 4] = function(f)
+        return f:read(1,1,1,1)
+    end,
+    [ 2] = function(f)
+        return f:read(1,1)
+    end,
+    [ 1] = function(f)
+        return f:read(1)
+    end,
+    [-2] = function(f)
+        local a, b = f:read(1,1)
+        return b, a
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        return d, c, b, a
     end
+}
 
+function io.characters(f,n)
+    if f then
+        return nextchar[n or 1], f
+    else
+        return nil, nil
+    end
 end
 
-do
-
-    local sb = string.byte
-
---~     local nextbyte = {
---~         [4] = function(f)
---~             local a = f:read(1)
---~             local b = f:read(1)
---~             local c = f:read(1)
---~             local d = f:read(1)
---~             if d then
---~                 return sb(a), sb(b), sb(c), sb(d)
---~             else
---~                 return nil, nil, nil, nil
---~             end
---~         end,
---~         [2] = function(f)
---~             local a = f:read(1)
---~             local b = f:read(1)
---~             if b then
---~                 return sb(a), sb(b)
---~             else
---~                 return nil, nil
---~             end
---~         end,
---~         [1] = function (f)
---~             local a = f:read(1)
---~             if a then
---~                 return sb(a)
---~             else
---~                 return nil
---~             end
---~         end,
---~         [-2] = function (f)
---~             local a = f:read(1)
---~             local b = f:read(1)
---~             if b then
---~                 return sb(b), sb(a)
---~             else
---~                 return nil, nil
---~             end
---~         end,
---~         [-4] = function(f)
---~             local a = f:read(1)
---~             local b = f:read(1)
---~             local c = f:read(1)
---~             local d = f:read(1)
---~             if d then
---~                 return sb(d), sb(c), sb(b), sb(a)
---~             else
---~                 return nil, nil, nil, nil
---~             end
---~         end
---~     }
-
-    local nextbyte = {
-        [4] = function(f)
-            local a, b, c, d = f:read(1,1,1,1)
-            if d then
-                return sb(a), sb(b), sb(c), sb(d)
-            else
-                return nil, nil, nil, nil
-            end
-        end,
-        [2] = function(f)
-            local a, b = f:read(1,1)
-            if b then
-                return sb(a), sb(b)
-            else
-                return nil, nil
-            end
-        end,
-        [1] = function (f)
-            local a = f:read(1)
-            if a then
-                return sb(a)
-            else
-                return nil
-            end
-        end,
-        [-2] = function (f)
-            local a, b = f:read(1,1)
-            if b then
-                return sb(b), sb(a)
-            else
-                return nil, nil
-            end
-        end,
-        [-4] = function(f)
-            local a, b, c, d = f:read(1,1,1,1)
-            if d then
-                return sb(d), sb(c), sb(b), sb(a)
-            else
-                return nil, nil, nil, nil
-            end
+local nextbyte = {
+    [4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(a), byte(b), byte(c), byte(d)
+        else
+            return nil, nil, nil, nil
         end
-    }
-
-    function io.bytes(f,n)
-        if f then
-            return nextbyte[n or 1], f
+    end,
+    [2] = function(f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(a), byte(b)
         else
             return nil, nil
         end
-    end
-
+    end,
+    [1] = function (f)
+        local a = f:read(1)
+        if a then
+            return byte(a)
+        else
+            return nil
+        end
+    end,
+    [-2] = function (f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(b), byte(a)
+        else
+            return nil, nil
+        end
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(d), byte(c), byte(b), byte(a)
+        else
+            return nil, nil, nil, nil
+        end
+    end
+}
+
+function io.bytes(f,n)
+    if f then
+        return nextbyte[n or 1], f
+    else
+        return nil, nil
+    end
 end
 
 function io.ask(question,default,options)
@@ -1373,35 +1392,21 @@ function io.ask(question,default,options)
 end
 
 
--- filename : l-md5.lua
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
-
-if not versions then versions = { } end versions['l-md5'] = 1.001
-
-if md5 then do
-
-    local function convert(str,fmt)
-        return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end))
-    end
-
-    if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end
-    if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end
-    if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end
+end -- of closure
 
-end end
+do -- create closure to overcome 200 locals limit
 
+if not modules then modules = { } end modules ['l-number'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
--- filename : l-number.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
-
-if not versions then versions = { } end versions['l-number'] = 1.001
+local format = string.format
 
-if not number then number = { } end
+number = number or { }
 
 -- a,b,c,d,e,f = number.toset(100101)
 
@@ -1409,8 +1414,6 @@ function number.toset(n)
     return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)")
 end
 
-local format = string.format
-
 function number.toevenhex(n)
     local s = format("%X",n)
     if #s % 2 == 0 then
@@ -1431,72 +1434,72 @@ end
 --
 -- of course dedicated "(.)(.)(.)(.)" matches are even faster
 
-do
-    local one = lpeg.C(1-lpeg.S(''))^1
+local one = lpeg.C(1-lpeg.S(''))^1
 
-    function number.toset(n)
-        return one:match(tostring(n))
-    end
+function number.toset(n)
+    return one:match(tostring(n))
 end
 
 
 
--- filename : l-set.lua
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
 
-if not versions then versions = { } end versions['l-set'] = 1.001
+end -- of closure
 
-if not set then set = { } end
+do -- create closure to overcome 200 locals limit
 
-do
+if not modules then modules = { } end modules ['l-set'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-    local nums   = { }
-    local tabs   = { }
-    local concat = table.concat
+set = set or { }
 
-    set.create = table.tohash
+local nums   = { }
+local tabs   = { }
+local concat = table.concat
 
-    function set.tonumber(t)
-        if next(t) then
-            local s = ""
-        --  we could save mem by sorting, but it slows down
-            for k, v in pairs(t) do
-                if v then
-                --  why bother about the leading space
-                    s = s .. " " .. k
-                end
-            end
-            if not nums[s] then
-                tabs[#tabs+1] = t
-                nums[s] = #tabs
+set.create = table.tohash
+
+function set.tonumber(t)
+    if next(t) then
+        local s = ""
+    --  we could save mem by sorting, but it slows down
+        for k, v in pairs(t) do
+            if v then
+            --  why bother about the leading space
+                s = s .. " " .. k
             end
-            return nums[s]
-        else
-            return 0
         end
-    end
-
-    function set.totable(n)
-        if n == 0 then
-            return { }
-        else
-            return tabs[n] or { }
+        if not nums[s] then
+            tabs[#tabs+1] = t
+            nums[s] = #tabs
         end
+        return nums[s]
+    else
+        return 0
     end
+end
 
-    function set.contains(n,s)
-        if type(n) == "table" then
-            return n[s]
-        elseif n == 0 then
-            return false
-        else
-            local t = tabs[n]
-            return t and t[s]
-        end
+function set.totable(n)
+    if n == 0 then
+        return { }
+    else
+        return tabs[n] or { }
     end
+end
 
+function set.contains(n,s)
+    if type(n) == "table" then
+        return n[s]
+    elseif n == 0 then
+        return false
+    else
+        local t = tabs[n]
+        return t and t[s]
+    end
 end
 
 --~ local c = set.create{'aap','noot','mies'}
@@ -1513,16 +1516,19 @@ end
 
 
 
--- filename : l-os.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+end -- of closure
 
+do -- create closure to overcome 200 locals limit
 
---~ print(table.serialize(os.uname()))
+if not modules then modules = { } end modules ['l-os'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lub.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-if not versions then versions = { } end versions['l-os'] = 1.001
+local find = string.find
 
 function os.resultof(command)
     return io.popen(command,"r"):read("*all")
@@ -1535,7 +1541,7 @@ if not os.spawn then os.spawn = os.execute end
 --~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new)
 
 if not io.fileseparator then
-    if string.find(os.getenv("PATH"),";") then
+    if find(os.getenv("PATH"),";") then
         io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows"
     else
         io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix"
@@ -1573,11 +1579,10 @@ end
 
 os.gettimeofday = os.gettimeofday or os.clock
 
-do
-    local startuptime = os.gettimeofday()
-    function os.runtime()
-        return os.gettimeofday() - startuptime
-    end
+local startuptime = os.gettimeofday()
+
+function os.runtime()
+    return os.gettimeofday() - startuptime
 end
 
 --~ print(os.gettimeofday()-os.time())
@@ -1586,27 +1591,92 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
+os.arch = os.arch or function()
+    local a = os.resultof("uname -m") or "linux"
+    os.arch = function()
+        return a
+    end
+    return a
+end
+
+local platform
 
--- filename : l-file.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+function os.currentplatform(name,default)
+    if not platform then
+        local name = os.name or os.platform or name -- os.name is built in, os.platform is mine
+        if not name then
+            platform = default or "linux"
+        elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then
+            if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then
+                platform = "mswin-64"
+            else
+                platform = "mswin"
+            end
+        else
+            local architecture = os.arch()
+            if name == "linux" then
+                if find(architecture,"x86_64") then
+                    platform = "linux-64"
+                elseif find(architecture,"ppc") then
+                    platform = "linux-ppc"
+                else
+                    platform = "linux"
+                end
+            elseif name == "macosx" then
+                if find(architecture,"i386") then
+                    platform = "osx-intel"
+                else
+                    platform = "osx-ppc"
+                end
+            elseif name == "sunos" then
+                if find(architecture,"sparc") then
+                    platform = "solaris-sparc"
+                else -- if architecture == 'i86pc'
+                    platform = "solaris-intel"
+                end
+            elseif name == "freebsd" then
+                if find(architecture,"amd64") then
+                    platform = "freebsd-amd64"
+                else
+                    platform = "freebsd"
+                end
+            else
+                platform = default or name
+            end
+        end
+        function os.currentplatform()
+            return platform
+        end
+    end
+    return platform
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-file'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-if not versions then versions = { } end versions['l-file'] = 1.001
+-- needs a cleanup
 
-if not file then file = { } end
+file = file or { }
 
 local concat = table.concat
+local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub
 
 function file.removesuffix(filename)
-    return (filename:gsub("%.[%a%d]+$",""))
+    return (gsub(filename,"%.[%a%d]+$",""))
 end
 
-file.stripsuffix = file.removesuffix
-
 function file.addsuffix(filename, suffix)
-    if not filename:find("%.[%a%d]+$") then
+    if not find(filename,"%.[%a%d]+$") then
         return filename .. "." .. suffix
     else
         return filename
@@ -1614,23 +1684,23 @@ function file.addsuffix(filename, suffix)
 end
 
 function file.replacesuffix(filename, suffix)
-    return (filename:gsub("%.[%a%d]+$","")) .. "." .. suffix
+    return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix
 end
 
-function file.dirname(name)
-    return name:match("^(.+)[/\\].-$") or ""
+function file.dirname(name,default)
+    return match(name,"^(.+)[/\\].-$") or (default or "")
 end
 
 function file.basename(name)
-    return name:match("^.+[/\\](.-)$") or name
+    return match(name,"^.+[/\\](.-)$") or name
 end
 
 function file.nameonly(name)
-    return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$",""))
+    return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$",""))
 end
 
 function file.extname(name)
-    return name:match("^.+%.([^/\\]-)$") or  ""
+    return match(name,"^.+%.([^/\\]-)$") or  ""
 end
 
 file.suffix = file.extname
@@ -1643,40 +1713,20 @@ file.suffix = file.extname
 
 function file.join(...)
     local pth = concat({...},"/")
-    pth = pth:gsub("\\","/")
-    local a, b = pth:match("^(.*://)(.*)$")
+    pth = gsub(pth,"\\","/")
+    local a, b = match(pth,"^(.*://)(.*)$")
     if a and b then
-        return a .. b:gsub("//+","/")
+        return a .. gsub(b,"//+","/")
     end
-    a, b = pth:match("^(//)(.*)$")
+    a, b = match(pth,"^(//)(.*)$")
     if a and b then
-        return a .. b:gsub("//+","/")
-    end
-    return (pth:gsub("//+","/"))
-end
-
-function file.is_writable(name)
-    local f = io.open(name, 'w')
-    if f then
-        f:close()
-        return true
-    else
-        return false
-    end
-end
-
-function file.is_readable(name)
-    local f = io.open(name,'r')
-    if f then
-        f:close()
-        return true
-    else
-        return false
+        return a .. gsub(b,"//+","/")
     end
+    return (gsub(pth,"//+","/"))
 end
 
 function file.iswritable(name)
-    local a = lfs.attributes(name)
+    local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,"."))
     return a and a.permissions:sub(2,2) == "w"
 end
 
@@ -1685,24 +1735,18 @@ function file.isreadable(name)
     return a and a.permissions:sub(1,1) == "r"
 end
 
---~ function file.split_path(str)
---~     if str:find(';') then
---~         return str:splitchr(";")
---~     else
---~         return str:splitchr(io.pathseparator)
---~     end
---~ end
+file.is_readable = file.isreadable
+file.is_writable = file.iswritable
 
 -- todo: lpeg
 
 function file.split_path(str)
     local t = { }
-    str = str:gsub("\\", "/")
-    str = str:gsub("(%a):([;/])", "%1\001%2")
-    for name in str:gmatch("([^;:]+)") do
+    str = gsub(str,"\\", "/")
+    str = gsub(str,"(%a):([;/])", "%1\001%2")
+    for name in gmatch(str,"([^;:]+)") do
         if name ~= "" then
-            name = name:gsub("\001",":")
-            t[#t+1] = name
+            t[#t+1] = gsub(name,"\001",":")
         end
     end
     return t
@@ -1713,15 +1757,15 @@ function file.join_path(tab)
 end
 
 function file.collapse_path(str)
-    str = str:gsub("/%./","/")
+    str = gsub(str,"/%./","/")
     local n, m = 1, 1
     while n > 0 or m > 0 do
-        str, n = str:gsub("[^/%.]+/%.%.$","")
-        str, m = str:gsub("[^/%.]+/%.%./","")
+        str, n = gsub(str,"[^/%.]+/%.%.$","")
+        str, m = gsub(str,"[^/%.]+/%.%./","")
     end
-    str = str:gsub("([^/])/$","%1")
-    str = str:gsub("^%./","")
-    str = str:gsub("/%.$","")
+    str = gsub(str,"([^/])/$","%1")
+    str = gsub(str,"^%./","")
+    str = gsub(str,"/%.$","")
     if str == "" then str = "." end
     return str
 end
@@ -1734,7 +1778,7 @@ end
 --~ print(file.collapse_path("a/b/c/../.."))
 
 function file.robustname(str)
-    return (str:gsub("[^%a%d%/%-%.\\]+","-"))
+    return (gsub(str,"[^%a%d%/%-%.\\]+","-"))
 end
 
 file.readdata = io.loaddata
@@ -1764,8 +1808,6 @@ end
 --~     return pattern:match(name)
 --~ end
 
---~ file.stripsuffix = file.removesuffix
-
 --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1
 
 --~ function file.basename(name)
@@ -1819,7 +1861,6 @@ end
 --~ end
 
 --~ local test = file.extname
---~ local test = file.stripsuffix
 --~ local test = file.basename
 --~ local test = file.dirname
 --~ local test = file.addsuffix
@@ -1836,206 +1877,307 @@ end
 
 --~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim)
 
+-- also rewrite previous
 
--- filename : l-dir.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+local letter    = lpeg.R("az","AZ") + lpeg.S("_-+")
+local separator = lpeg.P("://")
 
-if not versions then versions = { } end versions['l-dir'] = 1.001
+local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/")
+local rootbased = lpeg.P("/") + letter*lpeg.P(":")
 
-dir = { }
+-- ./name ../name  /name c: :// name/name
 
--- optimizing for no string.find (*) does not save time
+function file.is_qualified_path(filename)
+    return qualified:match(filename)
+end
+
+function file.is_rootbased_path(filename)
+    return rootbased:match(filename)
+end
 
-if lfs then do
 
-    local attributes = lfs.attributes
-    local walkdir    = lfs.dir
+end -- of closure
 
-    local function glob_pattern(path,patt,recurse,action)
-        local ok, scanner
-        if path == "/" then
-            ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
-        else
-            ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-md5'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This also provides file checksums and checkers.
+
+local gsub, format, byte = string.gsub, string.format, string.byte
+
+local function convert(str,fmt)
+    return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end))
+end
+
+if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end
+if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end
+if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end
+
+--~ if not md5.HEX then
+--~     local function remap(chr) return format("%02X",byte(chr)) end
+--~     function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.hex then
+--~     local function remap(chr) return format("%02x",byte(chr)) end
+--~     function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.dec then
+--~     local function remap(chr) return format("%03i",byte(chr)) end
+--~     function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+
+file.needs_updating_threshold = 1
+
+function file.needs_updating(oldname,newname) -- size modification access change
+    local oldtime = lfs.attributes(oldname, modification)
+    local newtime = lfs.attributes(newname, modification)
+    if newtime >= oldtime then
+        return false
+    elseif oldtime - newtime < file.needs_updating_threshold then
+        return false
+    else
+        return true
+    end
+end
+
+function file.checksum(name)
+    if md5 then
+        local data = io.loaddata(name)
+        if data then
+            return md5.HEX(data)
         end
-        if ok and type(scanner) == "function" then
-            if not path:find("/$") then path = path .. '/' end
-            for name in scanner do
-                local full = path .. name
-                local mode = attributes(full,'mode')
-                if mode == 'file' then
-                    if full:find(patt) then
-                        action(full)
-                    end
-                elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
-                    glob_pattern(full,patt,recurse,action)
+    end
+    return nil
+end
+
+function file.loadchecksum(name)
+    if md5 then
+        local data = io.loaddata(name .. ".md5")
+        return data and data:gsub("%s","")
+    end
+    return nil
+end
+
+function file.savechecksum(name, checksum)
+    if not checksum then checksum = file.checksum(name) end
+    if checksum then
+        io.savedata(name .. ".md5",checksum)
+        return checksum
+    end
+    return nil
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-dir'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type = type
+local find, gmatch = string.find, string.gmatch
+
+dir = dir or { }
+
+-- optimizing for no string.find (*) does not save time
+
+local attributes = lfs.attributes
+local walkdir    = lfs.dir
+
+local function glob_pattern(path,patt,recurse,action)
+    local ok, scanner
+    if path == "/" then
+        ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
+    else
+        ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
+    end
+    if ok and type(scanner) == "function" then
+        if not find(path,"/$") then path = path .. '/' end
+        for name in scanner do
+            local full = path .. name
+            local mode = attributes(full,'mode')
+            if mode == 'file' then
+                if find(full,patt) then
+                    action(full)
                 end
+            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
+                glob_pattern(full,patt,recurse,action)
             end
         end
     end
+end
 
-    dir.glob_pattern = glob_pattern
+dir.glob_pattern = glob_pattern
 
-    local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V
+local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V
 
-    local pattern = Ct {
-        [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3),
-        [2] = C(((1-S("*?/"))^0 * P("/"))^0),
-        [3] = C(P(1)^0)
-    }
+local pattern = Ct {
+    [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3),
+    [2] = C(((1-S("*?/"))^0 * P("/"))^0),
+    [3] = C(P(1)^0)
+}
 
-    local filter = Cs ( (
-        P("**") / ".*" +
-        P("*")  / "[^/]*" +
-        P("?")  / "[^/]" +
-        P(".")  / "%%." +
-        P("+")  / "%%+" +
-        P("-")  / "%%-" +
-        P(1)
-    )^0 )
-
-    local function glob(str,t)
-        if type(str) == "table" then
-            local t = t or { }
-            for _, s in ipairs(str) do
-                glob(s,t)
-            end
-            return t
-        elseif lfs.isfile(str) then
+local filter = Cs ( (
+    P("**") / ".*" +
+    P("*")  / "[^/]*" +
+    P("?")  / "[^/]" +
+    P(".")  / "%%." +
+    P("+")  / "%%+" +
+    P("-")  / "%%-" +
+    P(1)
+)^0 )
+
+local function glob(str,t)
+    if type(str) == "table" then
+        local t = t or { }
+        for s=1,#str do
+            glob(str[s],t)
+        end
+        return t
+    elseif lfs.isfile(str) then
+        local t = t or { }
+        t[#t+1] = str
+        return t
+    else
+        local split = pattern:match(str)
+        if split then
             local t = t or { }
-            t[#t+1] = str
+            local action = action or function(name) t[#t+1] = name end
+            local root, path, base = split[1], split[2], split[3]
+            local recurse = find(base,"%*%*")
+            local start = root .. path
+            local result = filter:match(start .. base)
+            glob_pattern(start,result,recurse,action)
             return t
         else
-            local split = pattern:match(str)
-            if split then
-                local t = t or { }
-                local action = action or function(name) t[#t+1] = name end
-                local root, path, base = split[1], split[2], split[3]
-                local recurse = base:find("%*%*")
-                local start = root .. path
-                local result = filter:match(start .. base)
-                glob_pattern(start,result,recurse,action)
-                return t
-            else
-                return { }
-            end
+            return { }
         end
     end
+end
 
-    dir.glob = glob
+dir.glob = glob
 
-    --~ list = dir.glob("**/*.tif")
-    --~ list = dir.glob("/**/*.tif")
-    --~ list = dir.glob("./**/*.tif")
-    --~ list = dir.glob("oeps/**/*.tif")
-    --~ list = dir.glob("/oeps/**/*.tif")
+--~ list = dir.glob("**/*.tif")
+--~ list = dir.glob("/**/*.tif")
+--~ list = dir.glob("./**/*.tif")
+--~ list = dir.glob("oeps/**/*.tif")
+--~ list = dir.glob("/oeps/**/*.tif")
 
-    local function globfiles(path,recurse,func,files) -- func == pattern or function
-        if type(func) == "string" then
-            local s = func -- alas, we need this indirect way
-            func = function(name) return name:find(s) end
-        end
-        files = files or { }
-        for name in walkdir(path) do
-            if name:find("^%.") then
-                --- skip
-            else
-                local mode = attributes(name,'mode')
-                if mode == "directory" then
-                    if recurse then
-                        globfiles(path .. "/" .. name,recurse,func,files)
-                    end
-                elseif mode == "file" then
-                    if func then
-                        if func(name) then
-                            files[#files+1] = path .. "/" .. name
-                        end
-                    else
+local function globfiles(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func -- alas, we need this indirect way
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if recurse then
+                    globfiles(path .. "/" .. name,recurse,func,files)
+                end
+            elseif mode == "file" then
+                if func then
+                    if func(name) then
                         files[#files+1] = path .. "/" .. name
                     end
+                else
+                    files[#files+1] = path .. "/" .. name
                 end
             end
         end
-        return files
     end
+    return files
+end
 
-    dir.globfiles = globfiles
+dir.globfiles = globfiles
 
-    -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
-    -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
-    -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
-    -- t = dir.glob("f:/minimal/tex/**/*")
-    -- print(dir.ls("f:/minimal/tex/**/*"))
-    -- print(dir.ls("*.tex"))
+-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
+-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
+-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
+-- t = dir.glob("f:/minimal/tex/**/*")
+-- print(dir.ls("f:/minimal/tex/**/*"))
+-- print(dir.ls("*.tex"))
 
-    function dir.ls(pattern)
-        return table.concat(glob(pattern),"\n")
-    end
+function dir.ls(pattern)
+    return table.concat(glob(pattern),"\n")
+end
 
-    --~ mkdirs("temp")
-    --~ mkdirs("a/b/c")
-    --~ mkdirs(".","/a/b/c")
-    --~ mkdirs("a","b","c")
+--~ mkdirs("temp")
+--~ mkdirs("a/b/c")
+--~ mkdirs(".","/a/b/c")
+--~ mkdirs("a","b","c")
 
-    local make_indeed = true -- false
+local make_indeed = true -- false
 
-    if string.find(os.getenv("PATH"),";") then
+if string.find(os.getenv("PATH"),";") then
 
-        function dir.mkdirs(...)
-            local str, pth = "", ""
-            for _, s in ipairs({...}) do
-                if s ~= "" then
-                    if str ~= "" then
-                        str = str .. "/" .. s
-                    else
-                        str = s
-                    end
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
                 end
             end
-            local first, middle, last
-            local drive = false
-            first, middle, last = str:match("^(//)(//*)(.*)$")
+        end
+        local first, middle, last
+        local drive = false
+        first, middle, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            -- empty network path == local path
+        else
+            first, last = str:match("^(//)/*(.-)$")
             if first then
-                -- empty network path == local path
+                middle, last = str:match("([^/]+)/+(.-)$")
+                if middle then
+                    pth = "//" .. middle
+                else
+                    pth = "//" .. last
+                    last = ""
+                end
             else
-                first, last = str:match("^(//)/*(.-)$")
+                first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$")
                 if first then
-                    middle, last = str:match("([^/]+)/+(.-)$")
-                    if middle then
-                        pth = "//" .. middle
-                    else
-                        pth = "//" .. last
-                        last = ""
-                    end
+                    pth, drive = first .. middle, true
                 else
-                    first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$")
-                    if first then
-                        pth, drive = first .. middle, true
-                    else
-                        middle, last = str:match("^(/*)(.-)$")
-                        if not middle then
-                            last = str
-                        end
+                    middle, last = str:match("^(/*)(.-)$")
+                    if not middle then
+                        last = str
                     end
                 end
             end
-            for s in last:gmatch("[^/]+") do
-                if pth == "" then
-                    pth = s
-                elseif drive then
-                    pth, drive = pth .. s, false
-                else
-                    pth = pth .. "/" .. s
-                end
-                if make_indeed and not lfs.isdir(pth) then
-                    lfs.mkdir(pth)
-                end
+        end
+        for s in gmatch(last,"[^/]+") do
+            if pth == "" then
+                pth = s
+            elseif drive then
+                pth, drive = pth .. s, false
+            else
+                pth = pth .. "/" .. s
+            end
+            if make_indeed and not lfs.isdir(pth) then
+                lfs.mkdir(pth)
             end
-            return pth, (lfs.isdir(pth) == true)
         end
+        return pth, (lfs.isdir(pth) == true)
+    end
 
 --~         print(dir.mkdirs("","","a","c"))
 --~         print(dir.mkdirs("a"))
@@ -2049,79 +2191,79 @@ if lfs then do
 --~         print(dir.mkdirs("///a/b/c"))
 --~         print(dir.mkdirs("a/bbb//ccc/"))
 
-        function dir.expand_name(str)
-            local first, nothing, last = str:match("^(//)(//*)(.*)$")
-            if first then
-                first = lfs.currentdir() .. "/"
-                first = first:gsub("\\","/")
-            end
-            if not first then
-                first, last = str:match("^(//)/*(.*)$")
-            end
-            if not first then
-                first, last = str:match("^([a-zA-Z]:)(.*)$")
-                if first and not last:find("^/") then
-                    local d = lfs.currentdir()
-                    if lfs.chdir(first) then
-                        first = lfs.currentdir()
-                        first = first:gsub("\\","/")
-                    end
-                    lfs.chdir(d)
+    function dir.expand_name(str)
+        local first, nothing, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            first = lfs.currentdir() .. "/"
+            first = first:gsub("\\","/")
+        end
+        if not first then
+            first, last = str:match("^(//)/*(.*)$")
+        end
+        if not first then
+            first, last = str:match("^([a-zA-Z]:)(.*)$")
+            if first and not find(last,"^/") then
+                local d = lfs.currentdir()
+                if lfs.chdir(first) then
+                    first = lfs.currentdir()
+                    first = first:gsub("\\","/")
                 end
+                lfs.chdir(d)
             end
-            if not first then
-                first, last = lfs.currentdir(), str
-                first = first:gsub("\\","/")
-            end
-            last = last:gsub("//","/")
-            last = last:gsub("/%./","/")
-            last = last:gsub("^/*","")
-            first = first:gsub("/*$","")
-            if last == "" then
-                return first
-            else
-                return first .. "/" .. last
-            end
         end
+        if not first then
+            first, last = lfs.currentdir(), str
+            first = first:gsub("\\","/")
+        end
+        last = last:gsub("//","/")
+        last = last:gsub("/%./","/")
+        last = last:gsub("^/*","")
+        first = first:gsub("/*$","")
+        if last == "" then
+            return first
+        else
+            return first .. "/" .. last
+        end
+    end
 
-    else
+else
 
-        function dir.mkdirs(...)
-            local str, pth = "", ""
-            for _, s in ipairs({...}) do
-                if s ~= "" then
-                    if str ~= "" then
-                        str = str .. "/" .. s
-                    else
-                        str = s
-                    end
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
                 end
             end
-            str = str:gsub("/+","/")
-            if str:find("^/") then
-                pth = "/"
-                for s in str:gmatch("[^/]+") do
-                    local first = (pth == "/")
-                    if first then
-                        pth = pth .. s
-                    else
-                        pth = pth .. "/" .. s
-                    end
-                    if make_indeed and not first and not lfs.isdir(pth) then
-                        lfs.mkdir(pth)
-                    end
-                end
-            else
-                pth = "."
-                for s in str:gmatch("[^/]+") do
+        end
+        str = str:gsub("/+","/")
+        if find(str,"^/") then
+            pth = "/"
+            for s in gmatch(str,"[^/]+") do
+                local first = (pth == "/")
+                if first then
+                    pth = pth .. s
+                else
                     pth = pth .. "/" .. s
-                    if make_indeed and not lfs.isdir(pth) then
-                        lfs.mkdir(pth)
-                    end
+                end
+                if make_indeed and not first and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        else
+            pth = "."
+            for s in gmatch(str,"[^/]+") do
+                pth = pth .. "/" .. s
+                if make_indeed and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
                 end
             end
-            return pth, (lfs.isdir(pth) == true)
         end
+        return pth, (lfs.isdir(pth) == true)
+    end
 
 --~         print(dir.mkdirs("","","a","c"))
 --~         print(dir.mkdirs("a"))
@@ -2131,30 +2273,35 @@ if lfs then do
 --~         print(dir.mkdirs("///a/b/c"))
 --~         print(dir.mkdirs("a/bbb//ccc/"))
 
-        function dir.expand_name(str)
-            if not str:find("^/") then
-                str = lfs.currentdir() .. "/" .. str
-            end
-            str = str:gsub("//","/")
-            str = str:gsub("/%./","/")
-            return str
+    function dir.expand_name(str)
+        if not find(str,"^/") then
+            str = lfs.currentdir() .. "/" .. str
         end
-
+        str = str:gsub("//","/")
+        str = str:gsub("/%./","/")
+        return str
     end
 
-    dir.makedirs = dir.mkdirs
+end
 
-end end
+dir.makedirs = dir.mkdirs
 
 
--- filename : l-boolean.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+end -- of closure
 
-if not versions then versions = { } end versions['l-boolean'] = 1.001
-if not boolean  then boolean  = { } end
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-boolean'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+boolean = boolean or { }
+
+local type, tonumber = type, tonumber
 
 function boolean.tonumber(b)
     if b then return 1 else return 0 end
@@ -2201,15 +2348,19 @@ function boolean.falsetrue()
 end
 
 
--- filename : l-math.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+end -- of closure
 
-if not versions then versions = { } end versions['l-math'] = 1.001
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-math'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-local floor = math.floor
+local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan
 
 if not math.round then
     function math.round(x)
@@ -2229,23 +2380,213 @@ if not math.mod then
     end
 end
 
+local pipi = 2*math.pi/360
+
+function math.sind(d)
+    return sin(d*pipi)
+end
+
+function math.cosd(d)
+    return cos(d*pipi)
+end
+
+function math.tand(d)
+    return tan(d*pipi)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
 
-if not modules then modules = { } end modules ['l-xml'] = {
+if not modules then modules = { } end modules ['l-utils'] = {
     version   = 1.001,
-    comment   = "this module is the basis for the lxml-* ones",
+    comment   = "companion to luat-lib.tex",
     author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
     copyright = "PRAGMA ADE / ConTeXt Development Team",
     license   = "see context related readme files"
 }
 
--- RJ: key=value ... lpeg.Ca(lpeg.Cc({}) * (pattern-producing-key-and-value / rawset)^0)
+-- hm, quite unreadable
+
+if not utils        then utils        = { } end
+if not utils.merger then utils.merger = { } end
+if not utils.lua    then utils.lua    = { } end
+
+utils.merger.m_begin = "begin library merge"
+utils.merger.m_end   = "end library merge"
+utils.merger.pattern =
+    "%c+" ..
+    "%-%-%s+" .. utils.merger.m_begin ..
+    "%c+(.-)%c+" ..
+    "%-%-%s+" .. utils.merger.m_end ..
+    "%c+"
+
+function utils.merger._self_fake_()
+    return
+        "-- " .. "created merged file" .. "\n\n" ..
+        "-- " .. utils.merger.m_begin  .. "\n\n" ..
+        "-- " .. utils.merger.m_end    .. "\n\n"
+end
+
+function utils.report(...)
+    print(...)
+end
+
+utils.merger.strip_comment = true
+
+function utils.merger._self_load_(name)
+    local f, data = io.open(name), ""
+    if f then
+        utils.report("reading merge from %s",name)
+        data = f:read("*all")
+        f:close()
+    else
+        utils.report("unknown file to merge %s",name)
+    end
+    if data and utils.merger.strip_comment then
+        -- saves some 20K
+        data = data:gsub("%-%-~[^\n\r]*[\r\n]", "")
+    end
+    return data or ""
+end
+
+function utils.merger._self_save_(name, data)
+    if data ~= "" then
+        local f = io.open(name,'w')
+        if f then
+            utils.report("saving merge from %s",name)
+            f:write(data)
+            f:close()
+        end
+    end
+end
+
+function utils.merger._self_swap_(data,code)
+    if data ~= "" then
+        return (data:gsub(utils.merger.pattern, function(s)
+            return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n"
+        end, 1))
+    else
+        return ""
+    end
+end
+
+--~ stripper:
+--~
+--~ data = string.gsub(data,"%-%-~[^\n]*\n","")
+--~ data = string.gsub(data,"\n\n+","\n")
+
+function utils.merger._self_libs_(libs,list)
+    local result, f, frozen = { }, nil, false
+    result[#result+1] = "\n"
+    if type(libs) == 'string' then libs = { libs } end
+    if type(list) == 'string' then list = { list } end
+    local foundpath = nil
+    for _, lib in ipairs(libs) do
+        for _, pth in ipairs(list) do
+            pth = string.gsub(pth,"\\","/") -- file.clean_path
+            utils.report("checking library path %s",pth)
+            local name = pth .. "/" .. lib
+            if lfs.isfile(name) then
+                foundpath = pth
+            end
+        end
+        if foundpath then break end
+    end
+    if foundpath then
+        utils.report("using library path %s",foundpath)
+        local right, wrong = { }, { }
+        for _, lib in ipairs(libs) do
+            local fullname = foundpath .. "/" .. lib
+            if lfs.isfile(fullname) then
+            --  right[#right+1] = lib
+                utils.report("merging library %s",fullname)
+                result[#result+1] = "do -- create closure to overcome 200 locals limit"
+                result[#result+1] = io.loaddata(fullname,true)
+                result[#result+1] = "end -- of closure"
+            else
+            --  wrong[#wrong+1] = lib
+                utils.report("no library %s",fullname)
+            end
+        end
+        if #right > 0 then
+            utils.report("merged libraries: %s",table.concat(right," "))
+        end
+        if #wrong > 0 then
+            utils.report("skipped libraries: %s",table.concat(wrong," "))
+        end
+    else
+        utils.report("no valid library path found")
+    end
+    return table.concat(result, "\n\n")
+end
+
+function utils.merger.selfcreate(libs,list,target)
+    if target then
+        utils.merger._self_save_(
+            target,
+            utils.merger._self_swap_(
+                utils.merger._self_fake_(),
+                utils.merger._self_libs_(libs,list)
+            )
+        )
+    end
+end
+
+function utils.merger.selfmerge(name,libs,list,target)
+    utils.merger._self_save_(
+        target or name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            utils.merger._self_libs_(libs,list)
+        )
+    )
+end
+
+function utils.merger.selfclean(name)
+    utils.merger._self_save_(
+        name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            ""
+        )
+    )
+end
+
+function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true
+ -- utils.report("compiling",luafile,"into",lucfile)
+    os.remove(lucfile)
+    local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile)
+    if strip ~= false then
+        command = "-s " .. command
+    end
+    local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0)
+    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+     -- utils.report("removing",luafile)
+        os.remove(luafile)
+    end
+    return done
+end
+
+
+
+end -- of closure
 
--- some code may move to l-xmlext
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-tab'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
 --[[ldx--
 
The parser used here is inspired by the variant discussed in the lua book, but
 handles comment and processing instructions, has a different structure, provides
-parent access; a first version used different tricky but was less optimized to we
+parent access; a first version used different trickery but was less optimized to we
 went this route. First we had a find based parser, now we have an  based one.
 The find based parser can be found in l-xml-edu.lua along with other older code.

 
@@ -2267,17 +2608,30 @@ optimize the code.

 --ldx]]--
 
 xml = xml or { }
-tex = tex or { }
 
-xml.trace_lpath = false
-xml.trace_print = false
-xml.trace_remap = false
+--~ local xml = xml
+
+local concat, remove, insert = table.concat, table.remove, table.insert
+local type, next, setmetatable = type, next, setmetatable
+local format, lower, find = string.format, string.lower, string.find
 
-local format, concat, remove, insert, type, next = string.format, table.concat, table.remove, table.insert, type, next
+--[[ldx--
+
This module can be used stand alone but also inside  in
+which case it hooks into the tracker code. Therefore we provide a few
+functions that set the tracers.

+--ldx]]--
 
---~ local pairs, next, type = pairs, next, type
+local trace_remap = false
+
+if trackers then
+    trackers.register("xml.remap", function(v) trace_remap = v end)
+end
 
--- todo: some things per xml file, like namespace remapping
+function xml.settrace(str,value)
+    if str == "remap" then
+        trace_remap = value or false
+    end
+end
 
 --[[ldx--
 
First a hack to enable namespace resolving. A namespace is characterized by
@@ -2289,64 +2643,60 @@ much cleaner.

 
 xml.xmlns = xml.xmlns or { }
 
-do
+local check = lpeg.P(false)
+local parse = check
 
-    local check = lpeg.P(false)
-    local parse = check
-
-    --[[ldx--
-    
The next function associates a namespace prefix with an . This
-    normally happens independent of parsing.

+--[[ldx--
+
The next function associates a namespace prefix with an . This
+normally happens independent of parsing.

 
-    
-    xml.registerns("mml","mathml")
-    
-    --ldx]]--
+
+xml.registerns("mml","mathml")
+
+--ldx]]--
 
-    function xml.registerns(namespace, pattern) -- pattern can be an lpeg
-        check = check + lpeg.C(lpeg.P(pattern:lower())) / namespace
-        parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) }
-    end
+function xml.registerns(namespace, pattern) -- pattern can be an lpeg
+    check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace
+    parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) }
+end
 
-    --[[ldx--
-    
The next function also registers a namespace, but this time we map a
-    given namespace prefix onto a registered one, using the given
-    . This used for attributes like xmlns:m.

+--[[ldx--
+
The next function also registers a namespace, but this time we map a
+given namespace prefix onto a registered one, using the given
+. This used for attributes like xmlns:m.

 
-    
-    xml.checkns("m","http://www.w3.org/mathml")
-    
-    --ldx]]--
+
+xml.checkns("m","http://www.w3.org/mathml")
+
+--ldx]]--
 
-    function xml.checkns(namespace,url)
-        local ns = parse:match(url:lower())
-        if ns and namespace ~= ns then
-            xml.xmlns[namespace] = ns
-        end
+function xml.checkns(namespace,url)
+    local ns = parse:match(lower(url))
+    if ns and namespace ~= ns then
+        xml.xmlns[namespace] = ns
     end
+end
 
-    --[[ldx--
-    
Next we provide a way to turn an  into a registered
-    namespace. This used for the xmlns attribute.

-
-    
-    resolvedns = xml.resolvens("http://www.w3.org/mathml")
-    
-
-    This returns mml.
-    --ldx]]--
+--[[ldx--
+
Next we provide a way to turn an  into a registered
+namespace. This used for the xmlns attribute.

 
-    function xml.resolvens(url)
-         return parse:match(url:lower()) or ""
-    end
+
+resolvedns = xml.resolvens("http://www.w3.org/mathml")
+
 
-    --[[ldx--
-    
A namespace in an element can be remapped onto the registered
-    one efficiently by using the xml.xmlns table.

-    --ldx]]--
+This returns mml.
+--ldx]]--
 
+function xml.resolvens(url)
+     return parse:match(lower(url)) or ""
 end
 
+--[[ldx--
+
A namespace in an element can be remapped onto the registered
+one efficiently by using the xml.xmlns table.

+--ldx]]--
+
 --[[ldx--
 
This version uses . We follow the same approach as before, stack and top and
 such. This version is about twice as fast which is mostly due to the fact that
@@ -2382,247 +2732,253 @@ element.

 
 xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes
 
-do
+-- not just one big nested table capture (lpeg overflow)
 
-    -- not just one big nested table capture (lpeg overflow)
+local nsremap, resolvens = xml.xmlns, xml.resolvens
 
-    local nsremap, resolvens = xml.xmlns, xml.resolvens
+local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {}
 
-    local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {}
+local mt = { __tostring = xml.text }
 
-    local mt = { __tostring = xml.text }
+function xml.check_error(top,toclose)
+    return ""
+end
 
-    function xml.check_error(top,toclose)
-        return ""
-    end
+local strip   = false
+local cleanup = false
 
-    local strip   = false
-    local cleanup = false
+function xml.set_text_cleanup(fnc)
+    cleanup = fnc
+end
 
-    function xml.set_text_cleanup(fnc)
-        cleanup = fnc
+local function add_attribute(namespace,tag,value)
+    if cleanup and #value > 0 then
+        value = cleanup(value) -- new
+    end
+    if tag == "xmlns" then
+        xmlns[#xmlns+1] = resolvens(value)
+        at[tag] = value
+    elseif namespace == "xmlns" then
+        xml.checkns(tag,value)
+        at["xmlns:" .. tag] = value
+    else
+        at[tag] = value
     end
+end
 
-    local function add_attribute(namespace,tag,value)
-        if tag == "xmlns" then
-            xmlns[#xmlns+1] = resolvens(value)
-            at[tag] = value
-        elseif namespace == "xmlns" then
-            xml.checkns(tag,value)
-            at["xmlns:" .. tag] = value
-        else
-            at[tag] = value
-        end
-    end
-    local function add_begin(spacing, namespace, tag)
-        if #spacing > 0 then
-            dt[#dt+1] = spacing
-        end
-        local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
-        top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] }
-        setmetatable(top, mt)
-        dt = top.dt
-        stack[#stack+1] = top
-        at = { }
-    end
-    local function add_end(spacing, namespace, tag)
-        if #spacing > 0 then
-            dt[#dt+1] = spacing
-        end
-        local toclose = remove(stack)
-        top = stack[#stack]
-        if #stack < 1 then
-            errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "")
-        elseif toclose.tg ~= tag then -- no namespace check
-            errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "")
-        end
-        dt = top.dt
-        dt[#dt+1] = toclose
-        if toclose.at.xmlns then
-            remove(xmlns)
-        end
-    end
-    local function add_empty(spacing, namespace, tag)
-        if #spacing > 0 then
-            dt[#dt+1] = spacing
-        end
-        local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
-        top = stack[#stack]
-        dt = top.dt
-        local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top }
-        dt[#dt+1] = t
-        setmetatable(t, mt)
-        if at.xmlns then
-            remove(xmlns)
-        end
-        at = { }
-    end
-    local function add_text(text)
-        if cleanup and #text > 0 then
-            dt[#dt+1] = cleanup(text)
-        else
-            dt[#dt+1] = text
-        end
+local function add_begin(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
     end
-    local function add_special(what, spacing, text)
-        if #spacing > 0 then
-            dt[#dt+1] = spacing
-        end
-        if strip and (what == "@cm@" or what == "@dt@") then
-            -- forget it
-        else
-            dt[#dt+1] = { special=true, ns="", tg=what, dt={text} }
-        end
-    end
-    local function set_message(txt)
-        errorstr = "garbage at the end of the file: " .. txt:gsub("([ \n\r\t]*)","")
-    end
-
-    local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V
-
-    local space            = S(' \r\n\t')
-    local open             = P('<')
-    local close            = P('>')
-    local squote           = S("'")
-    local dquote           = S('"')
-    local equal            = P('=')
-    local slash            = P('/')
-    local colon            = P(':')
-    local valid            = R('az', 'AZ', '09') + S('_-.')
-    local name_yes         = C(valid^1) * colon * C(valid^1)
-    local name_nop         = C(P(true)) * C(valid^1)
-    local name             = name_yes + name_nop
-
-    local utfbom           = P('\000\000\254\255') + P('\255\254\000\000') +
-                             P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture
-
-    local spacing          = C(space^0)
-    local justtext         = C((1-open)^1)
-    local somespace        = space^1
-    local optionalspace    = space^0
-
-    local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
-    local attribute        = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute
-    local attributes       = attribute^0
-
-    local text             = justtext / add_text
-    local balanced         = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example
-
-    local emptyelement     = (spacing * open         * name * attributes * optionalspace * slash * close) / add_empty
-    local beginelement     = (spacing * open         * name * attributes * optionalspace         * close) / add_begin
-    local endelement       = (spacing * open * slash * name              * optionalspace         * close) / add_end
-
-    local begincomment     = open * P("!--")
-    local endcomment       = P("--") * close
-    local begininstruction = open * P("?")
-    local endinstruction   = P("?") * close
-    local begincdata       = open * P("![CDATA[")
-    local endcdata         = P("]]") * close
-
-    local someinstruction  = C((1 - endinstruction)^0)
-    local somecomment      = C((1 - endcomment    )^0)
-    local somecdata        = C((1 - endcdata      )^0)
-
-    function entity(k,v) entities[k] = v end
-
-    local begindoctype     = open * P("!DOCTYPE")
-    local enddoctype       = close
-    local beginset         = P("[")
-    local endset           = P("]")
-    local doctypename      = C((1-somespace)^0)
-    local elementdoctype   = optionalspace * P(" 0 then
+        dt[#dt+1] = spacing
     end
+    local toclose = remove(stack)
+    top = stack[#stack]
+    if #stack < 1 then
+        errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "")
+    elseif toclose.tg ~= tag then -- no namespace check
+        errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "")
+    end
+    dt = top.dt
+    dt[#dt+1] = toclose
+    dt[0] = top
+    if toclose.at.xmlns then
+        remove(xmlns)
+    end
+end
 
-    --[[ldx--
-    
Packaging data in an xml like table is done with the following
-    function. Maybe it will go away (when not used).

-    --ldx]]--
+local function add_empty(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
+    top = stack[#stack]
+    dt = top.dt
+    local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top }
+    dt[#dt+1] = t
+    setmetatable(t, mt)
+    if at.xmlns then
+        remove(xmlns)
+    end
+    at = { }
+end
 
-    function xml.is_valid(root)
-        return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er
+local function add_text(text)
+    if cleanup and #text > 0 then
+        dt[#dt+1] = cleanup(text)
+    else
+        dt[#dt+1] = text
     end
+end
 
-    function xml.package(tag,attributes,data)
-        local ns, tg = tag:match("^(.-):?([^:]+)$")
-        local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} }
-        setmetatable(t, mt)
-        return t
+local function add_special(what, spacing, text)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
     end
+    if strip and (what == "@cm@" or what == "@dt@") then
+        -- forget it
+    else
+        dt[#dt+1] = { special=true, ns="", tg=what, dt={text} }
+    end
+end
+
+local function set_message(txt)
+    errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","")
+end
+
+local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V
+
+local space            = S(' \r\n\t')
+local open             = P('<')
+local close            = P('>')
+local squote           = S("'")
+local dquote           = S('"')
+local equal            = P('=')
+local slash            = P('/')
+local colon            = P(':')
+local valid            = R('az', 'AZ', '09') + S('_-.')
+local name_yes         = C(valid^1) * colon * C(valid^1)
+local name_nop         = C(P(true)) * C(valid^1)
+local name             = name_yes + name_nop
+
+local utfbom           = P('\000\000\254\255') + P('\255\254\000\000') +
+                         P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture
+
+local spacing          = C(space^0)
+local justtext         = C((1-open)^1)
+local somespace        = space^1
+local optionalspace    = space^0
+
+local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
+local attribute        = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute
+local attributes       = attribute^0
+
+local text             = justtext / add_text
+local balanced         = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example
+
+local emptyelement     = (spacing * open         * name * attributes * optionalspace * slash * close) / add_empty
+local beginelement     = (spacing * open         * name * attributes * optionalspace         * close) / add_begin
+local endelement       = (spacing * open * slash * name              * optionalspace         * close) / add_end
+
+local begincomment     = open * P("!--")
+local endcomment       = P("--") * close
+local begininstruction = open * P("?")
+local endinstruction   = P("?") * close
+local begincdata       = open * P("![CDATA[")
+local endcdata         = P("]]") * close
+
+local someinstruction  = C((1 - endinstruction)^0)
+local somecomment      = C((1 - endcomment    )^0)
+local somecdata        = C((1 - endcdata      )^0)
+
+local function entity(k,v) entities[k] = v end
+
+local begindoctype     = open * P("!DOCTYPE")
+local enddoctype       = close
+local beginset         = P("[")
+local endset           = P("]")
+local doctypename      = C((1-somespace)^0)
+local elementdoctype   = optionalspace * P("Packaging data in an xml like table is done with the following
+function. Maybe it will go away (when not used).

+--ldx]]--
+
+function xml.is_valid(root)
+    return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er
+end
 
-    xml.error_handler = (logs and logs.report) or (input and input.report) or print
+function xml.package(tag,attributes,data)
+    local ns, tg = tag:match("^(.-):?([^:]+)$")
+    local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} }
+    setmetatable(t, mt)
+    return t
+end
 
+function xml.is_valid(root)
+    return root and not root.error
 end
 
+xml.error_handler = (logs and logs.report) or (input and logs.report) or print
+
 --[[ldx--
 
We cannot load an  from a filehandle so we need to load
 the whole file first. The function accepts a string representing
@@ -2666,32 +3022,28 @@ generic table copier. Since we know what we're dealing with we
 can speed up things a bit. The second argument is not to be used!

 --ldx]]--
 
-do
-
-    function copy(old,tables)
-        if old then
-            tables = tables or { }
-            local new = { }
-            if not tables[old] then
-                tables[old] = new
-            end
-            for k,v in pairs(old) do
-                new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v
-            end
-            local mt = getmetatable(old)
-            if mt then
-                setmetatable(new,mt)
-            end
-            return new
-        else
-            return { }
+function copy(old,tables)
+    if old then
+        tables = tables or { }
+        local new = { }
+        if not tables[old] then
+            tables[old] = new
+        end
+        for k,v in pairs(old) do
+            new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v
+        end
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
         end
+        return new
+    else
+        return { }
     end
-
-    xml.copy = copy
-
 end
 
+xml.copy = copy
+
 --[[ldx--
 
In  serializing the tree or parts of the tree is a major
 actitivity which is why the following function is pretty optimized resulting
@@ -2700,207 +3052,204 @@ function for all components is about 15% slower than the concatinating
 alternative.

 --ldx]]--
 
-do
-
-    -- todo: add  when not present
-
-    local fallbackhandle = (tex and tex.sprint) or io.write
-
-    local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands)
-        if not e then
-            return
-        elseif not nocommands then
-            local ec = e.command
-            if ec ~= nil then -- we can have all kind of types
-                if e.special then
-                    local etg, edt = e.tg, e.dt
-                    local spc = specialconverter and specialconverter[etg]
-                    if spc then
-                        local result = spc(edt[1])
-                        if result then
-                            handle(result)
-                            return
-                        else
-                            -- no need to handle any further
-                        end
-                    end
-                end
-                local xc = xml.command
-                if xc then
-                    xc(e,ec)
-                    return
-                end
-            end
-        end
-        handle = handle or fallbackhandle
-        local etg = e.tg
-        if etg then
+-- todo: add  when not present
+
+local fallbackhandle = (tex and tex.sprint) or io.write
+
+local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands)
+    if not e then
+        return
+    elseif not nocommands then
+        local ec = e.command
+        if ec ~= nil then -- we can have all kind of types
             if e.special then
-                local edt = e.dt
+                local etg, edt = e.tg, e.dt
                 local spc = specialconverter and specialconverter[etg]
                 if spc then
                     local result = spc(edt[1])
                     if result then
                         handle(result)
+                        return
                     else
                         -- no need to handle any further
                     end
-                elseif etg == "@pi@" then
-                --  handle(format("",edt[1]))
-                    handle("")
-                elseif etg == "@cm@" then
-                --  handle(format("",edt[1]))
-                    handle("")
-                elseif etg == "@cd@" then
-                --  handle(format("",edt[1]))
-                    handle("")
-                elseif etg == "@dt@" then
-                --  handle(format("",edt[1]))
-                    handle("")
-                elseif etg == "@rt@" then
-                    serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands)
                 end
-            else
-                local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn
-                local ats = eat and next(eat) and { } -- type test maybe faster
-                if ats then
-                    if attributeconverter then
-                        for k,v in pairs(eat) do
-                            ats[#ats+1] = format('%s=%q',k,attributeconverter(v))
-                        end
-                    else
-                        for k,v in pairs(eat) do
-                            ats[#ats+1] = format('%s=%q',k,v)
-                        end
-                    end
+            end
+            local xc = xml.command
+            if xc then
+                xc(e,ec)
+                return
+            end
+        end
+    end
+    handle = handle or fallbackhandle
+    local etg = e.tg
+    if etg then
+        if e.special then
+            local edt = e.dt
+            local spc = specialconverter and specialconverter[etg]
+            if spc then
+                local result = spc(edt[1])
+                if result then
+                    handle(result)
+                else
+                    -- no need to handle any further
                 end
-                if ern and xml.trace_remap and ern ~= ens then
-                    ens = ern
+            elseif etg == "@pi@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@cm@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@cd@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@dt@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@rt@" then
+                serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands)
+            end
+        else
+            local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn
+            local ats = eat and next(eat) and { } -- type test maybe faster
+            if ats then
+                if attributeconverter then
+                    for k,v in next, eat do
+                        ats[#ats+1] = format('%s=%q',k,attributeconverter(v))
+                    end
+                else
+                    for k,v in next, eat do
+                        ats[#ats+1] = format('%s=%q',k,v)
+                    end
                 end
-                if ens ~= "" then
-                    if edt and #edt > 0 then
-                        if ats then
-                        --  handle(format("<%s:%s %s>",ens,etg,concat(ats," ")))
-                            handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">")
-                        else
-                        --  handle(format("<%s:%s>",ens,etg))
-                            handle("<" .. ens .. ":" .. etg .. ">")
-                        end
-                        for i=1,#edt do
-                            local e = edt[i]
-                            if type(e) == "string" then
-                                if textconverter then
-                                    handle(textconverter(e))
-                                else
-                                    handle(e)
-                                end
+            end
+            if ern and trace_remap and ern ~= ens then
+                ens = ern
+            end
+            if ens ~= "" then
+                if edt and #edt > 0 then
+                    if ats then
+                    --  handle(format("<%s:%s %s>",ens,etg,concat(ats," ")))
+                        handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">")
+                    else
+                    --  handle(format("<%s:%s>",ens,etg))
+                        handle("<" .. ens .. ":" .. etg .. ">")
+                    end
+                    for i=1,#edt do
+                        local e = edt[i]
+                        if type(e) == "string" then
+                            if textconverter then
+                                handle(textconverter(e))
                             else
-                                serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                                handle(e)
                             end
-                        end
-                    --  handle(format("",ens,etg))
-                        handle("")
-                    else
-                        if ats then
-                        --  handle(format("<%s:%s %s/>",ens,etg,concat(ats," ")))
-                            handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>")
                         else
-                        --  handle(format("<%s:%s/>",ens,etg))
-                            handle("<" .. ens .. ":" .. etg .. "/>")
+                            serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands)
                         end
                     end
+                --  handle(format("",ens,etg))
+                    handle("")
                 else
-                    if edt and #edt > 0 then
-                        if ats then
-                        --  handle(format("<%s %s>",etg,concat(ats," ")))
-                            handle("<" .. etg .. " " .. concat(ats," ") .. ">")
-                        else
-                        --  handle(format("<%s>",etg))
-                            handle("<" .. etg .. ">")
-                        end
-                        for i=1,#edt do
-                            local ei = edt[i]
-                            if type(ei) == "string" then
-                                if textconverter then
-                                    handle(textconverter(ei))
-                                else
-                                    handle(ei)
-                                end
+                    if ats then
+                    --  handle(format("<%s:%s %s/>",ens,etg,concat(ats," ")))
+                        handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>")
+                    else
+                    --  handle(format("<%s:%s/>",ens,etg))
+                        handle("<" .. ens .. ":" .. etg .. "/>")
+                    end
+                end
+            else
+                if edt and #edt > 0 then
+                    if ats then
+                    --  handle(format("<%s %s>",etg,concat(ats," ")))
+                        handle("<" .. etg .. " " .. concat(ats," ") .. ">")
+                    else
+                    --  handle(format("<%s>",etg))
+                        handle("<" .. etg .. ">")
+                    end
+                    for i=1,#edt do
+                        local ei = edt[i]
+                        if type(ei) == "string" then
+                            if textconverter then
+                                handle(textconverter(ei))
                             else
-                                serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                                handle(ei)
                             end
-                        end
-                    --  handle(format("",etg))
-                        handle("")
-                    else
-                        if ats then
-                        --  handle(format("<%s %s/>",etg,concat(ats," ")))
-                            handle("<" .. etg .. " " .. concat(ats," ") .. "/>")
                         else
-                        --  handle(format("<%s/>",etg))
-                            handle("<" .. etg .. "/>")
+                            serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
                         end
                     end
+                --  handle(format("",etg))
+                    handle("")
+                else
+                    if ats then
+                    --  handle(format("<%s %s/>",etg,concat(ats," ")))
+                        handle("<" .. etg .. " " .. concat(ats," ") .. "/>")
+                    else
+                    --  handle(format("<%s/>",etg))
+                        handle("<" .. etg .. "/>")
+                    end
                 end
             end
-        elseif type(e) == "string" then
-            if textconverter then
-                handle(textconverter(e))
-            else
-                handle(e)
-            end
+        end
+    elseif type(e) == "string" then
+        if textconverter then
+            handle(textconverter(e))
         else
-            for i=1,#e do
-                local ei = e[i]
-                if type(ei) == "string" then
-                    if textconverter then
-                        handle(textconverter(ei))
-                    else
-                        handle(ei)
-                    end
+            handle(e)
+        end
+    else
+        for i=1,#e do
+            local ei = e[i]
+            if type(ei) == "string" then
+                if textconverter then
+                    handle(textconverter(ei))
                 else
-                    serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                    handle(ei)
                 end
+            else
+                serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
             end
         end
     end
+end
 
-    xml.serialize = serialize
+xml.serialize = serialize
 
-    function xml.checkbom(root) -- can be made faster
-        if root.ri then
-            local dt, found = root.dt, false
-            for k,v in ipairs(dt) do
-                if type(v) == "table" and v.special and v.tg == "@pi" and v.dt:find("xml.*version=") then
-                    found = true
-                    break
-                end
-            end
-            if not found then
-                insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } )
-                insert(dt, 2, "\n" )
+function xml.checkbom(root) -- can be made faster
+    if root.ri then
+        local dt, found = root.dt, false
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then
+                found = true
+                break
             end
         end
+        if not found then
+            insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } )
+            insert(dt, 2, "\n" )
+        end
     end
+end
 
-    --[[ldx--
-    
At the cost of some 25% runtime overhead you can first convert the tree to a string
-    and then handle the lot.

-    --ldx]]--
+--[[ldx--
+
At the cost of some 25% runtime overhead you can first convert the tree to a string
+and then handle the lot.

+--ldx]]--
 
-    function xml.tostring(root) -- 25% overhead due to collecting
-        if root then
-            if type(root) == 'string' then
-                return root
-            elseif next(root) then -- next is faster than type (and >0 test)
-                local result = { }
-                serialize(root,function(s) result[#result+1] = s end)
-                return concat(result,"")
-            end
+function xml.tostring(root) -- 25% overhead due to collecting
+    if root then
+        if type(root) == 'string' then
+            return root
+        elseif next(root) then -- next is faster than type (and >0 test)
+            local result = { }
+            serialize(root,function(s) result[#result+1] = s end)
+            return concat(result,"")
         end
-        return ""
     end
-
+    return ""
 end
 
 --[[ldx--
@@ -3010,355 +3359,394 @@ function xml.assign(dt,k,root)
     end
 end
 
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-pth'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local concat, remove, insert = table.concat, table.remove, table.insert
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find
+
+--[[ldx--
+
This module can be used stand alone but also inside  in
+which case it hooks into the tracker code. Therefore we provide a few
+functions that set the tracers. Here we overload a previously defined
+function.

+--ldx]]--
+
+local trace_lpath = false
+
+if trackers then
+    trackers.register("xml.lpath", function(v) trace_lpath = v end)
+end
+
+local settrace = xml.settrace -- lxml-tab
+
+function xml.settrace(str,value)
+    if str == "lpath" then
+        trace_lpath = value or false
+    else
+        settrace(str,value) -- lxml-tab
+    end
+end
+
 --[[ldx--
 
We've now arrived at an intersting part: accessing the tree using a subset
 of  and since we're not compatible we call it . We
 will explain more about its usage in other documents.

 --ldx]]--
 
-local lpathcalls  = 0 -- statisctics
-local lpathcached = 0 -- statisctics
-
-do
-
-    xml.functions   = xml.functions   or { }
-    xml.expressions = xml.expressions or { }
-
-    local functions   = xml.functions
-    local expressions = xml.expressions
-
-    local actions = {
-        [10] = "stay",
-        [11] = "parent",
-        [12] = "subtree root",
-        [13] = "document root",
-        [14] = "any",
-        [15] = "many",
-        [16] = "initial",
-        [20] = "match",
-        [21] = "match one of",
-        [22] = "match and attribute eq",
-        [23] = "match and attribute ne",
-        [24] = "match one of and attribute eq",
-        [25] = "match one of and attribute ne",
-        [27] = "has attribute",
-        [28] = "has value",
-        [29] = "fast match",
-        [30] = "select",
-        [31] = "expression",
-        [40] = "processing instruction",
-    }
+local lpathcalls  = 0 -- statistics
+local lpathcached = 0 -- statistics
+
+xml.functions   = xml.functions   or { }
+xml.expressions = xml.expressions or { }
+
+local functions   = xml.functions
+local expressions = xml.expressions
+
+local actions = {
+    [10] = "stay",
+    [11] = "parent",
+    [12] = "subtree root",
+    [13] = "document root",
+    [14] = "any",
+    [15] = "many",
+    [16] = "initial",
+    [20] = "match",
+    [21] = "match one of",
+    [22] = "match and attribute eq",
+    [23] = "match and attribute ne",
+    [24] = "match one of and attribute eq",
+    [25] = "match one of and attribute ne",
+    [27] = "has attribute",
+    [28] = "has value",
+    [29] = "fast match",
+    [30] = "select",
+    [31] = "expression",
+    [40] = "processing instruction",
+}
 
-    -- a rather dumb lpeg
+-- a rather dumb lpeg
 
-    local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
+local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
 
-    -- instead of using functions we just parse a few names which saves a call
-    -- later on
+-- instead of using functions we just parse a few names which saves a call
+-- later on
 
-    local lp_position  = P("position()") / "ps"
-    local lp_index     = P("index()")    / "id"
-    local lp_text      = P("text()")     / "tx"
-    local lp_name      = P("name()")     / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')"
-    local lp_tag       = P("tag()")      / "tg" -- (rt.tg or '')
-    local lp_ns        = P("ns()")       / "ns" -- (rt.ns or '')
-    local lp_noequal   = P("!=")         / "~=" + P("<=") + P(">=") + P("==")
-    local lp_doequal   = P("=")          / "=="
-    local lp_attribute = P("@")          / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')")
+local lp_position  = P("position()") / "ps"
+local lp_index     = P("index()")    / "id"
+local lp_text      = P("text()")     / "tx"
+local lp_name      = P("name()")     / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')"
+local lp_tag       = P("tag()")      / "tg" -- (rt.tg or '')
+local lp_ns        = P("ns()")       / "ns" -- (rt.ns or '')
+local lp_noequal   = P("!=")         / "~=" + P("<=") + P(">=") + P("==")
+local lp_doequal   = P("=")          / "=="
+local lp_attribute = P("@")          / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')")
 
-    local lp_lua_function  = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling
-        return t .. "("
+local lp_lua_function  = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling
+    return t .. "("
+end
+
+local lp_function  = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling
+    if expressions[t] then
+        return "expressions." .. t .. "("
+    else
+        return "expressions.error("
     end
+end
 
-    local lp_function  = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling
-        if expressions[t] then
-            return "expressions." .. t .. "("
+local lparent  = lpeg.P("(")
+local rparent  = lpeg.P(")")
+local noparent = 1 - (lparent+rparent)
+local nested   = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent}
+local value    = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"}
+
+-- if we use a dedicated namespace then we don't need to pass rt and k
+
+local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s)
+    if expressions[t] then
+        if s then
+            return "expressions." .. t .. "(r,k," .. s ..")"
         else
-            return "expressions.error("
+            return "expressions." .. t .. "(r,k)"
         end
+    else
+        return "expressions.error(" .. t .. ")"
     end
+end
 
-    local lparent  = lpeg.P("(")
-    local rparent  = lpeg.P(")")
-    local noparent = 1 - (lparent+rparent)
-    local nested   = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent}
-    local value    = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"}
+local converter = lpeg.Cs ( (
+    lp_position +
+    lp_index +
+    lp_text + lp_name + -- fast one
+    lp_special +
+    lp_noequal + lp_doequal +
+    lp_attribute +
+    lp_lua_function +
+    lp_function +
+1 )^1 )
 
-    -- if we use a dedicated namespace then we don't need to pass rt and k
+-- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1
 
-    local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s)
-        if expressions[t] then
-            if s then
-                return "expressions." .. t .. "(r,k," .. s ..")"
-            else
-                return "expressions." .. t .. "(r,k)"
-            end
-        else
-            return "expressions.error(" .. t .. ")"
-        end
-    end
-
-    local converter = lpeg.Cs ( (
-        lp_position +
-        lp_index +
-        lp_text + lp_name + -- fast one
-        lp_special +
-        lp_noequal + lp_doequal +
-        lp_attribute +
-        lp_lua_function +
-        lp_function +
-    1 )^1 )
-
-    -- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1
-
-    local template = [[
-        return function(expressions,r,d,k,e,dt,ns,tg,id,ps)
-            local at, tx = e.at or { }, dt[1] or ""
-            return %s
-        end
-    ]]
-
-    local function make_expression(str)
-        str = converter:match(str)
-        return str, loadstring(format(template,str))()
-    end
-
-    local map = { }
-
-    local space             = S(' \r\n\t')
-    local squote            = S("'")
-    local dquote            = S('"')
-    local lparent           = P('(')
-    local rparent           = P(')')
-    local atsign            = P('@')
-    local lbracket          = P('[')
-    local rbracket          = P(']')
-    local exclam            = P('!')
-    local period            = P('.')
-    local eq                = P('==') + P('=')
-    local ne                = P('<>') + P('!=')
-    local star              = P('*')
-    local slash             = P('/')
-    local colon             = P(':')
-    local bar               = P('|')
-    local hat               = P('^')
-    local valid             = R('az', 'AZ', '09') + S('_-')
---~     local name_yes          = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:*
---~     local name_nop          = C(P(true)) * C(valid^1)
-    local name_yes          = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:*
-    local name_nop          = Cc("*") * C(valid^1)
-    local name              = name_yes + name_nop
-    local number            = C((S('+-')^0 * R('09')^1)) / tonumber
-    local names             = (bar^0 * name)^1
-    local morenames         = name * (bar^0 * name)^1
-    local instructiontag    = P('pi::')
-    local spacing           = C(space^0)
-    local somespace         = space^1
-    local optionalspace     = space^0
-    local text              = C(valid^0)
-    local value             = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
-    local empty             = 1-slash
-
-    local is_eq             = lbracket * atsign * name * eq * value * rbracket
-    local is_ne             = lbracket * atsign * name * ne * value * rbracket
-    local is_attribute      = lbracket * atsign * name              * rbracket
-    local is_value          = lbracket *          value             * rbracket
-    local is_number         = lbracket *          number            * rbracket
-
-    local nobracket         = 1-(lbracket+rbracket)  -- must be improved
-    local is_expression     = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket
-
-    local is_expression     = lbracket * (C(nobracket^1))/make_expression * rbracket
-
-    local is_one            =          name
-    local is_none           = exclam * name
-    local is_one_of         =          ((lparent * names * rparent) + morenames)
-    local is_none_of        = exclam * ((lparent * names * rparent) + morenames)
-
-    local stay                     = (period                )
-    local parent                   = (period * period       ) / function(   ) map[#map+1] = { 11             } end
-    local subtreeroot              = (slash + hat           ) / function(   ) map[#map+1] = { 12             } end
-    local documentroot             = (hat * hat             ) / function(   ) map[#map+1] = { 13             } end
-    local any                      = (star                  ) / function(   ) map[#map+1] = { 14             } end
-    local many                     = (star * star           ) / function(   ) map[#map+1] = { 15             } end
-    local initial                  = (hat * hat * hat       ) / function(   ) map[#map+1] = { 16             } end
-
-    local match                    = (is_one                ) / function(...) map[#map+1] = { 20, true , ... } end
-    local match_one_of             = (is_one_of             ) / function(...) map[#map+1] = { 21, true , ... } end
-    local dont_match               = (is_none               ) / function(...) map[#map+1] = { 20, false, ... } end
-    local dont_match_one_of        = (is_none_of            ) / function(...) map[#map+1] = { 21, false, ... } end
-
-    local match_and_eq             = (is_one     * is_eq    ) / function(...) map[#map+1] = { 22, true , ... } end
-    local match_and_ne             = (is_one     * is_ne    ) / function(...) map[#map+1] = { 23, true , ... } end
-    local dont_match_and_eq        = (is_none    * is_eq    ) / function(...) map[#map+1] = { 22, false, ... } end
-    local dont_match_and_ne        = (is_none    * is_ne    ) / function(...) map[#map+1] = { 23, false, ... } end
-
-    local match_one_of_and_eq      = (is_one_of  * is_eq    ) / function(...) map[#map+1] = { 24, true , ... } end
-    local match_one_of_and_ne      = (is_one_of  * is_ne    ) / function(...) map[#map+1] = { 25, true , ... } end
-    local dont_match_one_of_and_eq = (is_none_of * is_eq    ) / function(...) map[#map+1] = { 24, false, ... } end
-    local dont_match_one_of_and_ne = (is_none_of * is_ne    ) / function(...) map[#map+1] = { 25, false, ... } end
-
-    local has_attribute            = (is_one  * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end
-    local has_value                = (is_one  * is_value    ) / function(...) map[#map+1] = { 28, true , ... } end
-    local dont_has_attribute       = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end
-    local dont_has_value           = (is_none * is_value    ) / function(...) map[#map+1] = { 28, false, ... } end
-    local position                 = (is_one  * is_number   ) / function(...) map[#map+1] = { 30, true,  ... } end
-    local dont_position            = (is_none * is_number   ) / function(...) map[#map+1] = { 30, false, ... } end
-
-    local expression               = (is_one  * is_expression)/ function(...) map[#map+1] = { 31, true,  ... } end
-    local dont_expression          = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end
-
-    local self_expression          = (         is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
-                                                                              map[#map+1] = { 31, true,  "*", "*", ... } end
-    local dont_self_expression     = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
-                                                                              map[#map+1] = { 31, false, "*", "*", ... } end
-
-    local instruction              = (instructiontag * text ) / function(...) map[#map+1] = { 40,        ... } end
-    local nothing                  = (empty                 ) / function(   ) map[#map+1] = { 15             } end -- 15 ?
-    local crap                     = (1-slash)^1
-
-    -- a few ugly goodies:
-
-    local docroottag               = P('^^')             / function(   ) map[#map+1] = { 12             } end
-    local subroottag               = P('^')              / function(   ) map[#map+1] = { 13             } end
-    local roottag                  = P('root::')         / function(   ) map[#map+1] = { 12             } end
-    local parenttag                = P('parent::')       / function(   ) map[#map+1] = { 11             } end
-    local childtag                 = P('child::')
-    local selftag                  = P('self::')
-
-    -- there will be more and order will be optimized
-
-    local selector = (
-        instruction +
---~         many + any + -- brrr, not here !
-        parent + stay +
-        dont_position + position +
-        dont_match_one_of_and_eq + dont_match_one_of_and_ne +
-        match_one_of_and_eq + match_one_of_and_ne +
-        dont_match_and_eq + dont_match_and_ne +
-        match_and_eq + match_and_ne +
-        dont_expression + expression +
-        dont_self_expression + self_expression +
-        has_attribute + has_value +
-        dont_match_one_of + match_one_of +
-        dont_match + match +
-        many + any +
-        crap + empty
-    )
+local template = [[
+    return function(expressions,r,d,k,e,dt,ns,tg,id,ps)
+        local at, tx = e.at or { }, dt[1] or ""
+        return %s
+    end
+]]
 
-    local grammar = P { "startup",
-        startup  = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"),
-        followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1,
-    }
+local function make_expression(str)
+    str = converter:match(str)
+    return str, loadstring(format(template,str))()
+end
+
+local map = { }
+
+local space                    = S(' \r\n\t')
+local squote                   = S("'")
+local dquote                   = S('"')
+local lparent                  = P('(')
+local rparent                  = P(')')
+local atsign                   = P('@')
+local lbracket                 = P('[')
+local rbracket                 = P(']')
+local exclam                   = P('!')
+local period                   = P('.')
+local eq                       = P('==') + P('=')
+local ne                       = P('<>') + P('!=')
+local star                     = P('*')
+local slash                    = P('/')
+local colon                    = P(':')
+local bar                      = P('|')
+local hat                      = P('^')
+local valid                    = R('az', 'AZ', '09') + S('_-')
+local name_yes                 = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:*
+local name_nop                 = Cc("*") * C(valid^1)
+local name                     = name_yes + name_nop
+local number                   = C((S('+-')^0 * R('09')^1)) / tonumber
+local names                    = (bar^0 * name)^1
+local morenames                = name * (bar^0 * name)^1
+local instructiontag           = P('pi::')
+local spacing                  = C(space^0)
+local somespace                = space^1
+local optionalspace            = space^0
+local text                     = C(valid^0)
+local value                    = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
+local empty                    = 1-slash
+
+local is_eq                    = lbracket * atsign * name * eq * value * rbracket
+local is_ne                    = lbracket * atsign * name * ne * value * rbracket
+local is_attribute             = lbracket * atsign * name              * rbracket
+local is_value                 = lbracket *          value             * rbracket
+local is_number                = lbracket *          number            * rbracket
+
+local nobracket                = 1-(lbracket+rbracket)  -- must be improved
+local is_expression            = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket
+
+local is_expression            = lbracket * (C(nobracket^1))/make_expression * rbracket
+
+local is_one                   =          name
+local is_none                  = exclam * name
+local is_one_of                =          ((lparent * names * rparent) + morenames)
+local is_none_of               = exclam * ((lparent * names * rparent) + morenames)
+
+local stay                     = (period                )
+local parent                   = (period * period       ) / function(   ) map[#map+1] = { 11             } end
+local subtreeroot              = (slash + hat           ) / function(   ) map[#map+1] = { 12             } end
+local documentroot             = (hat * hat             ) / function(   ) map[#map+1] = { 13             } end
+local any                      = (star                  ) / function(   ) map[#map+1] = { 14             } end
+local many                     = (star * star           ) / function(   ) map[#map+1] = { 15             } end
+local initial                  = (hat * hat * hat       ) / function(   ) map[#map+1] = { 16             } end
+
+local match                    = (is_one                ) / function(...) map[#map+1] = { 20, true , ... } end
+local match_one_of             = (is_one_of             ) / function(...) map[#map+1] = { 21, true , ... } end
+local dont_match               = (is_none               ) / function(...) map[#map+1] = { 20, false, ... } end
+local dont_match_one_of        = (is_none_of            ) / function(...) map[#map+1] = { 21, false, ... } end
+
+local match_and_eq             = (is_one     * is_eq    ) / function(...) map[#map+1] = { 22, true , ... } end
+local match_and_ne             = (is_one     * is_ne    ) / function(...) map[#map+1] = { 23, true , ... } end
+local dont_match_and_eq        = (is_none    * is_eq    ) / function(...) map[#map+1] = { 22, false, ... } end
+local dont_match_and_ne        = (is_none    * is_ne    ) / function(...) map[#map+1] = { 23, false, ... } end
+
+local match_one_of_and_eq      = (is_one_of  * is_eq    ) / function(...) map[#map+1] = { 24, true , ... } end
+local match_one_of_and_ne      = (is_one_of  * is_ne    ) / function(...) map[#map+1] = { 25, true , ... } end
+local dont_match_one_of_and_eq = (is_none_of * is_eq    ) / function(...) map[#map+1] = { 24, false, ... } end
+local dont_match_one_of_and_ne = (is_none_of * is_ne    ) / function(...) map[#map+1] = { 25, false, ... } end
+
+local has_attribute            = (is_one  * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end
+local has_value                = (is_one  * is_value    ) / function(...) map[#map+1] = { 28, true , ... } end
+local dont_has_attribute       = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end
+local dont_has_value           = (is_none * is_value    ) / function(...) map[#map+1] = { 28, false, ... } end
+local position                 = (is_one  * is_number   ) / function(...) map[#map+1] = { 30, true,  ... } end
+local dont_position            = (is_none * is_number   ) / function(...) map[#map+1] = { 30, false, ... } end
+
+local expression               = (is_one  * is_expression)/ function(...) map[#map+1] = { 31, true,  ... } end
+local dont_expression          = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end
+
+local self_expression          = (         is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
+                                                                          map[#map+1] = { 31, true,  "*", "*", ... } end
+local dont_self_expression     = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
+                                                                          map[#map+1] = { 31, false, "*", "*", ... } end
+
+local instruction              = (instructiontag * text ) / function(...) map[#map+1] = { 40,        ... } end
+local nothing                  = (empty                 ) / function(   ) map[#map+1] = { 15             } end -- 15 ?
+local crap                     = (1-slash)^1
+
+-- a few ugly goodies:
+
+local docroottag               = P('^^')             / function(   ) map[#map+1] = { 12             } end
+local subroottag               = P('^')              / function(   ) map[#map+1] = { 13             } end
+local roottag                  = P('root::')         / function(   ) map[#map+1] = { 12             } end
+local parenttag                = P('parent::')       / function(   ) map[#map+1] = { 11             } end
+local childtag                 = P('child::')
+local selftag                  = P('self::')
+
+-- there will be more and order will be optimized
+
+local selector = (
+    instruction +
+--  many + any + -- brrr, not here !
+    parent + stay +
+    dont_position + position +
+    dont_match_one_of_and_eq + dont_match_one_of_and_ne +
+    match_one_of_and_eq + match_one_of_and_ne +
+    dont_match_and_eq + dont_match_and_ne +
+    match_and_eq + match_and_ne +
+    dont_expression + expression +
+    dont_self_expression + self_expression +
+    has_attribute + has_value +
+    dont_match_one_of + match_one_of +
+    dont_match + match +
+    many + any +
+    crap + empty
+)
+
+local grammar = P { "startup",
+    startup  = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"),
+    followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1,
+}
 
-    local function compose(str)
-        if not str or str == "" then
-            -- wildcard
+local function compose(str)
+    if not str or str == "" then
+        -- wildcard
+        return true
+    elseif str == '/' then
+        -- root
+        return false
+    else
+        map = { }
+        grammar:match(str)
+        if #map == 0 then
             return true
-        elseif str == '/' then
-            -- root
-            return false
         else
-            map = { }
-            grammar:match(str)
-            if #map == 0 then
-                return true
-            else
-                local m = map[1][1]
-                if #map == 1 then
-                    if m == 14 or m == 15 then
-                        -- wildcard
-                        return true
-                    elseif m == 12 then
-                        -- root
-                        return false
-                    end
-                elseif #map == 2 and m == 12 and map[2][1] == 20 then
-                --  return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } }
-                    map[2][1] = 29
-                    return { map[2] }
-                end
-                if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then
-                    insert(map, 1, { 16 })
+            local m = map[1][1]
+            if #map == 1 then
+                if m == 14 or m == 15 then
+                    -- wildcard
+                    return true
+                elseif m == 12 then
+                    -- root
+                    return false
                 end
-            --  print((table.serialize(map)):gsub("[ \n]+"," "))
-                return map
+            elseif #map == 2 and m == 12 and map[2][1] == 20 then
+            --  return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } }
+                map[2][1] = 29
+                return { map[2] }
+            end
+            if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then
+                insert(map, 1, { 16 })
             end
+        --  print(gsub(table.serialize(map),"[ \n]+"," "))
+            return map
         end
     end
+end
 
-    local cache = { }
+local cache = { }
 
-    function xml.lpath(pattern,trace)
-        lpathcalls = lpathcalls + 1
-        if type(pattern) == "string" then
-            local result = cache[pattern]
-            if result == nil then -- can be false which is valid -)
-                result = compose(pattern)
-                cache[pattern] = result
-                lpathcached = lpathcached + 1
-            end
-            if trace or xml.trace_lpath then
-                xml.lshow(result)
-            end
-            return result
-        else
-            return pattern
+function xml.lpath(pattern,trace)
+    lpathcalls = lpathcalls + 1
+    if type(pattern) == "string" then
+        local result = cache[pattern]
+        if result == nil then -- can be false which is valid -)
+            result = compose(pattern)
+            cache[pattern] = result
+            lpathcached = lpathcached + 1
         end
+        if trace or trace_lpath then
+            xml.lshow(result)
+        end
+        return result
+    else
+        return pattern
     end
+end
 
-    function lpath_cached_patterns()
-        return cache
-    end
-
-    local fallbackreport = (texio and texio.write) or io.write
+function xml.cached_patterns()
+    return cache
+end
 
-    function xml.lshow(pattern,report)
-        report = report or fallbackreport
-        local lp = xml.lpath(pattern)
-        if lp == false then
-            report(" -: root\n")
-        elseif lp == true then
-            report(" -: wildcard\n")
-        else
-            if type(pattern) == "string" then
-                report(format("pattern: %s\n",pattern))
-            end
-            for k,v in ipairs(lp) do
-                if #v > 1 then
-                    local t = { }
-                    for i=2,#v do
-                        local vv = v[i]
-                        if type(vv) == "string" then
-                            t[#t+1] = (vv ~= "" and vv) or "#"
-                        elseif type(vv) == "boolean" then
-                            t[#t+1] = (vv and "==") or "<>"
-                        end
+--  we run out of locals (limited to 200)
+--
+--  local fallbackreport = (texio and texio.write) or io.write
+
+function xml.lshow(pattern,report)
+--      report = report or fallbackreport
+    report = report or (texio and texio.write) or io.write
+    local lp = xml.lpath(pattern)
+    if lp == false then
+        report(" -: root\n")
+    elseif lp == true then
+        report(" -: wildcard\n")
+    else
+        if type(pattern) == "string" then
+            report(format("pattern: %s\n",pattern))
+        end
+        for k=1,#lp do
+            local v = lp[k]
+            if #v > 1 then
+                local t = { }
+                for i=2,#v do
+                    local vv = v[i]
+                    if type(vv) == "string" then
+                        t[#t+1] = (vv ~= "" and vv) or "#"
+                    elseif type(vv) == "boolean" then
+                        t[#t+1] = (vv and "==") or "<>"
                     end
-                    report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," ")))
-                else
-                    report(format("%2i: %s %s\n", k,v[1],actions[v[1]]))
                 end
+                report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," ")))
+            else
+                report(format("%2i: %s %s\n", k,v[1],actions[v[1]]))
             end
         end
     end
+end
 
-    function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e
-        local t = { ... }
-        local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport
-        if e == nil then
-            report("\n")
-        elseif type(e) ~= "table" then
-            report(tostring(e))
-        elseif e.tg then
-            report(tostring(e) .. "\n")
-        else
-            for i=1,#e do
-                report(tostring(e[i]) .. "\n")
-            end
+function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e
+    local t = { ... }
+--      local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport
+    local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write
+    if e == nil then
+        report("\n")
+    elseif type(e) ~= "table" then
+        report(tostring(e))
+    elseif e.tg then
+        report(tostring(e) .. "\n")
+    else
+        for i=1,#e do
+            report(tostring(e[i]) .. "\n")
         end
     end
-
 end
 
 --[[ldx--
@@ -3376,161 +3764,174 @@ advance what we want to do with the found element the handle gets three argument
 functions.

 --ldx]]--
 
-do
-
-    local functions   = xml.functions
-    local expressions = xml.expressions
-
-    expressions.contains = string.find
-    expressions.find     = string.find
-    expressions.upper    = string.upper
-    expressions.lower    = string.lower
-    expressions.number   = tonumber
-    expressions.boolean  = toboolean
+local functions   = xml.functions
+local expressions = xml.expressions
 
-    expressions.oneof = function(s,...) -- slow
-        local t = {...} for i=1,#t do if s == t[i] then return true end end return false
-    end
+expressions.contains = string.find
+expressions.find     = string.find
+expressions.upper    = string.upper
+expressions.lower    = string.lower
+expressions.number   = tonumber
+expressions.boolean  = toboolean
 
-    expressions.error = function(str)
-        xml.error_handler("unknown function in lpath expression",str or "?")
-        return false
-    end
+expressions.oneof = function(s,...) -- slow
+    local t = {...} for i=1,#t do if s == t[i] then return true end end return false
+end
 
-    functions.text = function(root,k,n) -- unchecked, maybe one deeper
-        local t = type(t)
-        if t == "string" then
-            return t
-        else -- todo n
-            local rdt = root.dt
-            return (rdt and rdt[k]) or root[k] or ""
-        end
-    end
+expressions.error = function(str)
+    xml.error_handler("unknown function in lpath expression",str or "?")
+    return false
+end
 
-    functions.name = function(d,k,n) -- ns + tg
-        local found = false
-        n = n or 0
-        if not k then
-            -- not found
-        elseif n == 0 then
-            local dk = d[k]
-            found = dk and (type(dk) == "table") and dk
-        elseif n < 0 then
-            for i=k-1,1,-1 do
-                local di = d[i]
-                if type(di) == "table" then
-                    if n == -1 then
-                        found = di
-                        break
-                    else
-                        n = n + 1
-                    end
+functions.text = function(root,k,n) -- unchecked, maybe one deeper
+    local t = type(t)
+    if t == "string" then
+        return t
+    else -- todo n
+        local rdt = root.dt
+        return (rdt and rdt[k]) or root[k] or ""
+    end
+end
+
+functions.name = function(d,k,n) -- ns + tg
+    local found = false
+    n = n or 0
+    if not k then
+        -- not found
+    elseif n == 0 then
+        local dk = d[k]
+        found = dk and (type(dk) == "table") and dk
+    elseif n < 0 then
+        for i=k-1,1,-1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == -1 then
+                    found = di
+                    break
+                else
+                    n = n + 1
                 end
             end
-        else
-            for i=k+1,#d,1 do
-                local di = d[i]
-                if type(di) == "table" then
-                    if n == 1 then
-                        found = di
-                        break
-                    else
-                        n = n - 1
-                    end
+        end
+    else
+        for i=k+1,#d,1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == 1 then
+                    found = di
+                    break
+                else
+                    n = n - 1
                 end
             end
         end
-        if found then
-            local ns, tg = found.rn or found.ns or "", found.tg
-            if ns ~= "" then
-                return ns .. ":" .. tg
-            else
-                return tg
-            end
+    end
+    if found then
+        local ns, tg = found.rn or found.ns or "", found.tg
+        if ns ~= "" then
+            return ns .. ":" .. tg
         else
-            return ""
+            return tg
         end
+    else
+        return ""
     end
+end
 
-    functions.tag = function(d,k,n) -- only tg
-        local found = false
-        n = n or 0
-        if not k then
-            -- not found
-        elseif n == 0 then
-            local dk = d[k]
-            found = dk and (type(dk) == "table") and dk
-        elseif n < 0 then
-            for i=k-1,1,-1 do
-                local di = d[i]
-                if type(di) == "table" then
-                    if n == -1 then
-                        found = di
-                        break
-                    else
-                        n = n + 1
-                    end
+functions.tag = function(d,k,n) -- only tg
+    local found = false
+    n = n or 0
+    if not k then
+        -- not found
+    elseif n == 0 then
+        local dk = d[k]
+        found = dk and (type(dk) == "table") and dk
+    elseif n < 0 then
+        for i=k-1,1,-1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == -1 then
+                    found = di
+                    break
+                else
+                    n = n + 1
                 end
             end
-        else
-            for i=k+1,#d,1 do
-                local di = d[i]
-                if type(di) == "table" then
-                    if n == 1 then
-                        found = di
-                        break
-                    else
-                        n = n - 1
-                    end
+        end
+    else
+        for i=k+1,#d,1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == 1 then
+                    found = di
+                    break
+                else
+                    n = n - 1
                 end
             end
         end
-        return (found and found.tg) or ""
     end
+    return (found and found.tg) or ""
+end
 
-    expressions.text = functions.text
-    expressions.name = functions.name
-    expressions.tag  = functions.tag
+expressions.text = functions.text
+expressions.name = functions.name
+expressions.tag  = functions.tag
 
-    local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces
-        if not root then -- error
-            return false
-        elseif pattern == false then -- root
-            handle(root,root.dt,root.ri)
-            return false
-        elseif pattern == true then -- wildcard
-            local rootdt = root.dt
-            if rootdt then
-                local start, stop, step = 1, #rootdt, 1
-                if reverse then
-                    start, stop, step = stop, start, -1
-                end
-                for k=start,stop,step do
-                    if handle(root,rootdt,root.ri or k)            then return false end
-                    if not traverse(rootdt[k],true,handle,reverse) then return false end
-                end
+local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces
+    if not root then -- error
+        return false
+    elseif pattern == false then -- root
+        handle(root,root.dt,root.ri)
+        return false
+    elseif pattern == true then -- wildcard
+        local rootdt = root.dt
+        if rootdt then
+            local start, stop, step = 1, #rootdt, 1
+            if reverse then
+                start, stop, step = stop, start, -1
             end
-            return false
-        elseif root.dt then
-            index = index or 1
-            local action = pattern[index]
-            local command = action[1]
-            if command == 29 then -- fast case /oeps
-                local rootdt = root.dt
-                for k=1,#rootdt do
-                    local e = rootdt[k]
-                    local tg = e.tg
-                    if e.tg then
-                        local ns = e.rn or e.ns
-                        local ns_a, tg_a = action[3], action[4]
-                        local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a)
-                        if not action[2] then matched = not matched end
-                        if matched then
-                            if handle(root,rootdt,k) then return false end
-                        end
+            for k=start,stop,step do
+                if handle(root,rootdt,root.ri or k)            then return false end
+                if not traverse(rootdt[k],true,handle,reverse) then return false end
+            end
+        end
+        return false
+    elseif root.dt then
+        index = index or 1
+        local action = pattern[index]
+        local command = action[1]
+        if command == 29 then -- fast case /oeps
+            local rootdt = root.dt
+            for k=1,#rootdt do
+                local e = rootdt[k]
+                local tg = e.tg
+                if e.tg then
+                    local ns = e.rn or e.ns
+                    local ns_a, tg_a = action[3], action[4]
+                    local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a)
+                    if not action[2] then matched = not matched end
+                    if matched then
+                        if handle(root,rootdt,k) then return false end
                     end
                 end
-            elseif command == 11 then -- parent
+            end
+        elseif command == 11 then -- parent
+            local ep = root.__p__ or parent
+            if index < #pattern then
+                if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+            elseif handle(root,rootdt,k) then
+                return false
+            end
+        else
+            if (command == 16 or command == 12) and index == 1 then -- initial
+            --  wildcard = true
+                wildcard = command == 16 -- ok?
+                index = index + 1
+                action = pattern[index]
+                command = action and action[1] or 0 -- something is wrong
+            end
+            if command == 11 then -- parent
                 local ep = root.__p__ or parent
                 if index < #pattern then
                     if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
@@ -3538,41 +3939,51 @@ do
                     return false
                 end
             else
-                if (command == 16 or command == 12) and index == 1 then -- initial
-                --  wildcard = true
-                    wildcard = command == 16 -- ok?
-                    index = index + 1
-                    action = pattern[index]
-                    command = action and action[1] or 0 -- something is wrong
-                end
-                if command == 11 then -- parent
-                    local ep = root.__p__ or parent
-                    if index < #pattern then
-                        if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
-                    elseif handle(root,rootdt,k) then
-                        return false
-                    end
-                else
-                    local rootdt = root.dt
-                    local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1
-                    if command == 30 then
-                        if action[5] < 0 then
-                            start, stop, step = stop, start, -1
-                            dn = -1
-                        end
-                    elseif reverse and index == #pattern then
+                local rootdt = root.dt
+                local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1
+                if command == 30 then
+                    if action[5] < 0 then
                         start, stop, step = stop, start, -1
+                        dn = -1
                     end
-                    local idx = 0
-                    local hsh = { } -- this will slooow down the lot
-                    for k=start,stop,step do -- we used to have functions for all but a case is faster
-                        local e = rootdt[k]
-                        local ns, tg = e.rn or e.ns, e.tg
-                        if tg then
-                         -- we can optimize this for simple searches, but it probably does not pay off
-                            hsh[tg] = (hsh[tg] or 0) + 1
-                            idx = idx + 1
-                            if command == 30 then
+                elseif reverse and index == #pattern then
+                    start, stop, step = stop, start, -1
+                end
+                local idx = 0
+                local hsh = { } -- this will slooow down the lot
+                for k=start,stop,step do -- we used to have functions for all but a case is faster
+                    local e = rootdt[k]
+                    local ns, tg = e.rn or e.ns, e.tg
+                    if tg then
+                     -- we can optimize this for simple searches, but it probably does not pay off
+                        hsh[tg] = (hsh[tg] or 0) + 1
+                        idx = idx + 1
+                        if command == 30 then
+                            local ns_a, tg_a = action[3], action[4]
+                            if tg == tg_a then
+                                matched = ns_a == "*" or ns == ns_a
+                            elseif tg_a == '*' then
+                                matched, multiple = ns_a == "*" or ns == ns_a, true
+                            else
+                                matched = false
+                            end
+                            if not action[2] then matched = not matched end
+                            if matched then
+                                n = n + dn
+                                if n == action[5] then
+                                    if index == #pattern then
+                                        if handle(root,rootdt,root.ri or k) then return false end
+                                    else
+                                        if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                    end
+                                    break
+                                end
+                            elseif wildcard then
+                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                            end
+                        else
+                            local matched, multiple = false, false
+                            if command == 20 then -- match
                                 local ns_a, tg_a = action[3], action[4]
                                 if tg == tg_a then
                                     matched = ns_a == "*" or ns == ns_a
@@ -3582,193 +3993,166 @@ do
                                     matched = false
                                 end
                                 if not action[2] then matched = not matched end
-                                if matched then
-                                    n = n + dn
-                                    if n == action[5] then
-                                        if index == #pattern then
-                                            if handle(root,rootdt,root.ri or k) then return false end
-                                        else
-                                            if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
-                                        end
+                            elseif command == 21 then -- match one of
+                                multiple = true
+                                for i=3,#action,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
                                         break
                                     end
-                                elseif wildcard then
-                                    if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
                                 end
-                            else
-                                local matched, multiple = false, false
-                                if command == 20 then -- match
-                                    local ns_a, tg_a = action[3], action[4]
-                                    if tg == tg_a then
-                                        matched = ns_a == "*" or ns == ns_a
-                                    elseif tg_a == '*' then
-                                        matched, multiple = ns_a == "*" or ns == ns_a, true
-                                    else
-                                        matched = false
-                                    end
-                                    if not action[2] then matched = not matched end
-                                elseif command == 21 then -- match one of
-                                    multiple = true
-                                    for i=3,#action,2 do
-                                        local ns_a, tg_a = action[i], action[i+1]
-                                        if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
-                                            matched = true
-                                            break
-                                        end
-                                    end
-                                    if not action[2] then matched = not matched end
-                                elseif command == 22 then -- eq
-                                    local ns_a, tg_a = action[3], action[4]
-                                    if tg == tg_a then
-                                        matched = ns_a == "*" or ns == ns_a
-                                    elseif tg_a == '*' then
-                                        matched, multiple = ns_a == "*" or ns == ns_a, true
-                                    else
-                                        matched = false
-                                    end
-                                    matched = matched and e.at[action[6]] == action[7]
-                                elseif command == 23 then -- ne
-                                    local ns_a, tg_a = action[3], action[4]
-                                    if tg == tg_a then
-                                        matched = ns_a == "*" or ns == ns_a
-                                    elseif tg_a == '*' then
-                                        matched, multiple = ns_a == "*" or ns == ns_a, true
-                                    else
-                                        matched = false
-                                    end
-                                    if not action[2] then matched = not matched end
-                                    matched = mached and e.at[action[6]] ~= action[7]
-                                elseif command == 24 then -- one of eq
-                                    multiple = true
-                                    for i=3,#action-2,2 do
-                                        local ns_a, tg_a = action[i], action[i+1]
-                                        if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
-                                            matched = true
-                                            break
-                                        end
-                                    end
-                                    if not action[2] then matched = not matched end
-                                    matched = matched and e.at[action[#action-1]] == action[#action]
-                                elseif command == 25 then -- one of ne
-                                    multiple = true
-                                    for i=3,#action-2,2 do
-                                        local ns_a, tg_a = action[i], action[i+1]
-                                        if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
-                                            matched = true
-                                            break
-                                        end
-                                    end
-                                    if not action[2] then matched = not matched end
-                                    matched = matched and e.at[action[#action-1]] ~= action[#action]
-                                elseif command == 27 then -- has attribute
-                                    local ns_a, tg_a = action[3], action[4]
-                                    if tg == tg_a then
-                                        matched = ns_a == "*" or ns == ns_a
-                                    elseif tg_a == '*' then
-                                        matched, multiple = ns_a == "*" or ns == ns_a, true
-                                    else
-                                        matched = false
-                                    end
-                                    if not action[2] then matched = not matched end
-                                    matched = matched and e.at[action[5]]
-                                elseif command == 28 then -- has value
-                                    local edt, ns_a, tg_a = e.dt, action[3], action[4]
-                                    if tg == tg_a then
-                                        matched = ns_a == "*" or ns == ns_a
-                                    elseif tg_a == '*' then
-                                        matched, multiple = ns_a == "*" or ns == ns_a, true
-                                    else
-                                        matched = false
-                                    end
-                                    if not action[2] then matched = not matched end
-                                    matched = matched and edt and edt[1] == action[5]
-                                elseif command == 31 then
-                                    local edt, ns_a, tg_a = e.dt, action[3], action[4]
-                                    if tg == tg_a then
-                                        matched = ns_a == "*" or ns == ns_a
-                                    elseif tg_a == '*' then
-                                        matched, multiple = ns_a == "*" or ns == ns_a, true
-                                    else
-                                        matched = false
-                                    end
-                                    if not action[2] then matched = not matched end
-                                    if matched then
-                                        matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1)
-                                    end
+                                if not action[2] then matched = not matched end
+                            elseif command == 22 then -- eq
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
                                 end
-                                if matched then -- combine tg test and at test
-                                    if index == #pattern then
-                                        if handle(root,rootdt,root.ri or k) then return false end
-                                        if wildcard then
-                                            if multiple then
-                                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
-                                            else
-                                             -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml
-                                                if not traverse(e,pattern,handle,reverse,index,root) then return false end
-                                            end
-                                        end
-                                    else
-                                        if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
-                                    end
-                                elseif command == 14 then -- any
-                                    if index == #pattern then
-                                        if handle(root,rootdt,root.ri or k) then return false end
-                                    else
-                                        if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
-                                    end
-                                elseif command == 15 then -- many
-                                    if index == #pattern then
-                                        if handle(root,rootdt,root.ri or k) then return false end
-                                    else
-                                        if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end
+                                matched = matched and e.at[action[6]] == action[7]
+                            elseif command == 23 then -- ne
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = mached and e.at[action[6]] ~= action[7]
+                            elseif command == 24 then -- one of eq
+                                multiple = true
+                                for i=3,#action-2,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
                                     end
-                                -- not here : 11
-                                elseif command == 11 then -- parent
-                                    local ep = e.__p__ or parent
-                                    if index < #pattern then
-                                        if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end
-                                    elseif handle(root,rootdt,k) then
-                                        return false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[#action-1]] == action[#action]
+                            elseif command == 25 then -- one of ne
+                                multiple = true
+                                for i=3,#action-2,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
                                     end
-                                elseif command == 40 and e.special and tg == "@pi@" then -- pi
-                                    local pi = action[2]
-                                    if pi ~= "" then
-                                        local pt = e.dt[1]
-                                        if pt and pt:find(pi) then
-                                            if handle(root,rootdt,k) then
-                                                return false
-                                            end
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[#action-1]] ~= action[#action]
+                            elseif command == 27 then -- has attribute
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[5]]
+                            elseif command == 28 then -- has value
+                                local edt, ns_a, tg_a = e.dt, action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and edt and edt[1] == action[5]
+                            elseif command == 31 then
+                                local edt, ns_a, tg_a = e.dt, action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                if matched then
+                                    matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1)
+                                end
+                            end
+                            if matched then -- combine tg test and at test
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                    if wildcard then
+                                        if multiple then
+                                            if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                                        else
+                                         -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml
+                                            if not traverse(e,pattern,handle,reverse,index,root) then return false end
                                         end
-                                    elseif handle(root,rootdt,k) then
-                                        return false
                                     end
-                                elseif wildcard then
-                                    if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                end
+                            elseif command == 14 then -- any
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                end
+                            elseif command == 15 then -- many
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end
                                 end
-                            end
-                        else
                             -- not here : 11
-                            if command == 11 then -- parent
+                            elseif command == 11 then -- parent
                                 local ep = e.__p__ or parent
                                 if index < #pattern then
-                                    if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+                                    if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end
+                                elseif handle(root,rootdt,k) then
+                                    return false
+                                end
+                            elseif command == 40 and e.special and tg == "@pi@" then -- pi
+                                local pi = action[2]
+                                if pi ~= "" then
+                                    local pt = e.dt[1]
+                                    if pt and pt:find(pi) then
+                                        if handle(root,rootdt,k) then
+                                            return false
+                                        end
+                                    end
                                 elseif handle(root,rootdt,k) then
                                     return false
                                 end
-                                break -- else loop
+                            elseif wildcard then
+                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                            end
+                        end
+                    else
+                        -- not here : 11
+                        if command == 11 then -- parent
+                            local ep = e.__p__ or parent
+                            if index < #pattern then
+                                if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+                            elseif handle(root,rootdt,k) then
+                                return false
                             end
+                            break -- else loop
                         end
                     end
                 end
             end
         end
-        return true
     end
-
-    xml.traverse = traverse
-
+    return true
 end
 
+xml.traverse = traverse
+
 --[[ldx--
 
Next come all kind of locators and manipulators. The most generic function here
 is xml.filter(root,pattern). All registers functions in the filters namespace
@@ -3779,399 +4163,414 @@ local r, d, k = xml.filter(root,"/a/b/c/position(4)"
 
 --ldx]]--
 
-do
+local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert
 
-    local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert
+xml.filters = { }
 
-    xml.filters = { }
+function xml.filters.default(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
+    return dt and dt[dk], rt, dt, dk
+end
 
-    function xml.filters.default(root,pattern)
-        local rt, dt, dk
-        traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
-        return dt and dt[dk], rt, dt, dk
-    end
-    function xml.filters.attributes(root,pattern,arguments)
-        local rt, dt, dk
-        traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
-        local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
-        if ekat then
-            if arguments then
-                return ekat[arguments] or "", rt, dt, dk
-            else
-                return ekat, rt, dt, dk
-            end
-        else
-            return { }, rt, dt, dk
-        end
-    end
-    function xml.filters.reverse(root,pattern)
-        local rt, dt, dk
-        traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
-        return dt and dt[dk], rt, dt, dk
-    end
-    function xml.filters.count(root,pattern,everything)
-        local n = 0
-        traverse(root, lpath(pattern), function(r,d,t)
-            if everything or type(d[t]) == "table" then
-                n = n + 1
-            end
-        end)
-        return n
-    end
-    function xml.filters.elements(root, pattern) -- == all
-        local t = { }
-        traverse(root, lpath(pattern), function(r,d,k)
-            local e = d[k]
-            if e then
-                t[#t+1] = e
-            end
-        end)
-        return t
-    end
-    function xml.filters.texts(root, pattern)
-        local t = { }
-        traverse(root, lpath(pattern), function(r,d,k)
-            local e = d[k]
-            if e and e.dt then
-                t[#t+1] = e.dt
-            end
-        end)
-        return t
-    end
-    function xml.filters.first(root,pattern)
-        local rt, dt, dk
-        traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
-        return dt and dt[dk], rt, dt, dk
-    end
-    function xml.filters.last(root,pattern)
-        local rt, dt, dk
-        traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
-        return dt and dt[dk], rt, dt, dk
-    end
-    function xml.filters.index(root,pattern,arguments)
-        local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1
-        if i and i ~= 0 then
-            if i < 0 then
-                reverse, i = true, -i
-            end
-            traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse)
-            if i == 0 then
-                return dt and dt[dk], rt, dt, dk
-            end
-        end
-        return nil, nil, nil, nil
-    end
-    function xml.filters.attribute(root,pattern,arguments)
-        local rt, dt, dk
-        traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
-        local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
-        return (ekat and (ekat[arguments] or ekat[arguments:gsub("^([\"\'])(.*)%1$","%2")])) or ""
-    end
-    function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow
-        local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments)
-        if dtk then -- n
-            local dtkdt = dtk.dt
-            if not dtkdt then
-                return "", rt, dt, dk
-            elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then
-                return dtkdt[1], rt, dt, dk
-            else
-                return xml.tostring(dtkdt), rt, dt, dk
-            end
+function xml.filters.attributes(root,pattern,arguments)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
+    local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
+    if ekat then
+        if arguments then
+            return ekat[arguments] or "", rt, dt, dk
         else
-            return "", rt, dt, dk
+            return ekat, rt, dt, dk
         end
+    else
+        return { }, rt, dt, dk
     end
-    function xml.filters.tag(root,pattern,n)
-        local tag = ""
-        traverse(root, lpath(pattern), function(r,d,k)
-            tag = xml.functions.tag(d,k,n and tonumber(n))
-            return true
-        end)
-        return tag
-    end
-    function xml.filters.name(root,pattern,n)
-        local tag = ""
-        traverse(root, lpath(pattern), function(r,d,k)
-            tag = xml.functions.name(d,k,n and tonumber(n))
-            return true
-        end)
-        return tag
-    end
-
-    --[[ldx--
-    
For splitting the filter function from the path specification, we can
-    use string matching or lpeg matching. Here the difference in speed is
-    neglectable but the lpeg variant is more robust.

-    --ldx]]--
-
-    --  not faster but hipper ... although ... i can't get rid of the trailing / in the path
-
-    local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
-
-    local slash     = P('/')
-    local name      = (R("az","AZ","--","__"))^1
-    local path      = C(((1-slash)^0 * slash)^1)
-    local argument  = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" }
-    local action    = Cc(1) * path * C(name) * argument
-    local attribute = Cc(2) * path * P('@') * C(name)
-    local direct    = Cc(3) * Cc("../*") * slash^0 * C(name) * argument
-
-    local parser    = direct + action + attribute
-
-    local filters          = xml.filters
-    local attribute_filter = xml.filters.attributes
-    local default_filter   = xml.filters.default
+end
 
-    -- todo: also hash, could be gc'd
+function xml.filters.reverse(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
+    return dt and dt[dk], rt, dt, dk
+end
 
-    function xml.filter(root,pattern)
-        local kind, a, b, c = parser:match(pattern)
---~ if xml.trace_lpath then
---~     print(pattern,kind,a,b,c)
---~ end
-        if kind == 1 or kind == 3 then
-            return (filters[b] or default_filter)(root,a,c)
-        elseif kind == 2 then
-            return attribute_filter(root,a,b)
-        else
-            return default_filter(root,pattern)
-        end
-    end
-
-    --~     slightly faster, but first we need a proper test file
-    --~
-    --~     local hash = { }
-    --~
-    --~     function xml.filter(root,pattern)
-    --~         local h = hash[pattern]
-    --~         if not h then
-    --~             local kind, a, b, c = parser:match(pattern)
-    --~             if kind == 1 then
-    --~                 h = { kind, filters[b] or default_filter, a, b, c }
-    --~             elseif kind == 2 then
-    --~                 h = { kind, attribute_filter, a, b, c }
-    --~             else
-    --~                 h = { kind, default_filter, a, b, c }
-    --~             end
-    --~             hash[pattern] = h
-    --~         end
-    --~         local kind = h[1]
-    --~         if kind == 1 then
-    --~             return h[2](root,h[2],h[4])
-    --~         elseif kind == 2 then
-    --~             return h[2](root,h[2],h[3])
-    --~         else
-    --~             return h[2](root,pattern)
-    --~         end
-    --~     end
-
-    --[[ldx--
-    
The following functions collect elements and texts.

-    --ldx]]--
-
-    -- still somewhat bugged
-
-    function xml.collect_elements(root, pattern, ignorespaces)
-        local rr, dd = { }, { }
-        traverse(root, lpath(pattern), function(r,d,k)
-            local dk = d and d[k]
-            if dk then
-                if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then
-                    -- ignore
-                else
-                    local n = #rr+1
-                    rr[n], dd[n] = r, dk
-                end
-            end
-        end)
-        return dd, rr
-    end
-
-    function xml.collect_texts(root, pattern, flatten)
-        local t = { } -- no r collector
-        traverse(root, lpath(pattern), function(r,d,k)
-            if d then
-                local ek = d[k]
-                local tx = ek and ek.dt
-                if flatten then
-                    if tx then
-                        t[#t+1] = xml.tostring(tx) or ""
-                    else
-                        t[#t+1] = ""
-                    end
-                else
-                    t[#t+1] = tx or ""
-                end
-            else
-                t[#t+1] = ""
-            end
-        end)
-        return t
-    end
+function xml.filters.count(root,pattern,everything)
+    local n = 0
+    traverse(root, lpath(pattern), function(r,d,t)
+        if everything or type(d[t]) == "table" then
+            n = n + 1
+        end
+    end)
+    return n
+end
 
-    function xml.collect_tags(root, pattern, nonamespace)
-        local t = { }
-        xml.traverse(root, xml.lpath(pattern), function(r,d,k)
-            local dk = d and d[k]
-            if dk and type(dk) == "table" then
-                local ns, tg = e.ns, e.tg
-                if nonamespace then
-                    t[#t+1] = tg -- if needed we can return an extra table
-                elseif ns == "" then
-                    t[#t+1] = tg
-                else
-                    t[#t+1] = ns .. ":" .. tg
-                end
-            end
-        end)
-        return #t > 0 and {}
-    end
+function xml.filters.elements(root, pattern) -- == all
+    local t = { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local e = d[k]
+        if e then
+            t[#t+1] = e
+        end
+    end)
+    return t
+end
 
-    --[[ldx--
-    
Often using an iterators looks nicer in the code than passing handler
-    functions. The  book describes how to use coroutines for that
-    purpose (). This permits
-    code like:

+function xml.filters.texts(root, pattern)
+    local t = { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local e = d[k]
+        if e and e.dt then
+            t[#t+1] = e.dt
+        end
+    end)
+    return t
+end
 
-    
-    for r, d, k in xml.elements(xml.load('text.xml'),"title") do
-        print(d[k])
-    end
-    
+function xml.filters.first(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
+    return dt and dt[dk], rt, dt, dk
+end
 
-    
Which will print all the titles in the document. The iterator variant takes
-    1.5 times the runtime of the function variant which is due to the overhead in
-    creating the wrapper. So, instead of:

+function xml.filters.last(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
+    return dt and dt[dk], rt, dt, dk
+end
 
-    
-    function xml.filters.first(root,pattern)
-        for rt,dt,dk in xml.elements(root,pattern)
+function xml.filters.index(root,pattern,arguments)
+    local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1
+    if i and i ~= 0 then
+        if i < 0 then
+            reverse, i = true, -i
+        end
+        traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse)
+        if i == 0 then
             return dt and dt[dk], rt, dt, dk
         end
-        return nil, nil, nil, nil
     end
-    
-
-    
We use the function variants in the filters.

-    --ldx]]--
+    return nil, nil, nil, nil
+end
 
-    local wrap, yield = coroutine.wrap, coroutine.yield
+function xml.filters.attribute(root,pattern,arguments)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
+    local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
+    return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or ""
+end
 
-    function xml.elements(root,pattern,reverse)
-        return wrap(function() traverse(root, lpath(pattern), yield, reverse) end)
+function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow
+    local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments)
+    if dtk then -- n
+        local dtkdt = dtk.dt
+        if not dtkdt then
+            return "", rt, dt, dk
+        elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then
+            return dtkdt[1], rt, dt, dk
+        else
+            return xml.tostring(dtkdt), rt, dt, dk
+        end
+    else
+        return "", rt, dt, dk
     end
+end
 
-    function xml.elements_only(root,pattern,reverse)
-        return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end)
-    end
+function xml.filters.tag(root,pattern,n)
+    local tag = ""
+    traverse(root, lpath(pattern), function(r,d,k)
+        tag = xml.functions.tag(d,k,n and tonumber(n))
+        return true
+    end)
+    return tag
+end
+
+function xml.filters.name(root,pattern,n)
+    local tag = ""
+    traverse(root, lpath(pattern), function(r,d,k)
+        tag = xml.functions.name(d,k,n and tonumber(n))
+        return true
+    end)
+    return tag
+end
+
+--[[ldx--
+
For splitting the filter function from the path specification, we can
+use string matching or lpeg matching. Here the difference in speed is
+neglectable but the lpeg variant is more robust.

+--ldx]]--
+
+--  not faster but hipper ... although ... i can't get rid of the trailing / in the path
+
+local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
+
+local slash     = P('/')
+local name      = (R("az","AZ","--","__"))^1
+local path      = C(((1-slash)^0 * slash)^1)
+local argument  = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" }
+local action    = Cc(1) * path * C(name) * argument
+local attribute = Cc(2) * path * P('@') * C(name)
+local direct    = Cc(3) * Cc("../*") * slash^0 * C(name) * argument
+
+local parser    = direct + action + attribute
+
+local filters          = xml.filters
+local attribute_filter = xml.filters.attributes
+local default_filter   = xml.filters.default
 
-    function xml.each_element(root, pattern, handle, reverse)
-        local ok
-        traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse)
-        return ok
+-- todo: also hash, could be gc'd
+
+function xml.filter(root,pattern)
+    local kind, a, b, c = parser:match(pattern)
+    if kind == 1 or kind == 3 then
+        return (filters[b] or default_filter)(root,a,c)
+    elseif kind == 2 then
+        return attribute_filter(root,a,b)
+    else
+        return default_filter(root,pattern)
     end
+end
 
-    function xml.process_elements(root, pattern, handle)
-        traverse(root, lpath(pattern), function(r,d,k)
-            local dkdt = d[k].dt
-            if dkdt then
-                for i=1,#dkdt do
-                    local v = dkdt[i]
-                    if v.tg then handle(v) end
-                end
+--~     slightly faster, but first we need a proper test file
+--~
+--~     local hash = { }
+--~
+--~     function xml.filter(root,pattern)
+--~         local h = hash[pattern]
+--~         if not h then
+--~             local kind, a, b, c = parser:match(pattern)
+--~             if kind == 1 then
+--~                 h = { kind, filters[b] or default_filter, a, b, c }
+--~             elseif kind == 2 then
+--~                 h = { kind, attribute_filter, a, b, c }
+--~             else
+--~                 h = { kind, default_filter, a, b, c }
+--~             end
+--~             hash[pattern] = h
+--~         end
+--~         local kind = h[1]
+--~         if kind == 1 then
+--~             return h[2](root,h[2],h[4])
+--~         elseif kind == 2 then
+--~             return h[2](root,h[2],h[3])
+--~         else
+--~             return h[2](root,pattern)
+--~         end
+--~     end
+
+--[[ldx--
+
The following functions collect elements and texts.

+--ldx]]--
+
+-- still somewhat bugged
+
+function xml.collect_elements(root, pattern, ignorespaces)
+    local rr, dd = { }, { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d and d[k]
+        if dk then
+            if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then
+                -- ignore
+            else
+                local n = #rr+1
+                rr[n], dd[n] = r, dk
             end
-        end)
-    end
+        end
+    end)
+    return dd, rr
+end
 
-    function xml.process_attributes(root, pattern, handle)
-        traverse(root, lpath(pattern), function(r,d,k)
+function xml.collect_texts(root, pattern, flatten)
+    local t = { } -- no r collector
+    traverse(root, lpath(pattern), function(r,d,k)
+        if d then
             local ek = d[k]
-            local a = ek.at or { }
-            handle(a)
-            if next(a) then -- next is faster than type (and >0 test)
-                ek.at = a
+            local tx = ek and ek.dt
+            if flatten then
+                if tx then
+                    t[#t+1] = xml.tostring(tx) or ""
+                else
+                    t[#t+1] = ""
+                end
             else
-                ek.at = nil
+                t[#t+1] = tx or ""
             end
-        end)
+        else
+            t[#t+1] = ""
+        end
+    end)
+    return t
+end
+
+function xml.collect_tags(root, pattern, nonamespace)
+    local t = { }
+    xml.traverse(root, xml.lpath(pattern), function(r,d,k)
+        local dk = d and d[k]
+        if dk and type(dk) == "table" then
+            local ns, tg = e.ns, e.tg
+            if nonamespace then
+                t[#t+1] = tg -- if needed we can return an extra table
+            elseif ns == "" then
+                t[#t+1] = tg
+            else
+                t[#t+1] = ns .. ":" .. tg
+            end
+        end
+    end)
+    return #t > 0 and {}
+end
+
+--[[ldx--
+
Often using an iterators looks nicer in the code than passing handler
+functions. The  book describes how to use coroutines for that
+purpose (). This permits
+code like:

+
+
+for r, d, k in xml.elements(xml.load('text.xml'),"title") do
+    print(d[k])
+end
+
+
+
Which will print all the titles in the document. The iterator variant takes
+1.5 times the runtime of the function variant which is due to the overhead in
+creating the wrapper. So, instead of:

+
+
+function xml.filters.first(root,pattern)
+    for rt,dt,dk in xml.elements(root,pattern)
+        return dt and dt[dk], rt, dt, dk
     end
+    return nil, nil, nil, nil
+end
+
+
+
We use the function variants in the filters.

+--ldx]]--
 
-    --[[ldx--
-    
We've now arrives at the functions that manipulate the tree.

-    --ldx]]--
+local wrap, yield = coroutine.wrap, coroutine.yield
 
-    function xml.inject_element(root, pattern, element, prepend)
-        if root and element then
-            local matches, collect = { }, nil
-            if type(element) == "string" then
-                element = convert(element,true)
+function xml.elements(root,pattern,reverse)
+    return wrap(function() traverse(root, lpath(pattern), yield, reverse) end)
+end
+
+function xml.elements_only(root,pattern,reverse)
+    return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end)
+end
+
+function xml.each_element(root, pattern, handle, reverse)
+    local ok
+    traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse)
+    return ok
+end
+
+function xml.process_elements(root, pattern, handle)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dkdt = d[k].dt
+        if dkdt then
+            for i=1,#dkdt do
+                local v = dkdt[i]
+                if v.tg then handle(v) end
             end
-            if element then
-                collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
-                traverse(root, lpath(pattern), collect)
-                for i=1,#matches do
-                    local m = matches[i]
-                    local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil
-                    if element.ri then
-                        element = element.dt[element.ri].dt
+        end
+    end)
+end
+
+function xml.process_attributes(root, pattern, handle)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local ek = d[k]
+        local a = ek.at or { }
+        handle(a)
+        if next(a) then -- next is faster than type (and >0 test)
+            ek.at = a
+        else
+            ek.at = nil
+        end
+    end)
+end
+
+--[[ldx--
+
We've now arrives at the functions that manipulate the tree.

+--ldx]]--
+
+function xml.inject_element(root, pattern, element, prepend)
+    if root and element then
+        local matches, collect = { }, nil
+        if type(element) == "string" then
+            element = convert(element,true)
+        end
+        if element then
+            collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
+            traverse(root, lpath(pattern), collect)
+            for i=1,#matches do
+                local m = matches[i]
+                local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil
+                if element.ri then
+                    element = element.dt[element.ri].dt
+                else
+                    element = element.dt
+                end
+                if r.ri then
+                    edt = r.dt[r.ri].dt
+                else
+                    edt = d and d[k] and d[k].dt
+                end
+                if edt then
+                    local be, af
+                    if prepend then
+                        be, af = xml.copy(element), edt
                     else
-                        element = element.dt
+                        be, af = edt, xml.copy(element)
                     end
-                    if r.ri then
-                        edt = r.dt[r.ri].dt
-                    else
-                        edt = d and d[k] and d[k].dt
+                    for i=1,#af do
+                        be[#be+1] = af[i]
                     end
-                    if edt then
-                        local be, af
-                        if prepend then
-                            be, af = xml.copy(element), edt
-                        else
-                            be, af = edt, xml.copy(element)
-                        end
-                        for i=1,#af do
-                            be[#be+1] = af[i]
-                        end
-                        if r.ri then
-                            r.dt[r.ri].dt = be
-                        else
-                            d[k].dt = be
-                        end
+                    if r.ri then
+                        r.dt[r.ri].dt = be
                     else
-                     -- r.dt = element.dt -- todo
+                        d[k].dt = be
                     end
+                else
+                 -- r.dt = element.dt -- todo
                 end
             end
         end
     end
+end
 
-    -- todo: copy !
+-- todo: copy !
 
-    function xml.insert_element(root, pattern, element, before) -- todo: element als functie
-        if root and element then
-            if pattern == "/" then
-                xml.inject_element(root, pattern, element, before)
-            else
-                local matches, collect = { }, nil
-                if type(element) == "string" then
-                    element = convert(element,true)
-                end
-                if element and element.ri then
-                    element = element.dt[element.ri]
-                end
-                if element then
-                    collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
-                    traverse(root, lpath(pattern), collect)
-                    for i=#matches,1,-1 do
-                        local m = matches[i]
-                        local r, d, k, element = m[1], m[2], m[3], m[4]
-                        if not before then k = k + 1 end
-                        if element.tg then
-                            insert(d,k,element) -- untested
-                        elseif element.dt then
-                            for _,v in ipairs(element.dt) do -- i added
-                                insert(d,k,v)
+function xml.insert_element(root, pattern, element, before) -- todo: element als functie
+    if root and element then
+        if pattern == "/" then
+            xml.inject_element(root, pattern, element, before)
+        else
+            local matches, collect = { }, nil
+            if type(element) == "string" then
+                element = convert(element,true)
+            end
+            if element and element.ri then
+                element = element.dt[element.ri]
+            end
+            if element then
+                collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
+                traverse(root, lpath(pattern), collect)
+                for i=#matches,1,-1 do
+                    local m = matches[i]
+                    local r, d, k, element = m[1], m[2], m[3], m[4]
+                    if not before then k = k + 1 end
+                    if element.tg then
+                        insert(d,k,element) -- untested
+--~                         elseif element.dt then
+--~                             for _,v in ipairs(element.dt) do -- i added
+--~                                 insert(d,k,v)
+--~                                 k = k + 1
+--~                             end
+--~                         end
+                    else
+                        local edt = element.dt
+                        if edt then
+                            for i=1,#edt do
+                                insert(d,k,edt[i])
                                 k = k + 1
                             end
                         end
@@ -4180,176 +4579,176 @@ do
             end
         end
     end
+end
 
-    xml.insert_element_after  =                 xml.insert_element
-    xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end
-    xml.inject_element_after  =                 xml.inject_element
-    xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end
+xml.insert_element_after  =                 xml.insert_element
+xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end
+xml.inject_element_after  =                 xml.inject_element
+xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end
 
-    function xml.delete_element(root, pattern)
-        local matches, deleted = { }, { }
-        local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end
-        traverse(root, lpath(pattern), collect)
-        for i=#matches,1,-1 do
-            local m = matches[i]
-            deleted[#deleted+1] = table.remove(m[2],m[3])
-        end
-        return deleted
+function xml.delete_element(root, pattern)
+    local matches, deleted = { }, { }
+    local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end
+    traverse(root, lpath(pattern), collect)
+    for i=#matches,1,-1 do
+        local m = matches[i]
+        deleted[#deleted+1] = remove(m[2],m[3])
     end
+    return deleted
+end
 
-    function xml.replace_element(root, pattern, element)
-        if type(element) == "string" then
-            element = convert(element,true)
-        end
-        if element and element.ri then
-            element = element.dt[element.ri]
-        end
-        if element then
-            traverse(root, lpath(pattern), function(rm, d, k)
-                d[k] = element.dt -- maybe not clever enough
-            end)
-        end
+function xml.replace_element(root, pattern, element)
+    if type(element) == "string" then
+        element = convert(element,true)
     end
-
-    local function load_data(name) -- == io.loaddata
-        local f, data = io.open(name), ""
-        if f then
-            data = f:read("*all",'b') -- 'b' ?
-            f:close()
-        end
-        return data
+    if element and element.ri then
+        element = element.dt[element.ri]
+    end
+    if element then
+        traverse(root, lpath(pattern), function(rm, d, k)
+            d[k] = element.dt -- maybe not clever enough
+        end)
     end
+end
 
-    function xml.include(xmldata,pattern,attribute,recursive,loaddata)
-        -- parse="text" (default: xml), encoding="" (todo)
-        -- attribute = attribute or 'href'
-        pattern = pattern or 'include'
-        loaddata = loaddata or load_data
-        local function include(r,d,k)
-            local ek, name = d[k], nil
-            if not attribute or attribute == "" then
-                local ekdt = ek.dt
-                name = (type(ekdt) == "table" and ekdt[1]) or ekdt
-            end
-            if not name then
-                if ek.at then
-                    for a in (attribute or "href"):gmatch("([^|]+)") do
-                        name = ek.at[a]
-                        if name then break end
-                    end
+local function load_data(name) -- == io.loaddata
+    local f, data = io.open(name), ""
+    if f then
+        data = f:read("*all",'b') -- 'b' ?
+        f:close()
+    end
+    return data
+end
+
+function xml.include(xmldata,pattern,attribute,recursive,loaddata)
+    -- parse="text" (default: xml), encoding="" (todo)
+    -- attribute = attribute or 'href'
+    pattern = pattern or 'include'
+    loaddata = loaddata or load_data
+    local function include(r,d,k)
+        local ek, name = d[k], nil
+        if not attribute or attribute == "" then
+            local ekdt = ek.dt
+            name = (type(ekdt) == "table" and ekdt[1]) or ekdt
+        end
+        if not name then
+            if ek.at then
+                for a in gmatch(attribute or "href","([^|]+)") do
+                    name = ek.at[a]
+                    if name then break end
                 end
             end
-            local data = (name and name ~= "" and loaddata(name)) or ""
-            if data == "" then
+        end
+        local data = (name and name ~= "" and loaddata(name)) or ""
+        if data == "" then
+            xml.empty(d,k)
+        elseif ek.at["parse"] == "text" then -- for the moment hard coded
+            d[k] = xml.escaped(data)
+        else
+            local xi = xml.convert(data)
+            if not xi then
                 xml.empty(d,k)
-            elseif ek.at["parse"] == "text" then -- for the moment hard coded
-                d[k] = xml.escaped(data)
             else
-                local xi = xml.convert(data)
-                if not xi then
-                    xml.empty(d,k)
-                else
-                    if recursive then
-                        xml.include(xi,pattern,attribute,recursive,loaddata)
-                    end
-                    xml.assign(d,k,xi)
+                if recursive then
+                    xml.include(xi,pattern,attribute,recursive,loaddata)
                 end
+                xml.assign(d,k,xi)
             end
         end
-        xml.each_element(xmldata, pattern, include)
     end
+    xml.each_element(xmldata, pattern, include)
+end
 
-    function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space !
-        traverse(root, lpath(pattern), function(r,d,k)
-            local dkdt = d[k].dt
-            if dkdt then -- can be optimized
-                local t = { }
-                for i=1,#dkdt do
-                    local str = dkdt[i]
-                    if type(str) == "string" then
+function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space !
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dkdt = d[k].dt
+        if dkdt then -- can be optimized
+            local t = { }
+            for i=1,#dkdt do
+                local str = dkdt[i]
+                if type(str) == "string" then
 
+                    if str == "" then
+                        -- stripped
+                    else
+                        if nolines then
+                            str = gsub(str,"[ \n\r\t]+"," ")
+                        end
                         if str == "" then
                             -- stripped
                         else
-                            if nolines then
-                                str = str:gsub("[ \n\r\t]+"," ")
-                            end
-                            if str == "" then
-                                -- stripped
-                            else
-                                t[#t+1] = str
-                            end
+                            t[#t+1] = str
                         end
-                    else
-                        t[#t+1] = str
                     end
+                else
+                    t[#t+1] = str
                 end
-                d[k].dt = t
             end
-        end)
-    end
+            d[k].dt = t
+        end
+    end)
+end
 
-    function xml.rename_space(root, oldspace, newspace) -- fast variant
-        local ndt = #root.dt
-        local rename = xml.rename_space
-        for i=1,ndt or 0 do
-            local e = root[i]
-            if type(e) == "table" then
-                if e.ns == oldspace then
-                    e.ns = newspace
-                    if e.rn then
-                        e.rn = newspace
-                    end
-                end
-                local edt = e.dt
-                if edt then
-                    rename(edt, oldspace, newspace)
+local function rename_space(root, oldspace, newspace) -- fast variant
+    local ndt = #root.dt
+    for i=1,ndt or 0 do
+        local e = root[i]
+        if type(e) == "table" then
+            if e.ns == oldspace then
+                e.ns = newspace
+                if e.rn then
+                    e.rn = newspace
                 end
             end
+            local edt = e.dt
+            if edt then
+                rename_space(edt, oldspace, newspace)
+            end
         end
     end
+end
 
-    function xml.remap_tag(root, pattern, newtg)
-        traverse(root, lpath(pattern), function(r,d,k)
-            d[k].tg = newtg
-        end)
-    end
-    function xml.remap_namespace(root, pattern, newns)
-        traverse(root, lpath(pattern), function(r,d,k)
-            d[k].ns = newns
-        end)
-    end
-    function xml.check_namespace(root, pattern, newns)
-        traverse(root, lpath(pattern), function(r,d,k)
-            local dk = d[k]
-            if (not dk.rn or dk.rn == "") and dk.ns == "" then
-                dk.rn = newns
-            end
-        end)
-    end
-    function xml.remap_name(root, pattern, newtg, newns, newrn)
-        traverse(root, lpath(pattern), function(r,d,k)
-            local dk = d[k]
-            dk.tg = newtg
-            dk.ns = newns
-            dk.rn = newrn
-        end)
-    end
+xml.rename_space = rename_space
 
-    function xml.filters.found(root,pattern,check_content)
-        local found = false
-        traverse(root, lpath(pattern), function(r,d,k)
-            if check_content then
-                local dk = d and d[k]
-                found = dk and dk.dt and next(dk.dt) and true
-            else
-                found = true
-            end
-            return true
-        end)
-        return found
-    end
+function xml.remap_tag(root, pattern, newtg)
+    traverse(root, lpath(pattern), function(r,d,k)
+        d[k].tg = newtg
+    end)
+end
+function xml.remap_namespace(root, pattern, newns)
+    traverse(root, lpath(pattern), function(r,d,k)
+        d[k].ns = newns
+    end)
+end
+function xml.check_namespace(root, pattern, newns)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d[k]
+        if (not dk.rn or dk.rn == "") and dk.ns == "" then
+            dk.rn = newns
+        end
+    end)
+end
+function xml.remap_name(root, pattern, newtg, newns, newrn)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d[k]
+        dk.tg = newtg
+        dk.ns = newns
+        dk.rn = newrn
+    end)
+end
 
+function xml.filters.found(root,pattern,check_content)
+    local found = false
+    traverse(root, lpath(pattern), function(r,d,k)
+        if check_content then
+            local dk = d and d[k]
+            found = dk and dk.dt and next(dk.dt) and true
+        else
+            found = true
+        end
+        return true
+    end)
+    return found
 end
 
 --[[ldx--
@@ -4387,10 +4786,12 @@ put them here instead of loading mode modules there then needed.

 --ldx]]--
 
 function xml.gsub(t,old,new)
-    if t.dt then
-        for k,v in ipairs(t.dt) do
+    local dt = t.dt
+    if dt then
+        for k=1,#dt do
+            local v = dt[k]
             if type(v) == "string" then
-                t.dt[k] = v:gsub(old,new)
+                dt[k] = gsub(v,old,new)
             else
                 xml.gsub(v,old,new)
             end
@@ -4415,45 +4816,41 @@ end
 --~ xml.escapes   = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' }
 --~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end
 
---~ function xml.escaped  (str) return str:gsub("(.)"   , xml.escapes  ) end
---~ function xml.unescaped(str) return str:gsub("(&.-;)", xml.unescapes) end
---~ function xml.cleansed (str) return str:gsub("<.->"  , ''           ) end -- "%b<>"
+--~ function xml.escaped  (str) return (gsub(str,"(.)"   , xml.escapes  )) end
+--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end
+--~ function xml.cleansed (str) return (gsub(str,"<.->"  , ''           )) end -- "%b<>"
 
-do
+local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
 
-    local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
-
-    -- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
-    --
-    -- 1021:0335:0287:0247
-
-    -- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
-    --
-    -- 1559:0257:0288:0190 (last one suggested by roberto)
+-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
+--
+-- 1021:0335:0287:0247
 
-    --    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
-    --    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
-    local normal  = (1 - S("<&>"))^0
-    local special = P("<")/"<" + P(">")/">" + P("&")/"&"
-    local escaped = Cs(normal * (special * normal)^0)
+-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
+--
+-- 1559:0257:0288:0190 (last one suggested by roberto)
 
-    -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
+--    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
+--    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+local normal  = (1 - S("<&>"))^0
+local special = P("<")/"<" + P(">")/">" + P("&")/"&"
+local escaped = Cs(normal * (special * normal)^0)
 
-    --    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
-    --    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
-    local normal    = (1 - S"&")^0
-    local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
-    local unescaped = Cs(normal * (special * normal)^0)
+-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
 
-    -- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
+--    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+--    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
+local normal    = (1 - S"&")^0
+local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
+local unescaped = Cs(normal * (special * normal)^0)
 
-    local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
+-- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
 
-    function xml.escaped  (str) return escaped  :match(str) end
-    function xml.unescaped(str) return unescaped:match(str) end
-    function xml.cleansed (str) return cleansed :match(str) end
+local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
 
-end
+function xml.escaped  (str) return escaped  :match(str) end
+function xml.unescaped(str) return unescaped:match(str) end
+function xml.cleansed (str) return cleansed :match(str) end
 
 function xml.join(t,separator,lastseparator)
     if #t > 0 then
@@ -4471,155 +4868,46 @@ function xml.join(t,separator,lastseparator)
     end
 end
 
+function xml.statistics()
+    return {
+        lpathcalls = lpathcalls,
+        lpathcached = lpathcached,
+    }
+end
 
---[[ldx--
-
We provide (at least here) two entity handlers. The more extensive
-resolver consults a hash first, tries to convert to  next,
-and finaly calls a handler when defines. When this all fails, the
-original entity is returned.

---ldx]]--
+--  xml.set_text_cleanup(xml.show_text_entities)
+--  xml.set_text_cleanup(xml.resolve_text_entities)
 
-do if unicode and unicode.utf8 then
+--~ xml.lshow("/../../../a/(b|c)[@d='e']/f")
+--~ xml.lshow("/../../../a/!(b|c)[@d='e']/f")
+--~ xml.lshow("/../../../a/!b[@d!='e']/f")
 
-    xml.entities = xml.entities or { } -- xml.entity_handler == function
+--~ x = xml.convert([[
+--~     
+--~         01
+--~         02
+--~         03
+--~         OK
+--~         05
+--~         06
+--~         ALSO OK
+--~     
+--~ ]])
 
-    function xml.entity_handler(e)
-        return format("[%s]",e)
-    end
+--~ xml.settrace("lpath",true)
 
-    local char = unicode.utf8.char
+--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']"))
+--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]"))
+--~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']"))
+--~ xml.xshow(xml.all  (x,"b[number(@n)>2 and number(@n)<6]"))
+--~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]"))
 
-    local function toutf(s)
-        return char(tonumber(s,16))
-    end
-
-    function utfize(root)
-        local d = root.dt
-        for k=1,#d do
-            local dk = d[k]
-            if type(dk) == "string" then
-            --  test prevents copying if no match
-                if dk:find("&#x.-;") then
-                    d[k] = dk:gsub("&#x(.-);",toutf)
-                end
-            else
-                utfize(dk)
-            end
-        end
-    end
-
-    xml.utfize = utfize
-
-    local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks
-        if e:find("#x") then
-            return char(tonumber(e:sub(3),16))
-        else
-            local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded)
-            if ee then
-                return ee
-            else
-                local h = xml.entity_handler
-                return (h and h(e)) or "&" .. e .. ";"
-            end
-        end
-    end
-
-    local function resolve_entities(root)
-        if not root.special or root.tg == "@rt@" then
-            local d = root.dt
-            for k=1,#d do
-                local dk = d[k]
-                if type(dk) == "string" then
-                    if dk:find("&.-;") then
-                        d[k] = dk:gsub("&(.-);",resolve)
-                    end
-                else
-                    resolve_entities(dk)
-                end
-            end
-        end
-    end
-
-    xml.resolve_entities = resolve_entities
-
-    function xml.utfize_text(str)
-        if str:find("&#") then
-            return (str:gsub("&#x(.-);",toutf))
-        else
-            return str
-        end
-    end
-
-    function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline
-        if str:find("&") then
-            return (str:gsub("&(.-);",resolve))
-        else
-            return str
-        end
-    end
-
-    function xml.show_text_entities(str)
-        if str:find("&") then
-            return (str:gsub("&(.-);","[%1]"))
-        else
-            return str
-        end
-    end
-
-    -- experimental, this will be done differently
-
-    function xml.merge_entities(root)
-        local documententities = root.entities
-        local allentities = xml.entities
-        if documententities then
-            for k, v in pairs(documententities) do
-                allentities[k] = v
-            end
-        end
-    end
-
-end end
-
-function xml.statistics()
-    return {
-        lpathcalls = lpathcalls,
-        lpathcached = lpathcached,
-    }
-end
-
---  xml.set_text_cleanup(xml.show_text_entities)
---  xml.set_text_cleanup(xml.resolve_text_entities)
-
---~ xml.lshow("/../../../a/(b|c)[@d='e']/f")
---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f")
---~ xml.lshow("/../../../a/!b[@d!='e']/f")
-
---~ x = xml.convert([[
---~     
---~         01
---~         02
---~         03
---~         OK
---~         05
---~         06
---~         ALSO OK
---~     
---~ ]])
-
---~ xml.trace_lpath = true
-
---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']"))
---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]"))
---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']"))
---~ xml.xshow(xml.all  (x,"b[number(@n)>2 and number(@n)<6]"))
---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]"))
-
---~ str = [[
---~ 
---~ 
---~     my secret
---~ 
---~ ]]
+--~ str = [[
+--~ 
+--~ 
+--~     my secret
+--~ 
+--~ ]]
 
 --~ x = xml.convert([[
 --~     0102xx03OK
@@ -4633,177 +4921,522 @@ end
 --~ print(xml.filter(x,"b/tag(1)"))
 
 
--- filename : l-utils.lua
--- comment  : split off from luat-lib
--- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license  : see context related readme files
+end -- of closure
 
-if not versions then versions = { } end versions['l-utils'] = 1.001
+do -- create closure to overcome 200 locals limit
 
-if not utils        then utils        = { } end
-if not utils.merger then utils.merger = { } end
-if not utils.lua    then utils.lua    = { } end
+if not modules then modules = { } end modules ['lxml-ent'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-utils.merger.m_begin = "begin library merge"
-utils.merger.m_end   = "end library merge"
-utils.merger.pattern =
-    "%c+" ..
-    "%-%-%s+" .. utils.merger.m_begin ..
-    "%c+(.-)%c+" ..
-    "%-%-%s+" .. utils.merger.m_end ..
-    "%c+"
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, gsub, find = string.format, string.gsub, string.find
+local utfchar = unicode.utf8.char
 
-function utils.merger._self_fake_()
-    return
-        "-- " .. "created merged file" .. "\n\n" ..
-        "-- " .. utils.merger.m_begin  .. "\n\n" ..
-        "-- " .. utils.merger.m_end    .. "\n\n"
-end
+--[[ldx--
+
We provide (at least here) two entity handlers. The more extensive
+resolver consults a hash first, tries to convert to  next,
+and finaly calls a handler when defines. When this all fails, the
+original entity is returned.

+--ldx]]--
 
-function utils.report(...)
-    print(...)
-end
+xml.entities = xml.entities or { } -- xml.entity_handler == function
 
-utils.merger.strip_comment = true
+function xml.entity_handler(e)
+    return format("[%s]",e)
+end
 
-function utils.merger._self_load_(name)
-    local f, data = io.open(name), ""
-    if f then
-        utils.report("reading merge from %s",name)
-        data = f:read("*all")
-        f:close()
-    else
-        utils.report("unknown file to merge %s",name)
-    end
-    if data and utils.merger.strip_comment then
-        -- saves some 20K
-        data = data:gsub("%-%-~[^\n\r]*[\r\n]", "")
-    end
-    return data or ""
+local function toutf(s)
+    return utfchar(tonumber(s,16))
 end
 
-function utils.merger._self_save_(name, data)
-    if data ~= "" then
-        local f = io.open(name,'w')
-        if f then
-            utils.report("saving merge from %s",name)
-            f:write(data)
-            f:close()
+local function utfize(root)
+    local d = root.dt
+    for k=1,#d do
+        local dk = d[k]
+        if type(dk) == "string" then
+        --  test prevents copying if no match
+            if find(dk,"&#x.-;") then
+                d[k] = gsub(dk,"&#x(.-);",toutf)
+            end
+        else
+            utfize(dk)
         end
     end
 end
 
-function utils.merger._self_swap_(data,code)
-    if data ~= "" then
-        return (data:gsub(utils.merger.pattern, function(s)
-            return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n"
-        end, 1))
+xml.utfize = utfize
+
+local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks
+    if find(e,"^#x") then
+        return utfchar(tonumber(e:sub(3),16))
+    elseif find(e,"^#") then
+        return utfchar(tonumber(e:sub(2)))
     else
-        return ""
+        local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded)
+        if ee then
+            return ee
+        else
+            local h = xml.entity_handler
+            return (h and h(e)) or "&" .. e .. ";"
+        end
     end
 end
 
-function utils.merger._self_libs_(libs,list)
-    local result, f = { }, nil
-    if type(libs) == 'string' then libs = { libs } end
-    if type(list) == 'string' then list = { list } end
-    for _, lib in ipairs(libs) do
-        for _, pth in ipairs(list) do
-            local name = string.gsub(pth .. "/" .. lib,"\\","/")
-            f = io.open(name)
-            if f then
-                utils.report("merging library %s",name)
-                result[#result+1] = f:read("*all")
-                f:close()
-                list = { pth } -- speed up the search
-                break
+local function resolve_entities(root)
+    if not root.special or root.tg == "@rt@" then
+        local d = root.dt
+        for k=1,#d do
+            local dk = d[k]
+            if type(dk) == "string" then
+                if find(dk,"&.-;") then
+                    d[k] = gsub(dk,"&(.-);",resolve)
+                end
             else
-                utils.report("no library %s",name)
+                resolve_entities(dk)
             end
         end
     end
-    return table.concat(result, "\n\n")
 end
 
-function utils.merger.selfcreate(libs,list,target)
-    if target then
-        utils.merger._self_save_(
-            target,
-            utils.merger._self_swap_(
-                utils.merger._self_fake_(),
-                utils.merger._self_libs_(libs,list)
-            )
-        )
+xml.resolve_entities = resolve_entities
+
+function xml.utfize_text(str)
+    if find(str,"&#") then
+        return (gsub(str,"&#x(.-);",toutf))
+    else
+        return str
     end
 end
 
-function utils.merger.selfmerge(name,libs,list,target)
-    utils.merger._self_save_(
-        target or name,
-        utils.merger._self_swap_(
-            utils.merger._self_load_(name),
-            utils.merger._self_libs_(libs,list)
-        )
-    )
+function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline
+    if find(str,"&") then
+        return (gsub(str,"&(.-);",resolve))
+    else
+        return str
+    end
 end
 
-function utils.merger.selfclean(name)
-    utils.merger._self_save_(
-        name,
-        utils.merger._self_swap_(
-            utils.merger._self_load_(name),
-            ""
-        )
-    )
+function xml.show_text_entities(str)
+    if find(str,"&") then
+        return (gsub(str,"&(.-);","[%1]"))
+    else
+        return str
+    end
 end
 
-function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true
- -- utils.report("compiling",luafile,"into",lucfile)
-    os.remove(lucfile)
-    local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile)
-    if strip ~= false then
-        command = "-s " .. command
-    end
-    local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0)
-    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
-     -- utils.report("removing",luafile)
-        os.remove(luafile)
+-- experimental, this will be done differently
+
+function xml.merge_entities(root)
+    local documententities = root.entities
+    local allentities = xml.entities
+    if documententities then
+        for k, v in next, documententities do
+            allentities[k] = v
+        end
     end
-    return done
 end
 
 
+end -- of closure
 
-if not modules then modules = { } end modules ['luat-lib'] = {
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-mis'] = {
     version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
     author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
     copyright = "PRAGMA ADE / ConTeXt Development Team",
-    license   = "see context related readme files",
-    comment   = "companion to luat-lib.tex",
+    license   = "see context related readme files"
 }
 
--- most code already moved to the l-*.lua and other luat-*.lua files
+local concat = table.concat
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, gsub = string.format, string.gsub
 
-os.setlocale(nil,nil) -- useless feature and even dangerous in luatex
+--[[ldx--
+
The following helper functions best belong to the lmxl-ini
+module. Some are here because we need then in the mk
+document and other manuals, others came up when playing with
+this module. Since this module is also used in  we've
+put them here instead of loading mode modules there then needed.

+--ldx]]--
 
-function os.setlocale()
-    -- no way you can mess with it
+function xml.gsub(t,old,new)
+    local dt = t.dt
+    if dt then
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "string" then
+                dt[k] = gsub(v,old,new)
+            else
+                xml.gsub(v,old,new)
+            end
+        end
+    end
 end
 
-if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
-    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual
+    if d and k and d[k-1] and type(d[k-1]) == "string" then
+        local s = d[k-1]:match("\n(%s+)")
+        xml.gsub(dk,"\n"..string.rep(" ",#s),"\n")
+    end
 end
 
-environment             = environment or { }
-environment.arguments   = { }
-environment.files       = { }
-environment.sortedflags = nil
+function xml.serialize_path(root,lpath,handle)
+    local dk, r, d, k = xml.first(root,lpath)
+    dk = xml.copy(dk)
+    xml.strip_leading_spaces(dk,d,k)
+    xml.serialize(dk,handle)
+end
 
-function environment.initialize_arguments(arg)
-    local arguments, files = { }, { }
-    environment.arguments, environment.files, environment.sortedflags = arguments, files, nil
-    for index, argument in pairs(arg) do
+--~ xml.escapes   = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' }
+--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end
+
+--~ function xml.escaped  (str) return (gsub(str,"(.)"   , xml.escapes  )) end
+--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end
+--~ function xml.cleansed (str) return (gsub(str,"<.->"  , ''           )) end -- "%b<>"
+
+local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
+
+-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
+--
+-- 1021:0335:0287:0247
+
+-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
+--
+-- 1559:0257:0288:0190 (last one suggested by roberto)
+
+--    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
+--    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+local normal  = (1 - S("<&>"))^0
+local special = P("<")/"<" + P(">")/">" + P("&")/"&"
+local escaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
+
+--    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+--    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
+local normal    = (1 - S"&")^0
+local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
+local unescaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
+
+local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
+
+xml.escaped_pattern   = escaped
+xml.unescaped_pattern = unescaped
+xml.cleansed_pattern  = cleansed
+
+function xml.escaped  (str) return escaped  :match(str) end
+function xml.unescaped(str) return unescaped:match(str) end
+function xml.cleansed (str) return cleansed :match(str) end
+
+function xml.join(t,separator,lastseparator)
+    if #t > 0 then
+        local result = { }
+        for k,v in pairs(t) do
+            result[k] = xml.tostring(v)
+        end
+        if lastseparator then
+            return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result]
+        else
+            return concat(result,separator)
+        end
+    else
+        return ""
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-tra'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- the  tag is kind of generic and used for functions that are not
+-- bound to a variable, like node.new, node.copy etc (contrary to for instance
+-- node.has_attribute which is bound to a has_attribute local variable in mkiv)
+
+debugger = debugger or { }
+
+local counters = { }
+local names = { }
+local getinfo = debug.getinfo
+local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch
+
+-- one
+
+local function hook()
+    local f = getinfo(2,"f").func
+    local n = getinfo(2,"Sn")
+--  if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end
+    if f then
+        local cf = counters[f]
+        if cf == nil then
+            counters[f] = 1
+            names[f] = n
+        else
+            counters[f] = cf + 1
+        end
+    end
+end
+local function getname(func)
+    local n = names[func]
+    if n then
+        if n.what == "C" then
+            return n.name or ''
+        else
+            -- source short_src linedefined what name namewhat nups func
+            local name = n.name or n.namewhat or n.what
+            if not name or name == "" then name = "?" end
+            return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name)
+        end
+    else
+        return "unknown"
+    end
+end
+function debugger.showstats(printer,threshold)
+    printer   = printer or texio.write or print
+    threshold = threshold or 0
+    local total, grandtotal, functions = 0, 0, 0
+    printer("\n") -- ugly but ok
+ -- table.sort(counters)
+    for func, count in pairs(counters) do
+        if count > threshold then
+            local name = getname(func)
+            if not name:find("for generator") then
+                printer(format("%8i  %s", count, name))
+                total = total + count
+            end
+        end
+        grandtotal = grandtotal + count
+        functions = functions + 1
+    end
+    printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+end
+
+-- two
+
+--~ local function hook()
+--~     local n = getinfo(2)
+--~     if n.what=="C" and not n.name then
+--~         local f = tostring(debug.traceback())
+--~         local cf = counters[f]
+--~         if cf == nil then
+--~             counters[f] = 1
+--~             names[f] = n
+--~         else
+--~             counters[f] = cf + 1
+--~         end
+--~     end
+--~ end
+--~ function debugger.showstats(printer,threshold)
+--~     printer   = printer or texio.write or print
+--~     threshold = threshold or 0
+--~     local total, grandtotal, functions = 0, 0, 0
+--~     printer("\n") -- ugly but ok
+--~  -- table.sort(counters)
+--~     for func, count in pairs(counters) do
+--~         if count > threshold then
+--~             printer(format("%8i  %s", count, func))
+--~             total = total + count
+--~         end
+--~         grandtotal = grandtotal + count
+--~         functions = functions + 1
+--~     end
+--~     printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+--~ end
+
+-- rest
+
+function debugger.savestats(filename,threshold)
+    local f = io.open(filename,'w')
+    if f then
+        debugger.showstats(function(str) f:write(str) end,threshold)
+        f:close()
+    end
+end
+
+function debugger.enable()
+    debug.sethook(hook,"c")
+end
+
+function debugger.disable()
+    debug.sethook()
+--~ counters[debug.getinfo(2,"f").func] = nil
+end
+
+function debugger.tracing()
+    local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0
+    if n > 0 then
+        function debugger.tracing() return true  end ; return true
+    else
+        function debugger.tracing() return false end ; return false
+    end
+end
+
+--~ debugger.enable()
+
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+
+--~ debugger.disable()
+
+--~ print("")
+--~ debugger.showstats()
+--~ print("")
+--~ debugger.showstats(print,3)
+
+trackers = trackers or { }
+
+local data, done = { }, { }
+
+local function set(what,value)
+    for w in gmatch(lower(what),"[^, ]+") do
+        for d, f in next, data do
+            if done[d] then
+                -- prevent recursion due to wildcards
+            elseif find(d,w) then
+                done[d] = true
+                for i=1,#f do
+                    f[i](value)
+                end
+            end
+        end
+    end
+end
+
+local function reset()
+    for d, f in next, data do
+        for i=1,#f do
+            f[i](false)
+        end
+    end
+end
+
+function trackers.register(what,...)
+    what = lower(what)
+    local w = data[what]
+    if not w then
+        w = { }
+        data[what] = w
+    end
+    for _, fnc in next, { ... } do
+        local typ = type(fnc)
+        if typ == "function" then
+            w[#w+1] = fnc
+        elseif typ == "string" then
+            w[#w+1] = function(value) set(fnc,value,nesting) end
+        end
+    end
+end
+
+function trackers.enable(what)
+    done = { }
+    set(what,true)
+end
+
+function trackers.disable(what)
+    done = { }
+    if not what or what == "" then
+        trackers.reset(what)
+    else
+        set(what,false)
+    end
+end
+
+function trackers.reset(what)
+    done = { }
+    reset()
+end
+
+function trackers.list() -- pattern
+    local list = table.sortedkeys(data)
+    local user, system = { }, { }
+    for l=1,#list do
+        local what = list[l]
+        if find(what,"^%*") then
+            system[#system+1] = what
+        else
+            user[#user+1] = what
+        end
+    end
+    return user, system
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-env'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- A former version provided functionality for non embeded core
+-- scripts i.e. runtime library loading. Given the amount of
+-- Lua code we use now, this no longer makes sense. Much of this
+-- evolved before bytecode arrays were available and so a lot of
+-- code has disappeared already.
+
+local trace_verbose  = false  trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+
+local format = string.format
+
+-- precautions
+
+os.setlocale(nil,nil) -- useless feature and even dangerous in luatex
+
+function os.setlocale()
+    -- no way you can mess with it
+end
+
+-- dirty tricks
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then
+    profiler.start("luatex-profile.log")
+end
+
+-- environment
+
+environment             = environment or { }
+environment.arguments   = { }
+environment.files       = { }
+environment.sortedflags = nil
+
+if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end
+if not environment.version or environment.version == "" then             environment.version = "unknown"   end
+if not environment.jobname                              then             environment.jobname = "unknown"   end
+
+function environment.initialize_arguments(arg)
+    local arguments, files = { }, { }
+    environment.arguments, environment.files, environment.sortedflags = arguments, files, nil
+    for index, argument in pairs(arg) do
         if index > 0 then
             local flag, value = argument:match("^%-+(.+)=(.-)$")
             if flag then
@@ -4821,24 +5454,20 @@ function environment.initialize_arguments(arg)
     environment.ownname = environment.ownname or arg[0] or 'unknown.lua'
 end
 
-function environment.showarguments()
-    for k,v in pairs(environment.arguments) do
-        print(k .. " : " .. tostring(v))
-    end
-    if #environment.files > 0 then
-        print("files : " .. table.concat(environment.files, " "))
-    end
-end
-
 function environment.setargument(name,value)
     environment.arguments[name] = value
 end
 
-function environment.argument(name) -- todo: default (plus typecheck on default)
+-- todo: defaults, better checks e.g on type (boolean versus string)
+--
+-- tricky: too many hits when we support partials unless we add
+-- a registration of arguments so from now on we have 'partial'
+
+function environment.argument(name,partial)
     local arguments, sortedflags = environment.arguments, environment.sortedflags
     if arguments[name] then
         return arguments[name]
-    else
+    elseif partial then
         if not sortedflags then
             sortedflags = { }
             for _,v in pairs(table.sortedkeys(arguments)) do
@@ -4846,6 +5475,7 @@ function environment.argument(name) -- todo: default (plus typecheck on default)
             end
             environment.sortedflags = sortedflags
         end
+        -- example of potential clash: ^mode ^modefile
         for _,v in ipairs(sortedflags) do
             if name:find(v) then
                 return arguments[v:sub(2,#v)]
@@ -4869,43 +5499,17 @@ function environment.split_arguments(separator) -- rather special, cut-off befor
     return before, after
 end
 
---~ function environment.reconstruct_commandline(arg)
---~     if not arg then arg = environment.original_arguments end
---~     local result = { }
---~     for _,a in ipairs(arg) do -- ipairs 1 .. #n
---~         local kk, vv = a:match("^(%-+.-)=(.+)$")
---~         if kk and vv then
---~             if vv:find(" ") then
---~                 vv = vv:unquote()
---~                 vv = vv:gsub('"','\\"')
---~                 result[#result+1] = kk .. "=" .. vv:quote()
---~             else
---~                 a = a:unquote()
---~                 a = a:gsub('"','\\"')
---~                 result[#result+1] = a
---~             end
---~         elseif a:find(" ") then
---~             a = a:unquote()
---~             a = a:gsub('"','\\"')
---~             result[#result+1] = a:quote()
---~         else
---~             result[#result+1] = a
---~         end
---~     end
---~     return table.join(result," ")
---~ end
-
 function environment.reconstruct_commandline(arg,noquote)
-    if not arg then arg = environment.original_arguments end
+    arg = arg or environment.original_arguments
     if noquote and #arg == 1 then
         local a = arg[1]
-        a = input.resolve(a)
+        a = resolvers.resolve(a)
         a = a:unquote()
         return a
-    elseif #arg == 1 then
+    elseif next(arg) then
         local result = { }
         for _,a in ipairs(arg) do -- ipairs 1 .. #n
-            a = input.resolve(a)
+            a = resolvers.resolve(a)
             a = a:unquote()
             a = a:gsub('"','\\"') -- tricky
             if a:find(" ") then
@@ -4915,6 +5519,8 @@ function environment.reconstruct_commandline(arg,noquote)
             end
         end
         return table.join(result," ")
+    else
+        return ""
     end
 end
 
@@ -4950,279 +5556,163 @@ if arg then
 
 end
 
+-- weird place ... depends on a not yet loaded module
 
-if not modules then modules = { } end modules ['luat-inp'] = {
-    version   = 1.001,
-    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
-    copyright = "PRAGMA ADE / ConTeXt Development Team",
-    license   = "see context related readme files",
-    comment   = "companion to luat-lib.tex",
-}
-
--- TODO: os.getenv -> os.env[]
--- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
--- TODO: check escaping in find etc, too much, too slow
+function environment.texfile(filename)
+    return resolvers.find_file(filename,'tex')
+end
 
--- This lib is multi-purpose and can be loaded again later on so that
--- additional functionality becomes available. We will split this
--- module in components once we're done with prototyping. This is the
--- first code I wrote for LuaTeX, so it needs some cleanup. Before changing
--- something in this module one can best check with Taco or Hans first; there
--- is some nasty trickery going on that relates to traditional kpse support.
+function environment.luafile(filename)
+    local resolved = resolvers.find_file(filename,'tex') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    resolved = resolvers.find_file(filename,'texmfscripts') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    return resolvers.find_file(filename,'luatexlibs') or ""
+end
 
--- To be considered: hash key lowercase, first entry in table filename
--- (any case), rest paths (so no need for optimization). Or maybe a
--- separate table that matches lowercase names to mixed case when
--- present. In that case the lower() cases can go away. I will do that
--- only when we run into problems with names ... well ... Iwona-Regular.
+environment.loadedluacode = loadfile -- can be overloaded
 
--- Beware, loading and saving is overloaded in luat-tmp!
+--~ function environment.loadedluacode(name)
+--~     if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then
+--~         local chunk = loadstring(io.loaddata("texluac.luc"))
+--~         os.remove("texluac.luc")
+--~         return chunk
+--~     else
+--~         environment.loadedluacode = loadfile -- can be overloaded
+--~         return loadfile(name)
+--~     end
+--~ end
 
-if not input            then input            = { } end
-if not input.suffixes   then input.suffixes   = { } end
-if not input.formats    then input.formats    = { } end
-if not input.aux        then input.aux        = { } end
-
-if not input.suffixmap  then input.suffixmap  = { } end
-
-if not input.locators   then input.locators   = { } end  -- locate databases
-if not input.hashers    then input.hashers    = { } end  -- load databases
-if not input.generators then input.generators = { } end  -- generate databases
-if not input.filters    then input.filters    = { } end  -- conversion filters
-
-local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys
-
-input.locators.notfound   = { nil }
-input.hashers.notfound    = { nil }
-input.generators.notfound = { nil }
-
-input.cacheversion = '1.0.1'
-input.banner       = nil
-input.verbose      = false
-input.debug        = false
-input.cnfname      = 'texmf.cnf'
-input.luaname      = 'texmfcnf.lua'
-input.lsrname      = 'ls-R'
-input.homedir      = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~'
-
---~ input.luasuffix    = 'tma'
---~ input.lucsuffix    = 'tmc'
-
--- for the moment we have .local but this will disappear
-input.cnfdefault   = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
-
--- chances are low that the cnf file is in the bin path
-input.cnfdefault   = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
-
--- we use a cleaned up list / format=any is a wildcard, as is *name
-
-input.formats['afm'] = 'AFMFONTS'       input.suffixes['afm'] = { 'afm' }
-input.formats['enc'] = 'ENCFONTS'       input.suffixes['enc'] = { 'enc' }
-input.formats['fmt'] = 'TEXFORMATS'     input.suffixes['fmt'] = { 'fmt' }
-input.formats['map'] = 'TEXFONTMAPS'    input.suffixes['map'] = { 'map' }
-input.formats['mp']  = 'MPINPUTS'       input.suffixes['mp']  = { 'mp' }
-input.formats['ocp'] = 'OCPINPUTS'      input.suffixes['ocp'] = { 'ocp' }
-input.formats['ofm'] = 'OFMFONTS'       input.suffixes['ofm'] = { 'ofm', 'tfm' }
-input.formats['otf'] = 'OPENTYPEFONTS'  input.suffixes['otf'] = { 'otf' } -- 'ttf'
-input.formats['opl'] = 'OPLFONTS'       input.suffixes['opl'] = { 'opl' }
-input.formats['otp'] = 'OTPINPUTS'      input.suffixes['otp'] = { 'otp' }
-input.formats['ovf'] = 'OVFFONTS'       input.suffixes['ovf'] = { 'ovf', 'vf' }
-input.formats['ovp'] = 'OVPFONTS'       input.suffixes['ovp'] = { 'ovp' }
-input.formats['tex'] = 'TEXINPUTS'      input.suffixes['tex'] = { 'tex' }
-input.formats['tfm'] = 'TFMFONTS'       input.suffixes['tfm'] = { 'tfm' }
-input.formats['ttf'] = 'TTFONTS'        input.suffixes['ttf'] = { 'ttf', 'ttc' }
-input.formats['pfb'] = 'T1FONTS'        input.suffixes['pfb'] = { 'pfb', 'pfa' }
-input.formats['vf']  = 'VFFONTS'        input.suffixes['vf']  = { 'vf' }
-
-input.formats['fea'] = 'FONTFEATURES'   input.suffixes['fea'] = { 'fea' }
-input.formats['cid'] = 'FONTCIDMAPS'    input.suffixes['cid'] = { 'cid', 'cidmap' }
-
-input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new
-input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua'
-
-input.formats ['lua'] = 'LUAINPUTS' -- new
-input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' }
+function environment.luafilechunk(filename) -- used for loading lua bytecode in the format
+    filename = file.replacesuffix(filename, "lua")
+    local fullname = environment.luafile(filename)
+    if fullname and fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading file %s", fullname)
+        end
+        return environment.loadedluacode(fullname)
+    else
+        if trace_verbose then
+            logs.report("fileio","unknown file %s", filename)
+        end
+        return nil
+    end
+end
 
--- here we catch a few new thingies (todo: add these paths to context.tmf)
---
--- FONTFEATURES  = .;$TEXMF/fonts/fea//
--- FONTCIDMAPS   = .;$TEXMF/fonts/cid//
+-- the next ones can use the previous ones / combine
 
-function input.checkconfigdata() -- not yet ok, no time for debugging now
-    local instance = input.instance
-    local function fix(varname,default)
-        local proname = varname .. "." .. instance.progname or "crap"
-        local p = instance.environment[proname]
-        local v = instance.environment[varname]
-        if not ((p and p ~= "") or (v and v ~= "")) then
-            instance.variables[varname] = default -- or environment?
+function environment.loadluafile(filename, version)
+    local lucname, luaname, chunk
+    local basename = file.removesuffix(filename)
+    if basename == filename then
+        lucname, luaname = basename .. ".luc",  basename .. ".lua"
+    else
+        lucname, luaname = nil, basename -- forced suffix
+    end
+    -- when not overloaded by explicit suffix we look for a luc file first
+    local fullname = (lucname and environment.luafile(lucname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+    end
+    if chunk then
+        assert(chunk)()
+        if version then
+            -- we check of the version number of this chunk matches
+            local v = version -- can be nil
+            if modules and modules[filename] then
+                v = modules[filename].version -- new method
+            elseif versions and versions[filename] then
+                v = versions[filename]        -- old method
+            end
+            if v == version then
+                return true
+            else
+                if trace_verbose then
+                    logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version)
+                end
+                environment.loadluafile(filename)
+            end
+        else
+            return true
         end
     end
-    local name = os.name
-    if name == "windows" then
-        fix("OSFONTDIR", "c:/windows/fonts//")
-    elseif name == "macosx" then
-        fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//")
-    else
-        -- bad luck
+    fullname = (luaname and environment.luafile(luaname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+        if not chunk then
+            if verbose then
+                logs.report("fileio","unknown file %s", filename)
+            end
+        else
+            assert(chunk)()
+            return true
+        end
     end
-    fix("LUAINPUTS"   , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm
-    fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
-    fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    return false
 end
 
--- backward compatible ones
 
-input.alternatives = { }
+end -- of closure
 
-input.alternatives['map files']            = 'map'
-input.alternatives['enc files']            = 'enc'
-input.alternatives['cid files']            = 'cid'
-input.alternatives['fea files']            = 'fea'
-input.alternatives['opentype fonts']       = 'otf'
-input.alternatives['truetype fonts']       = 'ttf'
-input.alternatives['truetype collections'] = 'ttc'
-input.alternatives['type1 fonts']          = 'pfb'
+do -- create closure to overcome 200 locals limit
 
--- obscure ones
+if not modules then modules = { } end modules ['trac-inf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
 
-input.formats ['misc fonts'] = ''
-input.suffixes['misc fonts'] = { }
+local format = string.format
 
-input.formats     ['sfd']                      = 'SFDFONTS'
-input.suffixes    ['sfd']                      = { 'sfd' }
-input.alternatives['subfont definition files'] = 'sfd'
+local statusinfo, n, registered = { }, 0, { }
 
--- In practice we will work within one tds tree, but i want to keep
--- the option open to build tools that look at multiple trees, which is
--- why we keep the tree specific data in a table. We used to pass the
--- instance but for practical pusposes we now avoid this and use a
--- instance variable.
+statistics = statistics or { }
 
-function input.newinstance()
+statistics.enable    = true
+statistics.threshold = 0.05
 
-    local instance = { }
+-- timing functions
 
-    instance.rootpath        = ''
-    instance.treepath        = ''
-    instance.progname        = 'context'
-    instance.engine          = 'luatex'
-    instance.format          = ''
-    instance.environment     = { }
-    instance.variables       = { }
-    instance.expansions      = { }
-    instance.files           = { }
-    instance.remap           = { }
-    instance.configuration   = { }
-    instance.setup           = { }
-    instance.order           = { }
-    instance.found           = { }
-    instance.foundintrees    = { }
-    instance.kpsevars        = { }
-    instance.hashes          = { }
-    instance.cnffiles        = { }
-    instance.luafiles        = { }
-    instance.lists           = { }
-    instance.remember        = true
-    instance.diskcache       = true
-    instance.renewcache      = false
-    instance.scandisk        = true
-    instance.cachepath       = nil
-    instance.loaderror       = false
-    instance.smallcache      = false
-    instance.sortdata        = false
-    instance.savelists       = true
-    instance.cleanuppaths    = true
-    instance.allresults      = false
-    instance.pattern         = nil    -- lists
-    instance.kpseonly        = false  -- lists
-    instance.loadtime        = 0
-    instance.starttime       = 0
-    instance.stoptime        = 0
-    instance.validfile       = function(path,name) return true end
-    instance.data            = { } -- only for loading
-    instance.force_suffixes  = true
-    instance.dummy_path_expr = "^!*unset/*$"
-    instance.fakepaths       = { }
-    instance.lsrmode         = false
+local clock = os.gettimeofday or os.clock
 
-    -- store once, freeze and faster (once reset we can best use instance.environment)
+function statistics.hastimer(instance)
+    return instance and instance.starttime
+end
 
-    for k,v in pairs(os.env) do
-        instance.environment[k] = input.bare_variable(v)
-    end
-
-    -- cross referencing, delayed because we can add suffixes
-
-    for k, v in pairs(input.suffixes) do
-        for _, vv in pairs(v) do
-            if vv then
-                input.suffixmap[vv] = k
-            end
+function statistics.starttiming(instance)
+    if instance then
+        local it = instance.timing
+        if not it then
+            it = 0
         end
-    end
-
-    return instance
-
-end
-
-input.instance = input.instance or nil
-
-function input.reset()
-    input.instance = input.newinstance()
-    return input.instance
-end
-
-function input.reset_hashes()
-    input.instance.lists = { }
-    input.instance.found = { }
-end
-
-function input.bare_variable(str) -- assumes str is a string
- -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1")
-    return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2"))
-end
-
-function input.settrace(n)
-    input.trace = tonumber(n or 0)
-    if input.trace > 0 then
-        input.verbose = true
-    end
-end
-
-input.log  = (texio and texio.write_nl) or print
-
-function input.report(...)
-    if input.verbose then
-        input.log("<<"..format(...)..">>")
-    end
-end
-
-function input.report(...)
-    if input.trace > 0 then -- extra test
-        input.log("<<"..format(...)..">>")
-    end
-end
-
-input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0))
-
--- These functions can be used to test the performance, especially
--- loading the database files.
-
-do
-    local clock = os.gettimeofday or os.clock
-
-    function input.starttiming(instance)
-        if instance then
+        if it == 0 then
             instance.starttime = clock()
             if not instance.loadtime then
                 instance.loadtime = 0
             end
         end
+        instance.timing = it + 1
     end
+end
 
-    function input.stoptiming(instance, report)
-        if instance then
+function statistics.stoptiming(instance, report)
+    if instance then
+        local it = instance.timing
+        if it > 1 then
+            instance.timing = it - 1
+        else
             local starttime = instance.starttime
             if starttime then
                 local stoptime = clock()
@@ -5230,2170 +5720,2434 @@ do
                 instance.stoptime = stoptime
                 instance.loadtime = instance.loadtime + loadtime
                 if report then
-                    input.report("load time %0.3f",loadtime)
+                    statistics.report("load time %0.3f",loadtime)
                 end
+                instance.timing = 0
                 return loadtime
             end
         end
-        return 0
     end
-
+    return 0
 end
 
-function input.elapsedtime(instance)
+function statistics.elapsedtime(instance)
     return format("%0.3f",(instance and instance.loadtime) or 0)
 end
 
-function input.report_loadtime(instance)
-    if instance then
-        input.report('total load time %s', input.elapsedtime(instance))
-    end
+function statistics.elapsedindeed(instance)
+    local t = (instance and instance.loadtime) or 0
+    return t > statistics.threshold
 end
 
-input.loadtime = input.elapsedtime
+-- general function
 
-function input.env(key)
-    return input.instance.environment[key] or input.osenv(key)
+function statistics.register(tag,fnc)
+    if statistics.enable and type(fnc) == "function" then
+        local rt = registered[tag] or (#statusinfo + 1)
+        statusinfo[rt] = { tag, fnc }
+        registered[tag] = rt
+        if #tag > n then n = #tag end
+    end
 end
 
-function input.osenv(key)
-    local ie = input.instance.environment
-    local value = ie[key]
-    if value == nil then
-     -- local e = os.getenv(key)
-        local e = os.env[key]
-        if e == nil then
-         -- value = "" -- false
-        else
-            value = input.bare_variable(e)
+function statistics.show(reporter)
+    if statistics.enable then
+        if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end
+        -- this code will move
+        local register = statistics.register
+        register("luatex banner", function()
+            return string.lower(status.banner)
+        end)
+        register("control sequences", function()
+            return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra)
+        end)
+        register("callbacks", function()
+            local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
+            return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
+        end)
+        register("current memory usage", statistics.memused)
+        register("runtime",statistics.runtime)
+--         --
+        for i=1,#statusinfo do
+            local s = statusinfo[i]
+            local r = s[2]()
+            if r then
+                reporter(s[1],r,n)
+            end
         end
-        ie[key] = value
+        statistics.enable = false
     end
-    return value or ""
 end
 
--- we follow a rather traditional approach:
---
--- (1) texmf.cnf given in TEXMFCNF
--- (2) texmf.cnf searched in default variable
---
--- also we now follow the stupid route: if not set then just assume *one*
--- cnf file under texmf (i.e. distribution)
+function statistics.show_job_stat(tag,data,n)
+    texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data))
+end
 
-input.ownpath     = input.ownpath or nil
-input.ownbin      = input.ownbin  or arg[-2] or arg[-1] or arg[0] or "luatex"
-input.autoselfdir = true -- false may be handy for debugging
+function statistics.memused() -- no math.round yet -)
+    local round = math.round or math.floor
+    return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000))
+end
 
-function input.getownpath()
-    if not input.ownpath then
-        if input.autoselfdir and os.selfdir then
-            input.ownpath = os.selfdir
-        else
-            local binary = input.ownbin
-            if os.platform == "windows" then
-                binary = file.replacesuffix(binary,"exe")
-            end
-            for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
-                local b = file.join(p,binary)
-                if lfs.isfile(b) then
-                    -- we assume that after changing to the path the currentdir function
-                    -- resolves to the real location and use this side effect here; this
-                    -- trick is needed because on the mac installations use symlinks in the
-                    -- path instead of real locations
-                    local olddir = lfs.currentdir()
-                    if lfs.chdir(p) then
-                        local pp = lfs.currentdir()
-                        if input.verbose and p ~= pp then
-                            input.report("following symlink %s to %s",p,pp)
-                        end
-                        input.ownpath = pp
-                        lfs.chdir(olddir)
-                    else
-                        if input.verbose then
-                            input.report("unable to check path %s",p)
-                        end
-                        input.ownpath =  p
-                    end
-                    break
-                end
-            end
-        end
-        if not input.ownpath then input.ownpath = '.' end
-    end
-    return input.ownpath
+if statistics.runtime then
+    -- already loaded and set
+elseif luatex and luatex.starttime then
+    statistics.starttime = luatex.starttime
+    statistics.loadtime = 0
+    statistics.timing = 0
+else
+    statistics.starttiming(statistics)
 end
 
-function input.identify_own()
-    local instance = input.instance
-    local ownpath = input.getownpath() or lfs.currentdir()
-    local ie = instance.environment
-    if ownpath then
-        if input.env('SELFAUTOLOC')    == "" then os.env['SELFAUTOLOC']    = file.collapse_path(ownpath) end
-        if input.env('SELFAUTODIR')    == "" then os.env['SELFAUTODIR']    = file.collapse_path(ownpath .. "/..") end
-        if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end
-    else
-        input.verbose = true
-        input.report("error: unable to locate ownpath")
-        os.exit()
-    end
-    if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end
-    if input.env('TEXOS')    == "" then os.env['TEXOS']    = input.env('SELFAUTODIR') end
-    if input.env('TEXROOT')  == "" then os.env['TEXROOT']  = input.env('SELFAUTOPARENT') end
-    if input.verbose then
-        for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do
-            input.report("variable %s set to %s",v,input.env(v) or "unknown")
-        end
-    end
-    function input.identify_own() end
+function statistics.runtime()
+    statistics.stoptiming(statistics)
+    return statistics.formatruntime(statistics.elapsedtime(statistics))
 end
 
-function input.identify_cnf()
-    local instance = input.instance
-    if #instance.cnffiles == 0 then
-        -- fallback
-        input.identify_own()
-        -- the real search
-        input.expand_variables()
-        local t = input.split_path(input.env('TEXMFCNF'))
-        t = input.aux.expanded_path(t)
-        input.aux.expand_vars(t) -- redundant
-        local function locate(filename,list)
-            for _,v in ipairs(t) do
-                local texmfcnf = input.normalize_name(file.join(v,filename))
-                if lfs.isfile(texmfcnf) then
-                    table.insert(list,texmfcnf)
-                end
-            end
-        end
-        locate(input.luaname,instance.luafiles)
-        locate(input.cnfname,instance.cnffiles)
-    end
+function statistics.formatruntime(runtime)
+    return format("%s seconds", statistics.elapsedtime(statistics))
 end
 
-function input.load_cnf()
-    local instance = input.instance
-    local function loadoldconfigdata()
-        for _, fname in ipairs(instance.cnffiles) do
-            input.aux.load_cnf(fname)
-        end
-    end
-    -- instance.cnffiles contain complete names now !
-    if #instance.cnffiles == 0 then
-        input.report("no cnf files found (TEXMFCNF may not be set/known)")
-    else
-        instance.rootpath = instance.cnffiles[1]
-        for k,fname in ipairs(instance.cnffiles) do
-            instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/'))
-        end
-        for i=1,3 do
-            instance.rootpath = file.dirname(instance.rootpath)
-        end
-        instance.rootpath = input.normalize_name(instance.rootpath)
-        if instance.lsrmode then
-            loadoldconfigdata()
-        elseif instance.diskcache and not instance.renewcache then
-            input.loadoldconfig(instance.cnffiles)
-            if instance.loaderror then
-                loadoldconfigdata()
-                input.saveoldconfig()
-            end
-        else
-            loadoldconfigdata()
-            if instance.renewcache then
-                input.saveoldconfig()
-            end
-        end
-        input.aux.collapse_cnf_data()
+function statistics.timed(action,report)
+    local timer = { }
+    report = report or logs.simple
+    statistics.starttiming(timer)
+    action()
+    statistics.stoptiming(timer)
+    report("total runtime: %s",statistics.elapsedtime(timer))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-log'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this is old code that needs an overhaul
+
+local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+
+if texlua then
+    write_nl = print
+    write    = io.write
+end
+
+--[[ldx--
+
This is a prelude to a more extensive logging module. For the sake
+of parsing log files, in addition to the standard logging we will
+provide an  structured file. Actually, any logging that
+is hooked into callbacks will be \XML\ by default.

+--ldx]]--
+
+logs     = logs     or { }
+logs.xml = logs.xml or { }
+logs.tex = logs.tex or { }
+
+--[[ldx--
+
This looks pretty ugly but we need to speed things up a bit.

+--ldx]]--
+
+logs.moreinfo = [[
+more information about ConTeXt and the tools that come with it can be found at:
+
+maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
+webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
+wiki     : http://contextgarden.net
+]]
+
+logs.levels = {
+    ['error']   = 1,
+    ['warning'] = 2,
+    ['info']    = 3,
+    ['debug']   = 4,
+}
+
+logs.functions = {
+    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct',
+    'start_run', 'stop_run',
+    'start_page_number', 'stop_page_number',
+    'report_output_pages', 'report_output_log',
+    'report_tex_stat', 'report_job_stat',
+    'show_open', 'show_close', 'show_load',
+}
+
+logs.tracers = {
+}
+
+logs.level = 0
+logs.mode  = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"))
+
+function logs.set_level(level)
+    logs.level = logs.levels[level] or level
+end
+
+function logs.set_method(method)
+    for _, v in next, logs.functions do
+        logs[v] = logs[method][v] or function() end
     end
-    input.checkconfigdata()
 end
 
-function input.load_lua()
-    local instance = input.instance
-    if #instance.luafiles == 0 then
-        -- yet harmless
+-- tex logging
+
+function logs.tex.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(category .. " | " .. format(fmt,...))
     else
-        instance.rootpath = instance.luafiles[1]
-        for k,fname in ipairs(instance.luafiles) do
-            instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/'))
-        end
-        for i=1,3 do
-            instance.rootpath = file.dirname(instance.rootpath)
-        end
-        instance.rootpath = input.normalize_name(instance.rootpath)
-        input.loadnewconfig()
-        input.aux.collapse_cnf_data()
+        write_nl(category .. " |")
     end
-    input.checkconfigdata()
 end
 
-function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared)
-    local instance = input.instance
-    for _,c in ipairs(instance.order) do
-        for k,v in pairs(c) do
-            if not instance.variables[k] then
-                if instance.environment[k] then
-                    instance.variables[k] = instance.environment[k]
-                else
-                    instance.kpsevars[k] = true
-                    instance.variables[k] = input.bare_variable(v)
-                end
-            end
-        end
+function logs.tex.line(fmt,...) -- new
+    if fmt then
+        write_nl(format(fmt,...))
+    else
+        write_nl("")
     end
 end
 
-function input.aux.load_cnf(fname)
-    local instance = input.instance
-    fname = input.clean_path(fname)
-    local lname = file.replacesuffix(fname,'lua')
-    local f = io.open(lname)
-    if f then -- this will go
-        f:close()
-        local dname = file.dirname(fname)
-        if not instance.configuration[dname] then
-            input.aux.load_configuration(dname,lname)
-            instance.order[#instance.order+1] = instance.configuration[dname]
-        end
-    else
-        f = io.open(fname)
-        if f then
-            input.report("loading %s", fname)
-            local line, data, n, k, v
-            local dname = file.dirname(fname)
-            if not instance.configuration[dname] then
-                instance.configuration[dname] = { }
-                instance.order[#instance.order+1] = instance.configuration[dname]
-            end
-            local data = instance.configuration[dname]
-            while true do
-                local line, n = f:read(), 0
-                if line then
-                    while true do -- join lines
-                        line, n = line:gsub("\\%s*$", "")
-                        if n > 0 then
-                            line = line .. f:read()
-                        else
-                            break
-                        end
-                    end
-                    if not line:find("^[%%#]") then
-                        local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$")
-                        if k and v and not data[k] then
-                            data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME")
-                            instance.kpsevars[k] = true
-                        end
-                    end
-                else
-                    break
-                end
+local texcount = tex and tex.count
+
+function logs.tex.start_page_number()
+    local real, user, sub = texcount[0], texcount[1], texcount[2]
+    if real > 0 then
+        if user > 0 then
+            if sub > 0 then
+                write(format("[%s.%s.%s",real,user,sub))
+            else
+                write(format("[%s.%s",real,user))
             end
-            f:close()
         else
-            input.report("skipping %s", fname)
+            write(format("[%s",real))
         end
+    else
+        write("[-")
     end
 end
 
--- database loading
+function logs.tex.stop_page_number()
+    write("]")
+end
 
-function input.load_hash()
-    local instance = input.instance
-    input.locatelists()
-    if instance.lsrmode then
-        input.loadlists()
-    elseif instance.diskcache and not instance.renewcache then
-        input.loadfiles()
-        if instance.loaderror then
-            input.loadlists()
-            input.savefiles()
-        end
+logs.tex.report_job_stat = statistics.show_job_stat
+
+-- xml logging
+
+function logs.xml.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",category,format(fmt,...)))
     else
-        input.loadlists()
-        if instance.renewcache then
-            input.savefiles()
-        end
+        write_nl(format("",category))
     end
 end
-
-function input.aux.append_hash(type,tag,name)
-    if input.trace > 0 then
-        input.logger("= hash append: %s",tag)
+function logs.xml.line(fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",format(fmt,...)))
+    else
+        write_nl("")
     end
-    table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } )
 end
 
-function input.aux.prepend_hash(type,tag,name)
-    if input.trace > 0 then
-        input.logger("= hash prepend: %s",tag)
-    end
-    table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } )
+function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
+function logs.xml.stop () if logs.level > 0 then tw("") end end
+function logs.xml.push () if logs.level > 0 then tw("" ) end end
+
+function logs.xml.start_run()
+    write_nl("")
+    write_nl("") --  xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng'
+    write_nl("")
 end
 
-function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash
-    local instance = input.instance
---  local t = input.expanded_path_list('TEXMF') -- full expansion
-    local t = input.split_path(input.env('TEXMF'))
-    table.insert(t,1,specification)
-    local newspec = table.join(t,";")
-    if instance.environment["TEXMF"] then
-        instance.environment["TEXMF"] = newspec
-    elseif instance.variables["TEXMF"] then
-        instance.variables["TEXMF"] = newspec
-    else
-        -- weird
-    end
-    input.expand_variables()
-    input.reset_hashes()
+function logs.xml.stop_run()
+    write_nl("")
 end
 
--- locators
+function logs.xml.start_page_number()
+    write_nl(format("
")
+    write_nl("")
+end
+
+function logs.xml.report_output_pages(p,b)
+    write_nl(format("", p))
+    write_nl(format("", b))
+    write_nl("")
+end
+
+function logs.xml.report_output_log()
 end
 
-function input.locatedatabase(specification)
-    return input.methodhandler('locators', specification)
+function logs.xml.report_tex_stat(k,v)
+    texiowrite_nl("log",""..tostring(v).."")
 end
 
-function input.locators.tex(specification)
-    if specification and specification ~= '' and lfs.isdir(specification) then
-        if input.trace > 0 then
-            input.logger('! tex locator found: %s',specification)
-        end
-        input.aux.append_hash('file',specification,filename)
-    elseif input.trace > 0 then
-        input.logger('? tex locator not found: %s',specification)
+local level = 0
+
+function logs.xml.show_open(name)
+    level = level + 1
+    texiowrite_nl(format("",level,name))
+end
+
+function logs.xml.show_close(name)
+    texiowrite(" ")
+    level = level - 1
+end
+
+function logs.xml.show_load(name)
+    texiowrite_nl(format("",level+1,name))
+end
+
+--
+
+local name, banner = 'report', 'context'
+
+local function report(category,fmt,...)
+    if fmt then
+        write_nl(format("%s | %s: %s",name,category,format(fmt,...)))
+    elseif category then
+        write_nl(format("%s | %s",name,category))
+    else
+        write_nl(format("%s |",name))
     end
 end
 
--- hashers
+local function simple(fmt,...)
+    if fmt then
+        write_nl(format("%s | %s",name,format(fmt,...)))
+    else
+        write_nl(format("%s |",name))
+    end
+end
 
-function input.hashdatabase(tag,name)
-    return input.methodhandler('hashers',tag,name)
+function logs.setprogram(_name_,_banner_,_verbose_)
+    name, banner = _name_, _banner_
+    if _verbose_ then
+        trackers.enable("resolvers.verbose")
+    end
+    logs.set_method("tex")
+    logs.report = report -- also used in libraries
+    logs.simple = simple -- only used in scripts !
+    if utils then
+        utils.report = simple
+    end
+    logs.verbose = _verbose_
 end
 
-function input.loadfiles()
-    local instance = input.instance
-    instance.loaderror = false
-    instance.files = { }
-    if not instance.renewcache then
-        for _, hash in ipairs(instance.hashes) do
-            input.hashdatabase(hash.tag,hash.name)
-            if instance.loaderror then break end
-        end
+function logs.setverbose(what)
+    if what then
+        trackers.enable("resolvers.verbose")
+    else
+        trackers.disable("resolvers.verbose")
     end
+    logs.verbose = what or false
 end
 
-function input.hashers.tex(tag,name)
-    input.aux.load_files(tag)
+function logs.extendbanner(_banner_,_verbose_)
+    banner = banner .. " | ".. _banner_
+    if _verbose_ ~= nil then
+        logs.setverbose(what)
+    end
 end
 
--- generators:
+logs.verbose = false
+logs.report  = logs.tex.report
+logs.simple  = logs.tex.report
 
-function input.loadlists()
-    for _, hash in ipairs(input.instance.hashes) do
-        input.generatedatabase(hash.tag)
+function logs.reportlines(str) -- todo: 
+    for line in str:gmatch("(.-)[\n\r]") do
+        logs.report(line)
     end
 end
 
-function input.generatedatabase(specification)
-    return input.methodhandler('generators', specification)
+function logs.reportline() -- for scripts too
+    logs.report()
 end
 
-local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t"))
+logs.simpleline = logs.reportline
 
-function input.generators.tex(specification)
-    local instance = input.instance
-    local tag = specification
-    if not instance.lsrmode and lfs.dir then
-        input.report("scanning path %s",specification)
-        instance.files[tag] = { }
-        local files = instance.files[tag]
-        local n, m, r = 0, 0, 0
-        local spec = specification .. '/'
-        local attributes = lfs.attributes
-        local directory = lfs.dir
-        local small = instance.smallcache
-        local function action(path)
-            local mode, full
-            if path then
-                full = spec .. path .. '/'
-            else
-                full = spec
-            end
-            for name in directory(full) do
-                if name:find("^%.") then
-                  -- skip
-            --  elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped
-                elseif weird:match(name) then
-                  -- texio.write_nl("skipping " .. name)
-                  -- skip
-                else
-                    mode = attributes(full..name,'mode')
-                    if mode == 'directory' then
-                        m = m + 1
-                        if path then
-                            action(path..'/'..name)
-                        else
-                            action(name)
-                        end
-                    elseif path and mode == 'file' then
-                        n = n + 1
-                        local f = files[name]
-                        if f then
-                            if not small then
-                                if type(f) == 'string' then
-                                    files[name] = { f, path }
-                                else
-                                  f[#f+1] = path
-                                end
-                            end
-                        else
-                            files[name] = path
-                            local lower = name:lower()
-                            if name ~= lower then
-                                files["remap:"..lower] = name
-                                r = r + 1
-                            end
-                        end
-                    end
-                end
-            end
-        end
-        action()
-        input.report("%s files found on %s directories with %s uppercase remappings",n,m,r)
-    else
-        local fullname = file.join(specification,input.lsrname)
-        local path     = '.'
-        local f        = io.open(fullname)
+function logs.help(message,option)
+    logs.report(banner)
+    logs.reportline()
+    logs.reportlines(message)
+    local moreinfo = logs.moreinfo or ""
+    if moreinfo ~= "" and option ~= "nomoreinfo" then
+        logs.reportline()
+        logs.reportlines(moreinfo)
+    end
+end
+
+logs.set_level('error')
+logs.set_method('tex')
+
+function logs.system(whereto,process,jobname,category,...)
+    for i=1,10 do
+        local f = io.open(whereto,"a")
         if f then
-            instance.files[tag] = { }
-            local files = instance.files[tag]
-            local small = instance.smallcache
-            input.report("loading lsr file %s",fullname)
-        --  for line in f:lines() do -- much slower then the next one
-            for line in (f:read("*a")):gmatch("(.-)\n") do
-                if line:find("^[%a%d]") then
-                    local fl = files[line]
-                    if fl then
-                        if not small then
-                            if type(fl) == 'string' then
-                                files[line] = { fl, path } -- table
-                            else
-                                fl[#fl+1] = path
-                            end
-                        end
-                    else
-                        files[line] = path -- string
-                        local lower = line:lower()
-                        if line ~= lower then
-                            files["remap:"..lower] = line
-                        end
-                    end
-                else
-                    path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line
-                end
-            end
+            f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)))
             f:close()
+            break
+        else
+            sleep(0.1)
         end
     end
 end
 
--- savers, todo
+--~ local syslogname = "oeps.xxx"
+--~
+--~ for i=1,10 do
+--~     logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123")
+--~ end
 
-function input.savefiles()
-    input.aux.save_data('files', function(k,v)
-        return input.instance.validfile(k,v) -- path, name
-    end)
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+    comment   = "companion to luat-lib.tex",
+}
+
+-- After a few years using the code the large luat-inp.lua file
+-- has been split up a bit. In the process some functionality was
+-- dropped:
+--
+-- * support for reading lsr files
+-- * selective scanning (subtrees)
+-- * some public auxiliary functions were made private
+--
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
+-- This lib is multi-purpose and can be loaded again later on so that
+-- additional functionality becomes available. We will split thislogs.report("fileio",
+-- module in components once we're done with prototyping. This is the
+-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing
+-- something in this module one can best check with Taco or Hans first; there
+-- is some nasty trickery going on that relates to traditional kpse support.
+
+-- To be considered: hash key lowercase, first entry in table filename
+-- (any case), rest paths (so no need for optimization). Or maybe a
+-- separate table that matches lowercase names to mixed case when
+-- present. In that case the lower() cases can go away. I will do that
+-- only when we run into problems with names ... well ... Iwona-Regular.
+
+-- Beware, loading and saving is overloaded in luat-tmp!
+
+local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
+local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
+local next, type = next, type
+
+local trace_locating, trace_detail, trace_verbose  = false, false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+trackers.register("resolvers.detail",   function(v) trace_detail   = v trackers.enable("resolvers.verbose,resolvers.detail") end)
+
+if not resolvers then
+    resolvers = {
+        suffixes     = { },
+        formats      = { },
+        dangerous    = { },
+        suffixmap    = { },
+        alternatives = { },
+        locators     = { },  -- locate databases
+        hashers      = { },  -- load databases
+        generators   = { },  -- generate databases
+    }
 end
 
--- A config (optionally) has the paths split in tables. Internally
--- we join them and split them after the expansion has taken place. This
--- is more convenient.
+local resolvers = resolvers
+
+resolvers.locators  .notfound = { nil }
+resolvers.hashers   .notfound = { nil }
+resolvers.generators.notfound = { nil }
+
+resolvers.cacheversion = '1.0.1'
+resolvers.cnfname      = 'texmf.cnf'
+resolvers.luaname      = 'texmfcnf.lua'
+resolvers.homedir      = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~'
+resolvers.cnfdefault   = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
+
+local dummy_path_expr = "^!*unset/*$"
+
+local formats      = resolvers.formats
+local suffixes     = resolvers.suffixes
+local dangerous    = resolvers.dangerous
+local suffixmap    = resolvers.suffixmap
+local alternatives = resolvers.alternatives
+
+formats['afm'] = 'AFMFONTS'       suffixes['afm'] = { 'afm' }
+formats['enc'] = 'ENCFONTS'       suffixes['enc'] = { 'enc' }
+formats['fmt'] = 'TEXFORMATS'     suffixes['fmt'] = { 'fmt' }
+formats['map'] = 'TEXFONTMAPS'    suffixes['map'] = { 'map' }
+formats['mp']  = 'MPINPUTS'       suffixes['mp']  = { 'mp' }
+formats['ocp'] = 'OCPINPUTS'      suffixes['ocp'] = { 'ocp' }
+formats['ofm'] = 'OFMFONTS'       suffixes['ofm'] = { 'ofm', 'tfm' }
+formats['otf'] = 'OPENTYPEFONTS'  suffixes['otf'] = { 'otf' } -- 'ttf'
+formats['opl'] = 'OPLFONTS'       suffixes['opl'] = { 'opl' }
+formats['otp'] = 'OTPINPUTS'      suffixes['otp'] = { 'otp' }
+formats['ovf'] = 'OVFFONTS'       suffixes['ovf'] = { 'ovf', 'vf' }
+formats['ovp'] = 'OVPFONTS'       suffixes['ovp'] = { 'ovp' }
+formats['tex'] = 'TEXINPUTS'      suffixes['tex'] = { 'tex' }
+formats['tfm'] = 'TFMFONTS'       suffixes['tfm'] = { 'tfm' }
+formats['ttf'] = 'TTFONTS'        suffixes['ttf'] = { 'ttf', 'ttc' }
+formats['pfb'] = 'T1FONTS'        suffixes['pfb'] = { 'pfb', 'pfa' }
+formats['vf']  = 'VFFONTS'        suffixes['vf']  = { 'vf' }
+
+formats['fea'] = 'FONTFEATURES'   suffixes['fea'] = { 'fea' }
+formats['cid'] = 'FONTCIDMAPS'    suffixes['cid'] = { 'cid', 'cidmap' }
+
+formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new
+suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua'
+
+formats ['lua'] = 'LUAINPUTS' -- new
+suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' }
 
-function input.splitconfig()
-    for i,c in ipairs(input.instance) do
-        for k,v in pairs(c) do
-            if type(v) == 'string' then
-                local t = file.split_path(v)
-                if #t > 1 then
-                    c[k] = t
-                end
-            end
-        end
+-- backward compatible ones
+
+alternatives['map files']            = 'map'
+alternatives['enc files']            = 'enc'
+alternatives['cid files']            = 'cid'
+alternatives['fea files']            = 'fea'
+alternatives['opentype fonts']       = 'otf'
+alternatives['truetype fonts']       = 'ttf'
+alternatives['truetype collections'] = 'ttc'
+alternatives['type1 fonts']          = 'pfb'
+
+-- obscure ones
+
+formats ['misc fonts'] = ''
+suffixes['misc fonts'] = { }
+
+formats     ['sfd']                      = 'SFDFONTS'
+suffixes    ['sfd']                      = { 'sfd' }
+alternatives['subfont definition files'] = 'sfd'
+
+-- In practice we will work within one tds tree, but i want to keep
+-- the option open to build tools that look at multiple trees, which is
+-- why we keep the tree specific data in a table. We used to pass the
+-- instance but for practical pusposes we now avoid this and use a
+-- instance variable.
+
+-- here we catch a few new thingies (todo: add these paths to context.tmf)
+--
+-- FONTFEATURES  = .;$TEXMF/fonts/fea//
+-- FONTCIDMAPS   = .;$TEXMF/fonts/cid//
+
+-- we always have one instance active
+
+resolvers.instance = resolvers.instance or nil -- the current one (slow access)
+local instance = resolvers.instance or nil -- the current one (fast access)
+
+function resolvers.newinstance()
+
+    -- store once, freeze and faster (once reset we can best use
+    -- instance.environment) maybe better have a register suffix
+    -- function
+
+    for k, v in next, suffixes do
+        for i=1,#v do
+            local vi = v[i]
+            if vi then
+                suffixmap[vi] = k
+            end
+        end
+    end
+
+    -- because vf searching is somewhat dangerous, we want to prevent
+    -- too liberal searching esp because we do a lookup on the current
+    -- path anyway; only tex (or any) is safe
+
+    for k, v in next, formats do
+        dangerous[k] = true
+    end
+    dangerous.tex = nil
+
+    -- the instance
+
+    local newinstance = {
+        rootpath        = '',
+        treepath        = '',
+        progname        = 'context',
+        engine          = 'luatex',
+        format          = '',
+        environment     = { },
+        variables       = { },
+        expansions      = { },
+        files           = { },
+        remap           = { },
+        configuration   = { },
+        setup           = { },
+        order           = { },
+        found           = { },
+        foundintrees    = { },
+        kpsevars        = { },
+        hashes          = { },
+        cnffiles        = { },
+        luafiles        = { },
+        lists           = { },
+        remember        = true,
+        diskcache       = true,
+        renewcache      = false,
+        scandisk        = true,
+        cachepath       = nil,
+        loaderror       = false,
+        sortdata        = false,
+        savelists       = true,
+        cleanuppaths    = true,
+        allresults      = false,
+        pattern         = nil, -- lists
+        data            = { }, -- only for loading
+        force_suffixes  = true,
+        fakepaths       = { },
+    }
+
+    local ne = newinstance.environment
+
+    for k,v in next, os.env do
+        ne[k] = resolvers.bare_variable(v)
     end
+
+    return newinstance
+
 end
 
-function input.joinconfig()
-    for i,c in ipairs(input.instance.order) do
-        for k,v in pairs(c) do
-            if type(v) == 'table' then
-                c[k] = file.join_path(v)
-            end
+function resolvers.setinstance(someinstance)
+    instance = someinstance
+    resolvers.instance = someinstance
+    return someinstance
+end
+
+function resolvers.reset()
+    return resolvers.setinstance(resolvers.newinstance())
+end
+
+local function reset_hashes()
+    instance.lists = { }
+    instance.found = { }
+end
+
+local function check_configuration() -- not yet ok, no time for debugging now
+    local ie = instance.environment
+    local function fix(varname,default)
+        local proname = varname .. "." .. instance.progname or "crap"
+        local p, v = ie[proname], ie[varname]
+        if not ((p and p ~= "") or (v and v ~= "")) then
+            instance.variables[varname] = default -- or environment?
         end
     end
-end
-function input.split_path(str)
-    if type(str) == 'table' then
-        return str
+    local name = os.name
+    if name == "windows" then
+        fix("OSFONTDIR", "c:/windows/fonts//")
+    elseif name == "macosx" then
+        fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//")
     else
-        return file.split_path(str)
+        -- bad luck
     end
+    fix("LUAINPUTS"   , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm
+    fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("LUATEXLIBS"  , ".;$TEXMF/luatex/lua//")
 end
-function input.join_path(str)
-    if type(str) == 'table' then
-        return file.join_path(str)
-    else
-        return str
+
+function resolvers.bare_variable(str) -- assumes str is a string
+    return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2"))
+end
+
+function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail'
+    if n then
+        trackers.disable("resolvers.*")
+        trackers.enable("resolvers."..n)
     end
 end
 
-function input.splitexpansions()
-    local ie = input.instance.expansions
-    for k,v in pairs(ie) do
-        local t, h = { }, { }
-        for _,vv in pairs(file.split_path(v)) do
-            if vv ~= "" and not h[vv] then
-                t[#t+1] = vv
-                h[vv] = true
-            end
-        end
-        if #t > 1 then
-            ie[k] = t
+resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE"))
+
+function resolvers.osenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
+        if e == nil then
+         -- value = "" -- false
         else
-            ie[k] = t[1]
+            value = resolvers.bare_variable(e)
         end
+        ie[key] = value
     end
+    return value or ""
 end
 
--- end of split/join code
+function resolvers.env(key)
+    return instance.environment[key] or resolvers.osenv(key)
+end
 
-function input.saveoldconfig()
-    input.splitconfig()
-    input.aux.save_data('configuration', nil)
-    input.joinconfig()
+--
+
+local function expand_vars(lst) -- simple vars
+    local variables, env = instance.variables, resolvers.env
+    local function resolve(a)
+        return variables[a] or env(a)
+    end
+    for k=1,#lst do
+        lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve)
+    end
 end
 
-input.configbanner = [[
--- This is a Luatex configuration file created by 'luatools.lua' or
--- 'luatex.exe' directly. For comment, suggestions and questions you can
--- contact the ConTeXt Development Team. This configuration file is
--- not copyrighted. [HH & TH]
-]]
+local function expanded_var(var) -- simple vars
+    local function resolve(a)
+        return instance.variables[a] or resolvers.env(a)
+    end
+    return (gsub(var,"%$([%a%d%_%-]+)",resolve))
+end
 
-function input.serialize(files)
-    -- This version is somewhat optimized for the kind of
-    -- tables that we deal with, so it's much faster than
-    -- the generic serializer. This makes sense because
-    -- luatools and mtxtools are called frequently. Okay,
-    -- we pay a small price for properly tabbed tables.
-    local t = { }
-    local function dump(k,v,m)
-        if type(v) == 'string' then
-            return m .. "['" .. k .. "']='" .. v .. "',"
-        elseif #v == 1 then
-            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+local function entry(entries,name)
+    if name and (name ~= "") then
+        name = gsub(name,'%$','')
+        local result = entries[name..'.'..instance.progname] or entries[name]
+        if result then
+            return result
         else
-            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+            result = resolvers.env(name)
+            if result then
+                instance.variables[name] = result
+                resolvers.expand_variables()
+                return instance.expansions[name] or ""
+            end
         end
     end
-    t[#t+1] = "return {"
-    if input.instance.sortdata then
-        for _, k in pairs(sortedkeys(files)) do
-            local fk  = files[k]
-            if type(fk) == 'table' then
-                t[#t+1] = "\t['" .. k .. "']={"
-                for _, kk in pairs(sortedkeys(fk)) do
-                    t[#t+1] = dump(kk,fk[kk],"\t\t")
-                end
-                t[#t+1] = "\t},"
-            else
-                t[#t+1] = dump(k,fk,"\t")
-            end
-        end
+    return ""
+end
+
+local function is_entry(entries,name)
+    if name and name ~= "" then
+        name = gsub(name,'%$','')
+        return (entries[name..'.'..instance.progname] or entries[name]) ~= nil
     else
-        for k, v in pairs(files) do
-            if type(v) == 'table' then
-                t[#t+1] = "\t['" .. k .. "']={"
-                for kk,vv in pairs(v) do
-                    t[#t+1] = dump(kk,vv,"\t\t")
-                end
-                t[#t+1] = "\t},"
-            else
-                t[#t+1] = dump(k,v,"\t")
-            end
-        end
+        return false
     end
-    t[#t+1] = "}"
-    return concat(t,"\n")
 end
 
-if not texmf then texmf = {} end -- no longer needed, at least not here
+-- {a,b,c,d}
+-- a,b,c/{p,q,r},d
+-- a,b,c/{p,q,r}/d/{x,y,z}//
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a{b,c}{d,e}f
+-- {a,b,c,d}
+-- {a,b,c/{p,q,r},d}
+-- {a,b,c/{p,q,r}/d/{x,y,z}//}
+-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
+-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
+-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
+
+-- this one is better and faster, but it took me a while to realize
+-- that this kind of replacement is cleaner than messy parsing and
+-- fuzzy concatenating we can probably gain a bit with selectively
+-- applying lpeg, but experiments with lpeg parsing this proved not to
+-- work that well; the parsing is ok, but dealing with the resulting
+-- table is a pain because we need to work inside-out recursively
 
-function input.aux.save_data(dataname, check, makename) -- untested without cache overload
-    for cachename, files in pairs(input.instance[dataname]) do
-        local name = (makename or file.join)(cachename,dataname)
-        local luaname, lucname = name .. ".lua", name .. ".luc"
-        input.report("preparing %s for %s",dataname,cachename)
-        for k, v in pairs(files) do
-            if not check or check(v,k) then -- path, name
-                if type(v) == "table" and #v == 1 then
-                    files[k] = v[1]
-                end
-            else
-                files[k] = nil -- false
+local function splitpathexpr(str, t, validate)
+    -- no need for further optimization as it is only called a
+    -- few times, we can use lpeg for the sub; we could move
+    -- the local functions outside the body
+    t = t or { }
+    str = gsub(str,",}",",@}")
+    str = gsub(str,"{,","{@,")
+ -- str = "@" .. str .. "@"
+    local ok, done
+    local function do_first(a,b)
+        local t = { }
+        for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_second(a,b)
+        local t = { }
+        for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_both(a,b)
+        local t = { }
+        for sa in gmatch(a,"[^,]+") do
+            for sb in gmatch(b,"[^,]+") do
+                t[#t+1] = sa .. sb
             end
         end
-        local data = {
-            type    = dataname,
-            root    = cachename,
-            version = input.cacheversion,
-            date    = os.date("%Y-%m-%d"),
-            time    = os.date("%H:%M:%S"),
-            content = files,
-        }
-        local ok = io.savedata(luaname,input.serialize(data))
-        if ok then
-            input.report("%s saved in %s",dataname,luaname)
-            if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip
-                input.report("%s compiled to %s",dataname,lucname)
-            else
-                input.report("compiling failed for %s, deleting file %s",dataname,lucname)
-                os.remove(lucname)
-            end
-        else
-            input.report("unable to save %s in %s (access error)",dataname,luaname)
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_three(a,b,c)
+        return a .. b.. c
+    end
+    while true do
+        done = false
+        while true do
+            str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both)
+            if ok > 0 then done = true else break end
+        end
+        str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three)
+        if ok > 0 then done = true end
+        if not done then break end
+    end
+    str = gsub(str,"[{}]", "")
+    str = gsub(str,"@","")
+    if validate then
+        for s in gmatch(str,"[^,]+") do
+            s = validate(s)
+            if s then t[#t+1] = s end
+        end
+    else
+        for s in gmatch(str,"[^,]+") do
+            t[#t+1] = s
         end
     end
+    return t
 end
 
-function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload
-    local instance = input.instance
-    filename = ((not filename or (filename == "")) and dataname) or filename
-    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
-    local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua")
-    if blob then
-        local data = blob()
-        if data and data.content and data.type == dataname and data.version == input.cacheversion then
-            input.report("loading %s for %s from %s",dataname,pathname,filename)
-            instance[dataname][pathname] = data.content
-        else
-            input.report("skipping %s for %s from %s",dataname,pathname,filename)
-            instance[dataname][pathname] = { }
-            instance.loaderror = true
+local function expanded_path_from_list(pathlist) -- maybe not a list, just a path
+    -- a previous version fed back into pathlist
+    local newlist, ok = { }, false
+    for k=1,#pathlist do
+        if find(pathlist[k],"[{}]") then
+            ok = true
+            break
+        end
+    end
+    if ok then
+        local function validate(s)
+            s = file.collapse_path(s)
+            return s ~= "" and not find(s,dummy_path_expr) and s
+        end
+        for k=1,#pathlist do
+            splitpathexpr(pathlist[k],newlist,validate)
         end
     else
-        input.report("skipping %s for %s from %s",dataname,pathname,filename)
+        for k=1,#pathlist do
+            for p in gmatch(pathlist[k],"([^,]+)") do
+                p = file.collapse_path(p)
+                if p ~= "" then newlist[#newlist+1] = p end
+            end
+        end
     end
+    return newlist
 end
 
--- some day i'll use the nested approach, but not yet (actually we even drop
--- engine/progname support since we have only luatex now)
+-- we follow a rather traditional approach:
 --
--- first texmfcnf.lua files are located, next the cached texmf.cnf files
+-- (1) texmf.cnf given in TEXMFCNF
+-- (2) texmf.cnf searched in default variable
 --
--- return {
---     TEXMFBOGUS = 'effe checken of dit werkt',
--- }
+-- also we now follow the stupid route: if not set then just assume *one*
+-- cnf file under texmf (i.e. distribution)
 
-function input.aux.load_texmfcnf(dataname,pathname)
-    local instance = input.instance
-    local filename = file.join(pathname,input.luaname)
-    local blob = loadfile(filename)
-    if blob then
-        local data = blob()
-        if data then
-            input.report("loading configuration file %s",filename)
-            if true then
-                -- flatten to variable.progname
-                local t = { }
-                for k, v in pairs(data) do -- v = progname
-                    if type(v) == "string" then
-                        t[k] = v
+resolvers.ownpath     = resolvers.ownpath or nil
+resolvers.ownbin      = resolvers.ownbin  or arg[-2] or arg[-1] or arg[0] or "luatex"
+resolvers.autoselfdir = true -- false may be handy for debugging
+
+function resolvers.getownpath()
+    if not resolvers.ownpath then
+        if resolvers.autoselfdir and os.selfdir then
+            resolvers.ownpath = os.selfdir
+        else
+            local binary = resolvers.ownbin
+            if os.platform == "windows" then
+                binary = file.replacesuffix(binary,"exe")
+            end
+            for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+                local b = file.join(p,binary)
+                if lfs.isfile(b) then
+                    -- we assume that after changing to the path the currentdir function
+                    -- resolves to the real location and use this side effect here; this
+                    -- trick is needed because on the mac installations use symlinks in the
+                    -- path instead of real locations
+                    local olddir = lfs.currentdir()
+                    if lfs.chdir(p) then
+                        local pp = lfs.currentdir()
+                        if trace_verbose and p ~= pp then
+                            logs.report("fileio","following symlink %s to %s",p,pp)
+                        end
+                        resolvers.ownpath = pp
+                        lfs.chdir(olddir)
                     else
-                        for kk, vv in pairs(v) do -- vv = variable
-                            if type(vv) == "string" then
-                                t[vv.."."..v] = kk
-                            end
+                        if trace_verbose then
+                            logs.report("fileio","unable to check path %s",p)
                         end
+                        resolvers.ownpath =  p
                     end
+                    break
                 end
-                instance[dataname][pathname] = t
-            else
-                instance[dataname][pathname] = data
             end
-        else
-            input.report("skipping configuration file %s",filename)
-            instance[dataname][pathname] = { }
-            instance.loaderror = true
         end
-    else
-        input.report("skipping configuration file %s",filename)
+        if not resolvers.ownpath then resolvers.ownpath = '.' end
     end
+    return resolvers.ownpath
 end
 
-function input.aux.load_configuration(dname,lname)
-    input.aux.load_data(dname,'configuration',lname and file.basename(lname))
-end
-function input.aux.load_files(tag)
-    input.aux.load_data(tag,'files')
-end
-
-function input.resetconfig()
-    input.identify_own()
-    local instance = input.instance
-    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
-end
+local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" }
 
-function input.loadnewconfig()
-    local instance = input.instance
-    for _, cnf in ipairs(instance.luafiles) do
-        local dname = file.dirname(cnf)
-        input.aux.load_texmfcnf('setup',dname)
-        instance.order[#instance.order+1] = instance.setup[dname]
-        if instance.loaderror then break end
+local function identify_own()
+    local ownpath = resolvers.getownpath() or lfs.currentdir()
+    local ie = instance.environment
+    if ownpath then
+        if resolvers.env('SELFAUTOLOC')    == "" then os.env['SELFAUTOLOC']    = file.collapse_path(ownpath) end
+        if resolvers.env('SELFAUTODIR')    == "" then os.env['SELFAUTODIR']    = file.collapse_path(ownpath .. "/..") end
+        if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end
+    else
+        logs.report("fileio","error: unable to locate ownpath")
+        os.exit()
     end
-end
-
-function input.loadoldconfig()
-    local instance = input.instance
-    if not instance.renewcache then
-        for _, cnf in ipairs(instance.cnffiles) do
-            local dname = file.dirname(cnf)
-            input.aux.load_configuration(dname)
-            instance.order[#instance.order+1] = instance.configuration[dname]
-            if instance.loaderror then break end
+    if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end
+    if resolvers.env('TEXOS')    == "" then os.env['TEXOS']    = resolvers.env('SELFAUTODIR') end
+    if resolvers.env('TEXROOT')  == "" then os.env['TEXROOT']  = resolvers.env('SELFAUTOPARENT') end
+    if trace_verbose then
+        for i=1,#own_places do
+            local v = own_places[i]
+            logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown")
         end
     end
-    input.joinconfig()
+    identify_own = function() end
 end
 
-function input.expand_variables()
-    local instance = input.instance
-    local expansions, environment, variables = { }, instance.environment, instance.variables
-    local env = input.env
-    instance.expansions = expansions
-    if instance.engine   ~= "" then environment['engine']   = instance.engine   end
-    if instance.progname ~= "" then environment['progname'] = instance.progname end
-    for k,v in pairs(environment) do
-        local a, b = k:match("^(%a+)%_(.*)%s*$")
-        if a and b then
-            expansions[a..'.'..b] = v
-        else
-            expansions[k] = v
-        end
-    end
-    for k,v in pairs(environment) do -- move environment to expansions
-        if not expansions[k] then expansions[k] = v end
-    end
-    for k,v in pairs(variables) do -- move variables to expansions
-        if not expansions[k] then expansions[k] = v end
-    end
-    while true do
-        local busy = false
-        for k,v in pairs(expansions) do
-            local s, n = v:gsub("%$([%a%d%_%-]+)", function(a)
-                busy = true
-                return expansions[a] or env(a)
-            end)
-            local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a)
-                busy = true
-                return expansions[a] or env(a)
-            end)
-            if n > 0 or m > 0 then
-                expansions[k]= s
+function resolvers.identify_cnf()
+    if #instance.cnffiles == 0 then
+        -- fallback
+        identify_own()
+        -- the real search
+        resolvers.expand_variables()
+        local t = resolvers.split_path(resolvers.env('TEXMFCNF'))
+        t = expanded_path_from_list(t)
+        expand_vars(t) -- redundant
+        local function locate(filename,list)
+            for i=1,#t do
+                local ti = t[i]
+                local texmfcnf = file.collapse_path(file.join(ti,filename))
+                if lfs.isfile(texmfcnf) then
+                    list[#list+1] = texmfcnf
+                end
             end
         end
-        if not busy then break end
-    end
-    for k,v in pairs(expansions) do
-        expansions[k] = v:gsub("\\", '/')
+        locate(resolvers.luaname,instance.luafiles)
+        locate(resolvers.cnfname,instance.cnffiles)
     end
 end
 
-function input.aux.expand_vars(lst) -- simple vars
-    local instance = input.instance
-    local variables, env = instance.variables, input.env
-    for k,v in pairs(lst) do
-        lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a)
-            return variables[a] or env(a)
-        end)
+local function load_cnf_file(fname)
+    fname = resolvers.clean_path(fname)
+    local lname = file.replacesuffix(fname,'lua')
+    local f = io.open(lname)
+    if f then -- this will go
+        f:close()
+        local dname = file.dirname(fname)
+        if not instance.configuration[dname] then
+            resolvers.load_data(dname,'configuration',lname and file.basename(lname))
+            instance.order[#instance.order+1] = instance.configuration[dname]
+        end
+    else
+        f = io.open(fname)
+        if f then
+            if trace_verbose then
+                logs.report("fileio","loading %s", fname)
+            end
+            local line, data, n, k, v
+            local dname = file.dirname(fname)
+            if not instance.configuration[dname] then
+                instance.configuration[dname] = { }
+                instance.order[#instance.order+1] = instance.configuration[dname]
+            end
+            local data = instance.configuration[dname]
+            while true do
+                local line, n = f:read(), 0
+                if line then
+                    while true do -- join lines
+                        line, n = gsub(line,"\\%s*$", "")
+                        if n > 0 then
+                            line = line .. f:read()
+                        else
+                            break
+                        end
+                    end
+                    if not find(line,"^[%%#]") then
+                        local l = gsub(line,"%s*%%.*$","")
+                        local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$")
+                        if k and v and not data[k] then
+                            v = gsub(v,"[%%#].*",'')
+                            data[k] = gsub(v,"~","$HOME")
+                            instance.kpsevars[k] = true
+                        end
+                    end
+                else
+                    break
+                end
+            end
+            f:close()
+        elseif trace_verbose then
+            logs.report("fileio","skipping %s", fname)
+        end
     end
 end
 
-function input.aux.expanded_var(var) -- simple vars
-    local instance = input.instance
-    return var:gsub("%$([%a%d%_%-]+)", function(a)
-        return instance.variables[a] or input.env(a)
-    end)
-end
-
-function input.aux.entry(entries,name)
-    if name and (name ~= "") then
-        local instance = input.instance
-        name = name:gsub('%$','')
-        local result = entries[name..'.'..instance.progname] or entries[name]
-        if result then
-            return result
-        else
-            result = input.env(name)
-            if result then
-                instance.variables[name] = result
-                input.expand_variables()
-                return instance.expansions[name] or ""
+local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in next, c do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = resolvers.bare_variable(v)
+                end
             end
         end
     end
-    return ""
-end
-function input.variable(name)
-    return input.aux.entry(input.instance.variables,name)
-end
-function input.expansion(name)
-    return input.aux.entry(input.instance.expansions,name)
 end
 
-function input.aux.is_entry(entries,name)
-    if name and name ~= "" then
-        name = name:gsub('%$','')
-        return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil
-    else
-        return false
+function resolvers.load_cnf()
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            load_cnf_file(fname)
+        end
     end
-end
-
-function input.is_variable(name)
-    return input.aux.is_entry(input.instance.variables,name)
-end
-
-function input.is_expansion(name)
-    return input.aux.is_entry(input.instance.expansions,name)
-end
-
-function input.unexpanded_path_list(str)
-    local pth = input.variable(str)
-    local lst = input.split_path(pth)
-    return input.aux.expanded_path(lst)
-end
-
-function input.unexpanded_path(str)
-    return file.join_path(input.unexpanded_path_list(str))
-end
-
-do
-    local done = { }
-
-    function input.reset_extra_path()
-        local instance = input.instance
-        local ep = instance.extra_paths
-        if not ep then
-            ep, done = { }, { }
-            instance.extra_paths = ep
-        elseif #ep > 0 then
-            instance.lists, done = { }, { }
+    -- instance.cnffiles contain complete names now !
+    if #instance.cnffiles == 0 then
+        if trace_verbose then
+            logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)")
         end
-    end
-
-    function input.register_extra_path(paths,subpaths)
-        local instance = input.instance
-        local ep = instance.extra_paths or { }
-        local n = #ep
-        if paths and paths ~= "" then
-            if subpaths and subpaths ~= "" then
-                for p in paths:gmatch("[^,]+") do
-                    -- we gmatch each step again, not that fast, but used seldom
-                    for s in subpaths:gmatch("[^,]+") do
-                        local ps = p .. "/" .. s
-                        if not done[ps] then
-                            ep[#ep+1] = input.clean_path(ps)
-                            done[ps] = true
-                        end
-                    end
-                end
-            else
-                for p in paths:gmatch("[^,]+") do
-                    if not done[p] then
-                        ep[#ep+1] = input.clean_path(p)
-                        done[p] = true
-                    end
-                end
-            end
-        elseif subpaths and subpaths ~= "" then
-            for i=1,n do
-                -- we gmatch each step again, not that fast, but used seldom
-                for s in subpaths:gmatch("[^,]+") do
-                    local ps = ep[i] .. "/" .. s
-                    if not done[ps] then
-                        ep[#ep+1] = input.clean_path(ps)
-                        done[ps] = true
-                    end
-                end
-            end
+    else
+        instance.rootpath = instance.cnffiles[1]
+        for k,fname in ipairs(instance.cnffiles) do
+            instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
         end
-        if #ep > 0 then
-            instance.extra_paths = ep -- register paths
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
         end
-        if #ep > n then
-            instance.lists = { } -- erase the cache
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        if instance.diskcache and not instance.renewcache then
+            resolvers.loadoldconfig(instance.cnffiles)
+            if instance.loaderror then
+                loadoldconfigdata()
+                resolvers.saveoldconfig()
+            end
+        else
+            loadoldconfigdata()
+            if instance.renewcache then
+                resolvers.saveoldconfig()
+            end
         end
+        collapse_cnf_data()
     end
-
+    check_configuration()
 end
 
-function input.expanded_path_list(str)
-    local instance = input.instance
-    local function made_list(list)
-        local ep = instance.extra_paths
-        if not ep or #ep == 0 then
-            return list
-        else
-            local done, new = { }, { }
-            -- honour . .. ../.. but only when at the start
-            for k, v in ipairs(list) do
-                if not done[v] then
-                    if v:find("^[%.%/]$") then
-                        done[v] = true
-                        new[#new+1] = v
-                    else
-                        break
-                    end
-                end
-            end
-            -- first the extra paths
-            for k, v in ipairs(ep) do
-                if not done[v] then
-                    done[v] = true
-                    new[#new+1] = v
-                end
-            end
-            -- next the formal paths
-            for k, v in ipairs(list) do
-                if not done[v] then
-                    done[v] = true
-                    new[#new+1] = v
-                end
-            end
-            return new
+function resolvers.load_lua()
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
         end
-    end
-    if not str then
-        return ep or { }
-    elseif instance.savelists then
-        -- engine+progname hash
-        str = str:gsub("%$","")
-        if not instance.lists[str] then -- cached
-            local lst = made_list(input.split_path(input.expansion(str)))
-            instance.lists[str] = input.aux.expanded_path(lst)
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
         end
-        return instance.lists[str]
-    else
-        local lst = input.split_path(input.expansion(str))
-        return made_list(input.aux.expanded_path(lst))
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        resolvers.loadnewconfig()
+        collapse_cnf_data()
     end
+    check_configuration()
 end
 
+-- database loading
 
-function input.clean_path_list(str)
-    local t = input.expanded_path_list(str)
-    if t then
-        for i=1,#t do
-            t[i] = file.collapse_path(input.clean_path(t[i]))
+function resolvers.load_hash()
+    resolvers.locatelists()
+    if instance.diskcache and not instance.renewcache then
+        resolvers.loadfiles()
+        if instance.loaderror then
+            resolvers.loadlists()
+            resolvers.savefiles()
+        end
+    else
+        resolvers.loadlists()
+        if instance.renewcache then
+            resolvers.savefiles()
         end
     end
-    return t
 end
 
-function input.expand_path(str)
-    return file.join_path(input.expanded_path_list(str))
+function resolvers.append_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash append: %s",tag)
+    end
+    insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } )
 end
 
-function input.expanded_path_list_from_var(str) -- brrr
-    local tmp = input.var_of_format_or_suffix(str:gsub("%$",""))
-    if tmp ~= "" then
-        return input.expanded_path_list(str)
-    else
-        return input.expanded_path_list(tmp)
+function resolvers.prepend_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash prepend: %s",tag)
     end
-end
-function input.expand_path_from_var(str)
-    return file.join_path(input.expanded_path_list_from_var(str))
+    insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } )
 end
 
-function input.format_of_var(str)
-    return input.formats[str] or input.formats[input.alternatives[str]] or ''
-end
-function input.format_of_suffix(str)
-    return input.suffixmap[file.extname(str)] or 'tex'
+function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash
+--  local t = resolvers.expanded_path_list('TEXMF') -- full expansion
+    local t = resolvers.split_path(resolvers.env('TEXMF'))
+    insert(t,1,specification)
+    local newspec = concat(t,";")
+    if instance.environment["TEXMF"] then
+        instance.environment["TEXMF"] = newspec
+    elseif instance.variables["TEXMF"] then
+        instance.variables["TEXMF"] = newspec
+    else
+        -- weird
+    end
+    resolvers.expand_variables()
+    reset_hashes()
 end
 
-function input.variable_of_format(str)
-    return input.formats[str] or input.formats[input.alternatives[str]] or ''
-end
+-- locators
 
-function input.var_of_format_or_suffix(str)
-    local v = input.formats[str]
-    if v then
-        return v
-    end
-    v = input.formats[input.alternatives[str]]
-    if v then
-        return v
-    end
-    v = input.suffixmap[file.extname(str)]
-    if v then
-        return input.formats[isf]
+function resolvers.locatelists()
+    for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do
+        if trace_verbose then
+            logs.report("fileio","locating list of %s",path)
+        end
+        resolvers.locatedatabase(file.collapse_path(path))
     end
-    return ''
 end
 
-function input.expand_braces(str) -- output variable and brace expansion of STRING
-    local ori = input.variable(str)
-    local pth = input.aux.expanded_path(input.split_path(ori))
-    return file.join_path(pth)
+function resolvers.locatedatabase(specification)
+    return resolvers.methodhandler('locators', specification)
 end
 
--- {a,b,c,d}
--- a,b,c/{p,q,r},d
--- a,b,c/{p,q,r}/d/{x,y,z}//
--- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
--- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
--- a{b,c}{d,e}f
--- {a,b,c,d}
--- {a,b,c/{p,q,r},d}
--- {a,b,c/{p,q,r}/d/{x,y,z}//}
--- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
--- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
--- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
-
--- this one is better and faster, but it took me a while to realize
--- that this kind of replacement is cleaner than messy parsing and
--- fuzzy concatenating we can probably gain a bit with selectively
--- applying lpeg, but experiments with lpeg parsing this proved not to
--- work that well; the parsing is ok, but dealing with the resulting
--- table is a pain because we need to work inside-out recursively
-
-function input.aux.splitpathexpr(str, t, validate)
-    -- no need for optimization, only called a few times, we can use lpeg for the sub
-    t = t or { }
-    str = str:gsub(",}",",@}")
-    str = str:gsub("{,","{@,")
- -- str = "@" .. str .. "@"
-    while true do
-        local done = false
-        while true do
-            local ok = false
-            str = str:gsub("([^{},]+){([^{}]+)}", function(a,b)
-                local t = { }
-                for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end
-                ok, done = true, true
-                return "{" .. concat(t,",") .. "}"
-            end)
-            if not ok then break end
-        end
-        while true do
-            local ok = false
-            str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b)
-                local t = { }
-                for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end
-                ok, done = true, true
-                return "{" .. concat(t,",") .. "}"
-            end)
-            if not ok then break end
-        end
-        while true do
-            local ok = false
-            str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b)
-                local t = { }
-                for sa in a:gmatch("[^,]+") do
-                    for sb in b:gmatch("[^,]+") do
-                        t[#t+1] = sa .. sb
-                    end
-                end
-                ok, done = true, true
-                return "{" .. concat(t,",") .. "}"
-            end)
-            if not ok then break end
-        end
-        str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c)
-            done = true
-            return a .. b.. c
-        end)
-        if not done then break end
-    end
-    str = str:gsub("[{}]", "")
-    str = str:gsub("@","")
-    if validate then
-        for s in str:gmatch("[^,]+") do
-            s = validate(s)
-            if s then t[#t+1] = s end
-        end
-    else
-        for s in str:gmatch("[^,]+") do
-            t[#t+1] = s
+function resolvers.locators.tex(specification)
+    if specification and specification ~= '' and lfs.isdir(specification) then
+        if trace_locating then
+            logs.report("fileio",'! tex locator found: %s',specification)
         end
+        resolvers.append_hash('file',specification,filename)
+    elseif trace_locating then
+        logs.report("fileio",'? tex locator not found: %s',specification)
     end
-    return t
 end
 
-function input.aux.expanded_path(pathlist) -- maybe not a list, just a path
-    local instance = input.instance
-    -- a previous version fed back into pathlist
-    local newlist, ok = { }, false
-    for _,v in ipairs(pathlist) do
-        if v:find("[{}]") then
-            ok = true
-            break
-        end
-    end
-    if ok then
-        for _, v in ipairs(pathlist) do
-            input.aux.splitpathexpr(v, newlist, function(s)
-                s = file.collapse_path(s)
-                return s ~= "" and not s:find(instance.dummy_path_expr) and s
-            end)
-        end
-    else
-        for _,v in ipairs(pathlist) do
-            for vv in string.gmatch(v..',',"(.-),") do
-                vv = file.collapse_path(v)
-                if vv ~= "" then newlist[#newlist+1] = vv end
-            end
+-- hashers
+
+function resolvers.hashdatabase(tag,name)
+    return resolvers.methodhandler('hashers',tag,name)
+end
+
+function resolvers.loadfiles()
+    instance.loaderror = false
+    instance.files = { }
+    if not instance.renewcache then
+        for _, hash in ipairs(instance.hashes) do
+            resolvers.hashdatabase(hash.tag,hash.name)
+            if instance.loaderror then break end
         end
     end
-    return newlist
 end
 
-input.is_readable = { }
+function resolvers.hashers.tex(tag,name)
+    resolvers.load_data(tag,'files')
+end
 
-function input.aux.is_readable(readable, name)
-    if input.trace > 2 then
-        if readable then
-            input.logger("+ readable: %s",name)
-        else
-            input.logger("- readable: %s", name)
-        end
+-- generators:
+
+function resolvers.loadlists()
+    for _, hash in ipairs(instance.hashes) do
+        resolvers.generatedatabase(hash.tag)
     end
-    return readable
 end
 
-function input.is_readable.file(name)
-    return input.aux.is_readable(lfs.isfile(name), name)
+function resolvers.generatedatabase(specification)
+    return resolvers.methodhandler('generators', specification)
 end
 
-input.is_readable.tex = input.is_readable.file
+-- starting with . or .. etc or funny char
 
--- name
--- name/name
+local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t"))
 
-function input.aux.collect_files(names)
-    local instance = input.instance
-    local filelist = { }
-    for _, fname in pairs(names) do
-        if fname then
-            if input.trace > 2 then
-                input.logger("? blobpath asked: %s",fname)
-            end
-            local bname = file.basename(fname)
-            local dname = file.dirname(fname)
-            if dname == "" or dname:find("^%.") then
-                dname = false
-            else
-                dname = "/" .. dname .. "$"
-            end
-            for _, hash in ipairs(instance.hashes) do
-                local blobpath = hash.tag
-                local files = blobpath and instance.files[blobpath]
-                if files then
-                    if input.trace > 2 then
-                        input.logger('? blobpath do: %s (%s)',blobpath,bname)
-                    end
-                    local blobfile = files[bname]
-                    if not blobfile then
-                        local rname = "remap:"..bname
-                        blobfile = files[rname]
-                        if blobfile then
-                            bname = files[rname]
-                            blobfile = files[bname]
-                        end
-                    end
-                    if blobfile then
-                        if type(blobfile) == 'string' then
-                            if not dname or blobfile:find(dname) then
-                                filelist[#filelist+1] = {
-                                    hash.type,
-                                    file.join(blobpath,blobfile,bname), -- search
-                                    input.concatinators[hash.type](blobpath,blobfile,bname) -- result
-                                }
+function resolvers.generators.tex(specification)
+    local tag = specification
+    if trace_verbose then
+        logs.report("fileio","scanning path %s",specification)
+    end
+    instance.files[tag] = { }
+    local files = instance.files[tag]
+    local n, m, r = 0, 0, 0
+    local spec = specification .. '/'
+    local attributes = lfs.attributes
+    local directory = lfs.dir
+    local function action(path)
+        local full
+        if path then
+            full = spec .. path .. '/'
+        else
+            full = spec
+        end
+        for name in directory(full) do
+            if not weird:match(name) then
+                local mode = attributes(full..name,'mode')
+                if mode == 'file' then
+                    if path then
+                        n = n + 1
+                        local f = files[name]
+                        if f then
+                            if type(f) == 'string' then
+                                files[name] = { f, path }
+                            else
+                                f[#f+1] = path
                             end
-                        else
-                            for _, vv in pairs(blobfile) do
-                                if not dname or vv:find(dname) then
-                                    filelist[#filelist+1] = {
-                                        hash.type,
-                                        file.join(blobpath,vv,bname), -- search
-                                        input.concatinators[hash.type](blobpath,vv,bname) -- result
-                                    }
-                                end
+                        else -- probably unique anyway
+                            files[name] = path
+                            local lower = lower(name)
+                            if name ~= lower then
+                                files["remap:"..lower] = name
+                                r = r + 1
                             end
                         end
                     end
-                elseif input.trace > 1 then
-                    input.logger('! blobpath no: %s (%s)',blobpath,bname)
+                elseif mode == 'directory' then
+                    m = m + 1
+                    if path then
+                        action(path..'/'..name)
+                    else
+                        action(name)
+                    end
                 end
             end
         end
     end
-    if #filelist > 0 then
-        return filelist
-    else
-        return nil
+    action()
+    if trace_verbose then
+        logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r)
     end
 end
 
-function input.suffix_of_format(str)
-    if input.suffixes[str] then
-        return input.suffixes[str][1]
-    else
-        return ""
-    end
-end
+-- savers, todo
 
-function input.suffixes_of_format(str)
-    if input.suffixes[str] then
-        return input.suffixes[str]
-    else
-        return {}
-    end
+function resolvers.savefiles()
+    resolvers.save_data('files')
 end
 
-do
+-- A config (optionally) has the paths split in tables. Internally
+-- we join them and split them after the expansion has taken place. This
+-- is more convenient.
 
-    -- called about 700 times for an empty doc (font initializations etc)
-    -- i need to weed the font files for redundant calls
+function resolvers.splitconfig()
+    for i,c in ipairs(instance) do
+        for k,v in pairs(c) do
+            if type(v) == 'string' then
+                local t = file.split_path(v)
+                if #t > 1 then
+                    c[k] = t
+                end
+            end
+        end
+    end
+end
 
-    local letter     = lpeg.R("az","AZ")
-    local separator  = lpeg.P("://")
-
-    local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator
-    local rootbased = lpeg.P("/") + letter*lpeg.P(":")
-
-    -- ./name ../name  /name c: ://
-    function input.aux.qualified_path(filename)
-        return qualified:match(filename)
+function resolvers.joinconfig()
+    for i,c in ipairs(instance.order) do
+        for k,v in pairs(c) do -- ipairs?
+            if type(v) == 'table' then
+                c[k] = file.join_path(v)
+            end
+        end
     end
-    function input.aux.rootbased_path(filename)
-        return rootbased:match(filename)
+end
+function resolvers.split_path(str)
+    if type(str) == 'table' then
+        return str
+    else
+        return file.split_path(str)
     end
-
-    function input.normalize_name(original)
-        return original
+end
+function resolvers.join_path(str)
+    if type(str) == 'table' then
+        return file.join_path(str)
+    else
+        return str
     end
-
-    input.normalize_name = file.collapse_path
-
 end
 
-function input.aux.register_in_trees(name)
-    if not name:find("^%.") then
-        local instance = input.instance
-        instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one
+function resolvers.splitexpansions()
+    local ie = instance.expansions
+    for k,v in next, ie do
+        local t, h = { }, { }
+        for _,vv in ipairs(file.split_path(v)) do
+            if vv ~= "" and not h[vv] then
+                t[#t+1] = vv
+                h[vv] = true
+            end
+        end
+        if #t > 1 then
+            ie[k] = t
+        else
+            ie[k] = t[1]
+        end
     end
 end
 
--- split the next one up, better for jit
+-- end of split/join code
+
+function resolvers.saveoldconfig()
+    resolvers.splitconfig()
+    resolvers.save_data('configuration')
+    resolvers.joinconfig()
+end
+
+resolvers.configbanner = [[
+-- This is a Luatex configuration file created by 'luatools.lua' or
+-- 'luatex.exe' directly. For comment, suggestions and questions you can
+-- contact the ConTeXt Development Team. This configuration file is
+-- not copyrighted. [HH & TH]
+]]
 
-function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc)
-    local instance = input.instance
-    local result = { }
-    local stamp  = nil
-    filename = input.normalize_name(filename)  -- elsewhere
-    filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere
-    -- speed up / beware: format problem
-    if instance.remember then
-        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format
-        if instance.found[stamp] then
-            if input.trace > 0 then
-                input.logger('! remembered: %s',filename)
-            end
-            return instance.found[stamp]
+function resolvers.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local function dump(k,v,m) -- could be moved inline
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
         end
     end
-    if filename:find('%*') then
-        if input.trace > 0 then
-            input.logger('! wildcard: %s', filename)
-        end
-        result = input.find_wildcard_files(filename)
-    elseif input.aux.qualified_path(filename) then
-        if input.is_readable.file(filename) then
-            if input.trace > 0 then
-                input.logger('! qualified: %s', filename)
-            end
-            result = { filename }
-        else
-            local forcedname, ok = "", false
-            if file.extname(filename) == "" then
-                if instance.format == "" then
-                    forcedname = filename .. ".tex"
-                    if input.is_readable.file(forcedname) then
-                        if input.trace > 0 then
-                            input.logger('! no suffix, forcing standard filetype: tex')
-                        end
-                        result, ok = { forcedname }, true
-                    end
-                else
-                    for _, s in pairs(input.suffixes_of_format(instance.format)) do
-                        forcedname = filename .. "." .. s
-                        if input.is_readable.file(forcedname) then
-                            if input.trace > 0 then
-                                input.logger('! no suffix, forcing format filetype: %s', s)
-                            end
-                            result, ok = { forcedname }, true
-                            break
-                        end
-                    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sortedkeys(files)) do -- ipairs
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sortedkeys(fk)) do -- ipairs
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
                 end
-            end
-            if not ok and input.trace > 0 then
-                input.logger('? qualified: %s', filename)
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
             end
         end
     else
-        -- search spec
-        local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename)
-        if ext == "" then
-            if not instance.force_suffixes then
-                wantedfiles[#wantedfiles+1] = filename
-            end
-        else
-            wantedfiles[#wantedfiles+1] = filename
-        end
-        if instance.format == "" then
-            if ext == "" then
-                local forcedname = filename .. '.tex'
-                wantedfiles[#wantedfiles+1] = forcedname
-                filetype = input.format_of_suffix(forcedname)
-                    if input.trace > 0 then
-                        input.logger('! forcing filetype: %s',filetype)
-                    end
-            else
-                filetype = input.format_of_suffix(filename)
-                if input.trace > 0 then
-                    input.logger('! using suffix based filetype: %s',filetype)
-                end
-            end
-        else
-            if ext == "" then
-                for _, s in pairs(input.suffixes_of_format(instance.format)) do
-                    wantedfiles[#wantedfiles+1] = filename .. "." .. s
+        for k, v in next, files do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in next, v do
+                    t[#t+1] = dump(kk,vv,"\t\t")
                 end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,v,"\t")
             end
-            filetype = instance.format
-            if input.trace > 0 then
-                input.logger('! using given filetype: %s',filetype)
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+function resolvers.save_data(dataname, makename) -- untested without cache overload
+    for cachename, files in next, instance[dataname] do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. ".lua", name .. ".luc"
+        if trace_verbose then
+            logs.report("fileio","preparing %s for %s",dataname,cachename)
+        end
+        for k, v in next, files do
+            if type(v) == "table" and #v == 1 then
+                files[k] = v[1]
             end
         end
-        local typespec = input.variable_of_format(filetype)
-        local pathlist = input.expanded_path_list(typespec)
-        if not pathlist or #pathlist == 0 then
-            -- no pathlist, access check only / todo == wildcard
-            if input.trace > 2 then
-                input.logger('? filename: %s',filename)
-                input.logger('? filetype: %s',filetype or '?')
-                input.logger('? wanted files: %s',concat(wantedfiles," | "))
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = resolvers.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local ok = io.savedata(luaname,resolvers.serialize(data))
+        if ok then
+            if trace_verbose then
+                logs.report("fileio","%s saved in %s",dataname,luaname)
             end
-            for _, fname in pairs(wantedfiles) do
-                if fname and input.is_readable.file(fname) then
-                    filename, done = fname, true
-                    result[#result+1] = file.join('.',fname)
-                    break
+            if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip
+                if trace_verbose then
+                    logs.report("fileio","%s compiled to %s",dataname,lucname)
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname)
                 end
+                os.remove(lucname)
             end
-            -- this is actually 'other text files' or 'any' or 'whatever'
-            local filelist = input.aux.collect_files(wantedfiles)
-            local fl = filelist and filelist[1]
-            if fl then
-                filename = fl[3]
-                result[#result+1] = filename
-                done = true
+        elseif trace_verbose then
+            logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname)
+        end
+    end
+end
+
+function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua")
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then
+            if trace_verbose then
+                logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename)
             end
+            instance[dataname][pathname] = data.content
         else
-            -- list search
-            local filelist = input.aux.collect_files(wantedfiles)
-            local doscan, recurse
-            if input.trace > 2 then
-                input.logger('? filename: %s',filename)
-            --                if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end
-            --                if filelist then input.logger('? file list: %s',concat(filelist," | ")) end
+            if trace_verbose then
+                logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
             end
-            -- a bit messy ... esp the doscan setting here
-            for _, path in pairs(pathlist) do
-                if path:find("^!!") then doscan  = false else doscan  = true  end
-                if path:find("//$") then recurse = true  else recurse = false end
-                local pathname = path:gsub("^!+", '')
-                done = false
-                -- using file list
-                if filelist and not (done and not instance.allresults) and recurse then
-                    -- compare list entries with permitted pattern
-                    pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences
-                    pathname = pathname:gsub("/+$", '/.*')      -- later usage of pathname
-                    pathname = pathname:gsub("//", '/.-/')      -- not ok for /// but harmless
-                    local expr = "^" .. pathname
-                    -- input.debug('?',expr)
-                    for _, fl in ipairs(filelist) do
-                        local f = fl[2]
-                        if f:find(expr) then
-                            -- input.debug('T',' '..f)
-                            if input.trace > 2 then
-                                input.logger('= found in hash: %s',f)
-                            end
-                            --- todo, test for readable
-                            result[#result+1] = fl[3]
-                            input.aux.register_in_trees(f) -- for tracing used files
-                            done = true
-                            if not instance.allresults then break end
-                        else
-                            -- input.debug('F',' '..f)
-                        end
-                    end
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
+        end
+    elseif trace_verbose then
+        logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+    end
+end
+
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
+
+function resolvers.resetconfig()
+    identify_own()
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function resolvers.loadnewconfig()
+    for _, cnf in ipairs(instance.luafiles) do
+        local pathname = file.dirname(cnf)
+        local filename = file.join(pathname,resolvers.luaname)
+        local blob = loadfile(filename)
+        if blob then
+            local data = blob()
+            if data then
+                if trace_verbose then
+                    logs.report("fileio","loading configuration file %s",filename)
                 end
-                if not done and doscan then
-                    -- check if on disk / unchecked / does not work at all / also zips
-                    if input.method_is_file(pathname) then -- ?
-                        local pname = pathname:gsub("%.%*$",'')
-                        if not pname:find("%*") then
-                            local ppname = pname:gsub("/+$","")
-                            if input.aux.can_be_dir(ppname) then
-                                for _, w in pairs(wantedfiles) do
-                                    local fname = file.join(ppname,w)
-                                    if input.is_readable.file(fname) then
-                                        if input.trace > 2 then
-                                            input.logger('= found by scanning: %s',fname)
-                                        end
-                                        result[#result+1] = fname
-                                        done = true
-                                        if not instance.allresults then break end
-                                    end
+                if true then
+                    -- flatten to variable.progname
+                    local t = { }
+                    for k, v in next, data do -- v = progname
+                        if type(v) == "string" then
+                            t[k] = v
+                        else
+                            for kk, vv in next, v do -- vv = variable
+                                if type(vv) == "string" then
+                                    t[vv.."."..v] = kk
                                 end
-                            else
-                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
                             end
                         end
                     end
+                    instance['setup'][pathname] = t
+                else
+                    instance['setup'][pathname] = data
                 end
-                if not done and doscan then
-                    -- todo: slow path scanning
+            else
+                if trace_verbose then
+                    logs.report("fileio","skipping configuration file %s",filename)
                 end
-                if done and not instance.allresults then break end
+                instance['setup'][pathname] = { }
+                instance.loaderror = true
             end
+        elseif trace_verbose then
+            logs.report("fileio","skipping configuration file %s",filename)
         end
+        instance.order[#instance.order+1] = instance.setup[pathname]
+        if instance.loaderror then break end
     end
-    for k,v in pairs(result) do
-        result[k] = file.collapse_path(v)
-    end
-    if instance.remember then
-        instance.found[stamp] = result
-    end
-    return result
 end
 
-input.aux._find_file_ = input.aux.find_file -- frozen variant
-
-function input.aux.find_file(filename) -- maybe make a lowres cache too
-    local result = input.aux._find_file_(filename)
-    if #result == 0 then
-        local lowered = filename:lower()
-        if filename ~= lowered then
-            return input.aux._find_file_(lowered)
+function resolvers.loadoldconfig()
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            resolvers.load_data(dname,'configuration')
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
         end
     end
-    return result
+    resolvers.joinconfig()
 end
 
-function input.aux.can_be_dir(name)
-    local instance = input.instance
-    if not instance.fakepaths[name] then
-        if lfs.isdir(name) then
-            instance.fakepaths[name] = 1 -- directory
+function resolvers.expand_variables()
+    local expansions, environment, variables = { }, instance.environment, instance.variables
+    local env = resolvers.env
+    instance.expansions = expansions
+    if instance.engine   ~= "" then environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then environment['progname'] = instance.progname end
+    for k,v in next, environment do
+        local a, b = match(k,"^(%a+)%_(.*)%s*$")
+        if a and b then
+            expansions[a..'.'..b] = v
         else
-            instance.fakepaths[name] = 2 -- no directory
+            expansions[k] = v
+        end
+    end
+    for k,v in next, environment do -- move environment to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    for k,v in next, variables do -- move variables to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    local busy = false
+    local function resolve(a)
+        busy = true
+        return expansions[a] or env(a)
+    end
+    while true do
+        busy = false
+        for k,v in next, expansions do
+            local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve)
+            local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve)
+            if n > 0 or m > 0 then
+                expansions[k]= s
+            end
         end
+        if not busy then break end
+    end
+    for k,v in next, expansions do
+        expansions[k] = gsub(v,"\\", '/')
     end
-    return (instance.fakepaths[name] == 1)
 end
 
-if not input.concatinators  then input.concatinators = { } end
+function resolvers.variable(name)
+    return entry(instance.variables,name)
+end
 
-input.concatinators.tex  = file.join
-input.concatinators.file = input.concatinators.tex
+function resolvers.expansion(name)
+    return entry(instance.expansions,name)
+end
 
-function input.find_files(filename,filetype,mustexist)
-    local instance = input.instance
-    if type(mustexist) == boolean then
-        -- all set
-    elseif type(filetype) == 'boolean' then
-        filetype, mustexist = nil, false
-    elseif type(filetype) ~= 'string' then
-        filetype, mustexist = nil, false
-    end
-    instance.format = filetype or ''
-    local t = input.aux.find_file(filename,true)
-    instance.format = ''
-    return t
+function resolvers.is_variable(name)
+    return is_entry(instance.variables,name)
 end
 
-function input.find_file(filename,filetype,mustexist)
-    return (input.find_files(filename,filetype,mustexist)[1] or "")
+function resolvers.is_expansion(name)
+    return is_entry(instance.expansions,name)
 end
 
-function input.find_given_files(filename)
-    local instance = input.instance
-    local bname, result = file.basename(filename), { }
-    for k, hash in ipairs(instance.hashes) do
-        local files = instance.files[hash.tag]
-        local blist = files[bname]
-        if not blist then
-            local rname = "remap:"..bname
-            blist = files[rname]
-            if blist then
-                bname = files[rname]
-                blist = files[bname]
-            end
-        end
-        if blist then
-            if type(blist) == 'string' then
-                result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or ""
-                if not instance.allresults then break end
-            else
-                for kk,vv in pairs(blist) do
-                    result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or ""
-                    if not instance.allresults then break end
-                end
-            end
-        end
-    end
-    return result
+function resolvers.unexpanded_path_list(str)
+    local pth = resolvers.variable(str)
+    local lst = resolvers.split_path(pth)
+    return expanded_path_from_list(lst)
 end
 
-function input.find_given_file(filename)
-    return (input.find_given_files(filename)[1] or "")
+function resolvers.unexpanded_path(str)
+    return file.join_path(resolvers.unexpanded_path_list(str))
 end
 
-function input.find_wildcard_files(filename) -- todo: remap:
-    local instance = input.instance
-    local result = { }
-    local bname, dname = file.basename(filename), file.dirname(filename)
-    local path = dname:gsub("^*/","")
-    path = path:gsub("*",".*")
-    path = path:gsub("-","%%-")
-    if dname == "" then
-        path = ".*"
+do -- no longer needed
+
+    local done = { }
+
+    function resolvers.reset_extra_path()
+        local ep = instance.extra_paths
+        if not ep then
+            ep, done = { }, { }
+            instance.extra_paths = ep
+        elseif #ep > 0 then
+            instance.lists, done = { }, { }
+        end
     end
-    local name = bname
-    name = name:gsub("*",".*")
-    name = name:gsub("-","%%-")
-    path = path:lower()
-    name = name:lower()
-    local function doit(blist,bname,hash,allresults)
-        local done = false
-        if blist then
-            if type(blist) == 'string' then
-                -- make function and share code
-                if (blist:lower()):find(path) then
-                    result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or ""
-                    done = true
+
+    function resolvers.register_extra_path(paths,subpaths)
+        local ep = instance.extra_paths or { }
+        local n = #ep
+        if paths and paths ~= "" then
+            if subpaths and subpaths ~= "" then
+                for p in gmatch(paths,"[^,]+") do
+                    -- we gmatch each step again, not that fast, but used seldom
+                    for s in gmatch(subpaths,"[^,]+") do
+                        local ps = p .. "/" .. s
+                        if not done[ps] then
+                            ep[#ep+1] = resolvers.clean_path(ps)
+                            done[ps] = true
+                        end
+                    end
                 end
             else
-                for kk,vv in pairs(blist) do
-                    if (vv:lower()):find(path) then
-                        result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or ""
-                        done = true
-                        if not allresults then break end
+                for p in gmatch(paths,"[^,]+") do
+                    if not done[p] then
+                        ep[#ep+1] = resolvers.clean_path(p)
+                        done[p] = true
                     end
                 end
             end
-        end
-        return done
-    end
-    local files, allresults, done = instance.files, instance.allresults, false
-    if name:find("%*") then
-        for k, hash in ipairs(instance.hashes) do
-            for kk, hh in pairs(files[hash.tag]) do
-                if not kk:find("^remap:") then
-                    if (kk:lower()):find(name) then
-                        if doit(hh,kk,hash,allresults) then done = true end
-                        if done and not allresults then break end
+        elseif subpaths and subpaths ~= "" then
+            for i=1,n do
+                -- we gmatch each step again, not that fast, but used seldom
+                for s in gmatch(subpaths,"[^,]+") do
+                    local ps = ep[i] .. "/" .. s
+                    if not done[ps] then
+                        ep[#ep+1] = resolvers.clean_path(ps)
+                        done[ps] = true
                     end
                 end
             end
         end
-    else
-        for k, hash in ipairs(instance.hashes) do
-            if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end
-            if done and not allresults then break end
-        end
-    end
-    -- we can consider also searching the paths not in the database, but then
-    -- we end up with a messy search (all // in all path specs)
-    return result
-end
-
-function input.find_wildcard_file(filename)
-    return (input.find_wildcard_files(filename)[1] or "")
-end
-
--- main user functions
-
-function input.save_used_files_in_trees(filename,jobname)
-    local instance = input.instance
-    if not filename then filename = 'luatex.jlg' end
-    local f = io.open(filename,'w')
-    if f then
-        f:write("\n")
-        f:write("\n")
-        if jobname then
-            f:write("\t" .. jobname .. "\n")
+        if #ep > 0 then
+            instance.extra_paths = ep -- register paths
         end
-        f:write("\t\n")
-        for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs
-            f:write("\t\t" .. v .. "\n")
+        if #ep > n then
+            instance.lists = { } -- erase the cache
         end
-        f:write("\t\n")
-        f:write("\n")
-        f:close()
     end
-end
-
-function input.automount()
-    -- implemented later
-end
 
-function input.load()
-    input.starttiming(input.instance)
-    input.resetconfig()
-    input.identify_cnf()
-    input.load_lua()
-    input.expand_variables()
-    input.load_cnf()
-    input.expand_variables()
-    input.load_hash()
-    input.automount()
-    input.stoptiming(input.instance)
 end
 
-function input.for_files(command, files, filetype, mustexist)
-    if files and #files > 0 then
-        local function report(str)
-            if input.verbose then
-                input.report(str) -- has already verbose
-            else
-                print(str)
+local function made_list(instance,list)
+    local ep = instance.extra_paths
+    if not ep or #ep == 0 then
+        return list
+    else
+        local done, new = { }, { }
+        -- honour . .. ../.. but only when at the start
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                if find(v,"^[%.%/]$") then
+                    done[v] = true
+                    new[#new+1] = v
+                else
+                    break
+                end
             end
         end
-        if input.verbose then
-            report('')
+        -- first the extra paths
+        for k=1,#ep do
+            local v = ep[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
         end
-        for _, file in pairs(files) do
-            local result = command(file,filetype,mustexist)
-            if type(result) == 'string' then
-                report(result)
-            else
-                for _,v in pairs(result) do
-                    report(v)
-                end
+        -- next the formal paths
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
             end
         end
+        return new
     end
 end
 
--- strtab
-
-input.var_value  = input.variable   -- output the value of variable $STRING.
-input.expand_var = input.expansion  -- output variable expansion of STRING.
-
-function input.show_path(str)     -- output search path for file type NAME
-    return file.join_path(input.expanded_path_list(input.format_of_var(str)))
-end
-
--- input.find_file(filename)
--- input.find_file(filename, filetype, mustexist)
--- input.find_file(filename, mustexist)
--- input.find_file(filename, filetype)
-
-function input.aux.register_file(files, name, path)
-    if files[name] then
-        if type(files[name]) == 'string' then
-            files[name] = { files[name], path }
-        else
-            files[name] = path
+function resolvers.clean_path_list(str)
+    local t = resolvers.expanded_path_list(str)
+    if t then
+        for i=1,#t do
+            t[i] = file.collapse_path(resolvers.clean_path(t[i]))
         end
-    else
-        files[name] = path
-    end
-end
-
-if not input.finders  then input.finders  = { } end
-if not input.openers  then input.openers  = { } end
-if not input.loaders  then input.loaders  = { } end
-
-input.finders.notfound  = { nil }
-input.openers.notfound  = { nil }
-input.loaders.notfound  = { false, nil, 0 }
-
-function input.splitmethod(filename)
-    if not filename then
-        return { } -- safeguard
-    elseif type(filename) == "table" then
-        return filename -- already split
-    elseif not filename:find("://") then
-        return { scheme="file", path = filename, original=filename } -- quick hack
-    else
-        return url.hashed(filename)
     end
+    return t
 end
 
-function input.method_is_file(filename)
-    return input.splitmethod(filename).scheme == 'file'
+function resolvers.expand_path(str)
+    return file.join_path(resolvers.expanded_path_list(str))
 end
 
-function table.sequenced(t,sep) -- temp here
-    local s = { }
-    for k, v in pairs(t) do
-        s[#s+1] = k .. "=" .. v
+function resolvers.expanded_path_list(str)
+    if not str then
+        return ep or { }
+    elseif instance.savelists then
+        -- engine+progname hash
+        str = gsub(str,"%$","")
+        if not instance.lists[str] then -- cached
+            local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str)))
+            instance.lists[str] = expanded_path_from_list(lst)
+        end
+        return instance.lists[str]
+    else
+        local lst = resolvers.split_path(resolvers.expansion(str))
+        return made_list(instance,expanded_path_from_list(lst))
     end
-    return concat(s, sep or " | ")
 end
 
-function input.methodhandler(what, filename, filetype) -- ...
-    local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb
-    local scheme = specification.scheme
-    if input[what][scheme] then
-        if input.trace > 0 then
-            input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification))
-        end
-        return input[what][scheme](filename,filetype) -- todo: specification
+function resolvers.expanded_path_list_from_var(str) -- brrr
+    local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$",""))
+    if tmp ~= "" then
+        return resolvers.expanded_path_list(str)
     else
-        return input[what].tex(filename,filetype) -- todo: specification
+        return resolvers.expanded_path_list(tmp)
     end
 end
 
--- also inside next test?
+function resolvers.expand_path_from_var(str)
+    return file.join_path(resolvers.expanded_path_list_from_var(str))
+end
 
-function input.findtexfile(filename, filetype)
-    return input.methodhandler('finders',input.normalize_name(filename), filetype)
+function resolvers.format_of_var(str)
+    return formats[str] or formats[alternatives[str]] or ''
 end
-function input.opentexfile(filename)
-    return input.methodhandler('openers',input.normalize_name(filename))
+function resolvers.format_of_suffix(str)
+    return suffixmap[file.extname(str)] or 'tex'
 end
 
-function input.findbinfile(filename, filetype)
-    return input.methodhandler('finders',input.normalize_name(filename), filetype)
-end
-function input.openbinfile(filename)
-    return input.methodhandler('loaders',input.normalize_name(filename))
+function resolvers.variable_of_format(str)
+    return formats[str] or formats[alternatives[str]] or ''
 end
 
-function input.loadbinfile(filename, filetype)
-    local fname = input.findbinfile(input.normalize_name(filename), filetype)
-    if fname and fname ~= "" then
-        return input.openbinfile(fname)
-    else
-        return unpack(input.loaders.notfound)
+function resolvers.var_of_format_or_suffix(str)
+    local v = formats[str]
+    if v then
+        return v
+    end
+    v = formats[alternatives[str]]
+    if v then
+        return v
+    end
+    v = suffixmap[file.extname(str)]
+    if v then
+        return formats[isf]
     end
+    return ''
 end
 
-function input.texdatablob(filename, filetype)
-    local ok, data, size = input.loadbinfile(filename, filetype)
-    return data or ""
+function resolvers.expand_braces(str) -- output variable and brace expansion of STRING
+    local ori = resolvers.variable(str)
+    local pth = expanded_path_from_list(resolvers.split_path(ori))
+    return file.join_path(pth)
 end
 
-input.loadtexfile = input.texdatablob
+resolvers.isreadable = { }
 
-function input.openfile(filename)
-    local fullname = input.findtexfile(filename)
-    if fullname and (fullname ~= "") then
-        return input.opentexfile(fullname)
-    else
-        return nil
+function resolvers.isreadable.file(name)
+    local readable = lfs.isfile(name) -- brrr
+    if trace_detail then
+        if readable then
+            logs.report("fileio","+ readable: %s",name)
+        else
+            logs.report("fileio","- readable: %s", name)
+        end
     end
+    return readable
 end
 
-function input.logmode()
-    return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower()
-end
-
--- this is a prelude to engine/progname specific configuration files
--- in which case we can omit files meant for other programs and
--- packages
+resolvers.isreadable.tex = resolvers.isreadable.file
 
---- ctx
-
--- maybe texinputs + font paths
--- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc
-
-input.validators            = { }
-input.validators.visibility = { }
+-- name
+-- name/name
 
-function input.validators.visibility.default(path, name)
-    return true
+local function collect_files(names)
+    local filelist = { }
+    for k=1,#names do
+        local fname = names[k]
+        if trace_detail then
+            logs.report("fileio","? blobpath asked: %s",fname)
+        end
+        local bname = file.basename(fname)
+        local dname = file.dirname(fname)
+        if dname == "" or find(dname,"^%.") then
+            dname = false
+        else
+            dname = "/" .. dname .. "$"
+        end
+        local hashes = instance.hashes
+        for h=1,#hashes do
+            local hash = hashes[h]
+            local blobpath = hash.tag
+            local files = blobpath and instance.files[blobpath]
+            if files then
+                if trace_detail then
+                    logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname)
+                end
+                local blobfile = files[bname]
+                if not blobfile then
+                    local rname = "remap:"..bname
+                    blobfile = files[rname]
+                    if blobfile then
+                        bname = files[rname]
+                        blobfile = files[bname]
+                    end
+                end
+                if blobfile then
+                    if type(blobfile) == 'string' then
+                        if not dname or find(blobfile,dname) then
+                            filelist[#filelist+1] = {
+                                hash.type,
+                                file.join(blobpath,blobfile,bname), -- search
+                                resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result
+                            }
+                        end
+                    else
+                        for kk=1,#blobfile do
+                            local vv = blobfile[kk]
+                            if not dname or find(vv,dname) then
+                                filelist[#filelist+1] = {
+                                    hash.type,
+                                    file.join(blobpath,vv,bname), -- search
+                                    resolvers.concatinators[hash.type](blobpath,vv,bname) -- result
+                                }
+                            end
+                        end
+                    end
+                end
+            elseif trace_locating then
+                logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname)
+            end
+        end
+    end
+    if #filelist > 0 then
+        return filelist
+    else
+        return nil
+    end
 end
 
-function input.validators.visibility.context(path, name)
-    path = path[1] or path -- some day a loop
-    return not (
-        path:find("latex")    or
---      path:find("doc")      or
-        path:find("tex4ht")   or
-        path:find("source")   or
---      path:find("config")   or
---      path:find("metafont") or
-        path:find("lists$")   or
-        name:find("%.tpm$")   or
-        name:find("%.bak$")
-    )
+function resolvers.suffix_of_format(str)
+    if suffixes[str] then
+        return suffixes[str][1]
+    else
+        return ""
+    end
 end
 
--- todo: describe which functions are public (maybe input.private. ... )
-
--- beware: i need to check where we still need a / on windows:
-
-function input.clean_path(str)
-    if str then
-        str = str:gsub("\\","/")
-        str = str:gsub("^!+","")
-        str = str:gsub("^~",input.homedir)
-        return str
+function resolvers.suffixes_of_format(str)
+    if suffixes[str] then
+        return suffixes[str]
     else
-        return nil
+        return {}
     end
 end
 
-function input.do_with_path(name,func)
-    for _, v in pairs(input.expanded_path_list(name)) do
-        func("^"..input.clean_path(v))
+function resolvers.register_in_trees(name)
+    if not find(name,"^%.") then
+        instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one
     end
 end
 
-function input.do_with_var(name,func)
-    func(input.aux.expanded_var(name))
+-- split the next one up for readability (bu this module needs a cleanup anyway)
+
+local function can_be_dir(name) -- can become local
+    local fakepaths = instance.fakepaths
+    if not fakepaths[name] then
+        if lfs.isdir(name) then
+            fakepaths[name] = 1 -- directory
+        else
+            fakepaths[name] = 2 -- no directory
+        end
+    end
+    return (fakepaths[name] == 1)
 end
 
-function input.with_files(pattern,handle)
-    local instance = input.instance
-    for _, hash in ipairs(instance.hashes) do
-        local blobpath = hash.tag
-        local blobtype = hash.type
-        if blobpath then
-            local files = instance.files[blobpath]
-            if files then
-                for k,v in pairs(files) do
-                    if k:find("^remap:") then
-                        k = files[k]
-                        v = files[k] -- chained
+local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc)
+    local result = collected or { }
+    local stamp  = nil
+    filename = file.collapse_path(filename)  -- elsewhere
+    filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere
+    -- speed up / beware: format problem
+    if instance.remember then
+        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format
+        if instance.found[stamp] then
+            if trace_locating then
+                logs.report("fileio",'! remembered: %s',filename)
+            end
+            return instance.found[stamp]
+        end
+    end
+    if not dangerous[instance.format or "?"] then
+        if resolvers.isreadable.file(filename) then
+            if trace_detail then
+                logs.report("fileio",'= found directly: %s',filename)
+            end
+            instance.found[stamp] = { filename }
+            return { filename }
+        end
+    end
+    if find(filename,'%*') then
+        if trace_locating then
+            logs.report("fileio",'! wildcard: %s', filename)
+        end
+        result = resolvers.find_wildcard_files(filename)
+    elseif file.is_qualified_path(filename) then
+        if resolvers.isreadable.file(filename) then
+            if trace_locating then
+                logs.report("fileio",'! qualified: %s', filename)
+            end
+            result = { filename }
+        else
+            local forcedname, ok, suffix = "", false, file.extname(filename)
+            if suffix == "" then -- why
+                if instance.format == "" then
+                    forcedname = filename .. ".tex"
+                    if resolvers.isreadable.file(forcedname) then
+                        if trace_locating then
+                            logs.report("fileio",'! no suffix, forcing standard filetype: tex')
+                        end
+                        result, ok = { forcedname }, true
                     end
-                    if k:find(pattern) then
-                        if type(v) == "string" then
-                            handle(blobtype,blobpath,v,k)
-                        else
-                            for _,vv in pairs(v) do
-                                handle(blobtype,blobpath,vv,k)
+                else
+                    local suffixes = resolvers.suffixes_of_format(instance.format)
+                    for _, s in next, suffixes do
+                        forcedname = filename .. "." .. s
+                        if resolvers.isreadable.file(forcedname) then
+                            if trace_locating then
+                                logs.report("fileio",'! no suffix, forcing format filetype: %s', s)
                             end
+                            result, ok = { forcedname }, true
+                            break
                         end
                     end
                 end
             end
+            if not ok and suffix ~= "" then
+                -- try to find in tree (no suffix manipulation), here we search for the
+                -- matching last part of the name
+                local basename = file.basename(filename)
+                local pattern = (filename .. "$"):gsub("([%.%-])","%%%1")
+                local savedformat = instance.format
+                local format = savedformat or ""
+                if format == "" then
+                    instance.format = resolvers.format_of_suffix(suffix)
+                end
+                if not format then
+                    instance.format = "othertextfiles" -- kind of everything, maybe texinput is better
+                end
+                --
+                local resolved = collect_instance_files(basename)
+                if #result == 0 then
+                    local lowered = lower(basename)
+                    if filename ~= lowered then
+                        resolved = collect_instance_files(lowered)
+                    end
+                end
+                resolvers.format = savedformat
+                --
+                for r=1,#resolved do
+                    local rr = resolved[r]
+                    if rr:find(pattern) then
+                        result[#result+1], ok = rr, true
+                    end
+                end
+                -- a real wildcard:
+                --
+                -- if not ok then
+                --     local filelist = collect_files({basename})
+                --     for f=1,#filelist do
+                --         local ff = filelist[f][3] or ""
+                --         if ff:find(pattern) then
+                --             result[#result+1], ok = ff, true
+                --         end
+                --     end
+                -- end
+            end
+            if not ok and trace_locating then
+                logs.report("fileio",'? qualified: %s', filename)
+            end
         end
-    end
-end
-
-function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix
-    local scriptpath = "scripts/context/lua"
-    newname = file.addsuffix(newname,"lua")
-    local oldscript = input.clean_path(oldname)
-    input.report("to be replaced old script %s", oldscript)
-    local newscripts = input.find_files(newname) or { }
-    if #newscripts == 0 then
-        input.report("unable to locate new script")
     else
-        for _, newscript in ipairs(newscripts) do
-            newscript = input.clean_path(newscript)
-            input.report("checking new script %s", newscript)
-            if oldscript == newscript then
-                input.report("old and new script are the same")
-            elseif not newscript:find(scriptpath) then
-                input.report("new script should come from %s",scriptpath)
-            elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then
-                input.report("invalid new script name")
+        -- search spec
+        local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename)
+        if ext == "" then
+            if not instance.force_suffixes then
+                wantedfiles[#wantedfiles+1] = filename
+            end
+        else
+            wantedfiles[#wantedfiles+1] = filename
+        end
+        if instance.format == "" then
+            if ext == "" then
+                local forcedname = filename .. '.tex'
+                wantedfiles[#wantedfiles+1] = forcedname
+                filetype = resolvers.format_of_suffix(forcedname)
+                if trace_locating then
+                    logs.report("fileio",'! forcing filetype: %s',filetype)
+                end
             else
-                local newdata = io.loaddata(newscript)
-                if newdata then
-                    input.report("old script content replaced by new content")
-                    io.savedata(oldscript,newdata)
+                filetype = resolvers.format_of_suffix(filename)
+                if trace_locating then
+                    logs.report("fileio",'! using suffix based filetype: %s',filetype)
+                end
+            end
+        else
+            if ext == "" then
+                local suffixes = resolvers.suffixes_of_format(instance.format)
+                for _, s in next, suffixes do
+                    wantedfiles[#wantedfiles+1] = filename .. "." .. s
+                end
+            end
+            filetype = instance.format
+            if trace_locating then
+                logs.report("fileio",'! using given filetype: %s',filetype)
+            end
+        end
+        local typespec = resolvers.variable_of_format(filetype)
+        local pathlist = resolvers.expanded_path_list(typespec)
+        if not pathlist or #pathlist == 0 then
+            -- no pathlist, access check only / todo == wildcard
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+                logs.report("fileio",'? filetype: %s',filetype or '?')
+                logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | "))
+            end
+            for k=1,#wantedfiles do
+                local fname = wantedfiles[k]
+                if fname and resolvers.isreadable.file(fname) then
+                    filename, done = fname, true
+                    result[#result+1] = file.join('.',fname)
                     break
-                else
-                    input.report("unable to load new script")
                 end
             end
+            -- this is actually 'other text files' or 'any' or 'whatever'
+            local filelist = collect_files(wantedfiles)
+            local fl = filelist and filelist[1]
+            if fl then
+                filename = fl[3]
+                result[#result+1] = filename
+                done = true
+            end
+        else
+            -- list search
+            local filelist = collect_files(wantedfiles)
+            local doscan, recurse
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+            end
+            -- a bit messy ... esp the doscan setting here
+            for k=1,#pathlist do
+                local path = pathlist[k]
+                if find(path,"^!!") then doscan  = false else doscan  = true  end
+                if find(path,"//$") then recurse = true  else recurse = false end
+                local pathname = gsub(path,"^!+", '')
+                done = false
+                -- using file list
+                if filelist and not (done and not instance.allresults) and recurse then
+                    -- compare list entries with permitted pattern
+                    pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences
+                    pathname = gsub(pathname,"/+$", '/.*')      -- later usage of pathname
+                    pathname = gsub(pathname,"//", '/.-/')      -- not ok for /// but harmless
+                    local expr = "^" .. pathname
+                    for k=1,#filelist do
+                        local fl = filelist[k]
+                        local f = fl[2]
+                        if find(f,expr) then
+                            if trace_detail then
+                                logs.report("fileio",'= found in hash: %s',f)
+                            end
+                            --- todo, test for readable
+                            result[#result+1] = fl[3]
+                            resolvers.register_in_trees(f) -- for tracing used files
+                            done = true
+                            if not instance.allresults then break end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- check if on disk / unchecked / does not work at all / also zips
+                    if resolvers.splitmethod(pathname).scheme == 'file' then -- ?
+                        local pname = gsub(pathname,"%.%*$",'')
+                        if not find(pname,"%*") then
+                            local ppname = gsub(pname,"/+$","")
+                            if can_be_dir(ppname) then
+                                for k=1,#wantedfiles do
+                                    local w = wantedfiles[k]
+                                    local fname = file.join(ppname,w)
+                                    if resolvers.isreadable.file(fname) then
+                                        if trace_detail then
+                                            logs.report("fileio",'= found by scanning: %s',fname)
+                                        end
+                                        result[#result+1] = fname
+                                        done = true
+                                        if not instance.allresults then break end
+                                    end
+                                end
+                            else
+                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
+                            end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- todo: slow path scanning
+                end
+                if done and not instance.allresults then break end
+            end
         end
     end
+    for k=1,#result do
+        result[k] = file.collapse_path(result[k])
+    end
+    if instance.remember then
+        instance.found[stamp] = result
+    end
+    return result
 end
 
+if not resolvers.concatinators  then resolvers.concatinators = { } end
 
---~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {})))
-
--- command line resolver:
+resolvers.concatinators.tex  = file.join
+resolvers.concatinators.file = resolvers.concatinators.tex
 
---~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex"))
-
-do
-
-    local resolvers = { }
-
-    resolvers.environment = function(str)
-        return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "")
+function resolvers.find_files(filename,filetype,mustexist)
+    if type(mustexist) == boolean then
+        -- all set
+    elseif type(filetype) == 'boolean' then
+        filetype, mustexist = nil, false
+    elseif type(filetype) ~= 'string' then
+        filetype, mustexist = nil, false
     end
-    resolvers.relative = function(str,n)
-        if io.exists(str) then
-            -- nothing
-        elseif io.exists("./" .. str) then
-            str = "./" .. str
-        else
-            local p = "../"
-            for i=1,n or 2 do
-                if io.exists(p .. str) then
-                    str = p .. str
-                    break
-                else
-                    p = p .. "../"
-                end
-            end
+    instance.format = filetype or ''
+    local result = collect_instance_files(filename)
+    if #result == 0 then
+        local lowered = lower(filename)
+        if filename ~= lowered then
+            return collect_instance_files(lowered)
         end
-        return input.clean_path(str)
-    end
-    resolvers.locate = function(str)
-        local fullname = input.find_given_file(str) or ""
-        return input.clean_path((fullname ~= "" and fullname) or str)
-    end
-    resolvers.filename = function(str)
-        local fullname = input.find_given_file(str) or ""
-        return input.clean_path(file.basename((fullname ~= "" and fullname) or str))
-    end
-    resolvers.pathname = function(str)
-        local fullname = input.find_given_file(str) or ""
-        return input.clean_path(file.dirname((fullname ~= "" and fullname) or str))
     end
+    instance.format = ''
+    return result
+end
 
-    resolvers.env  = resolvers.environment
-    resolvers.rel  = resolvers.relative
-    resolvers.loc  = resolvers.locate
-    resolvers.kpse = resolvers.locate
-    resolvers.full = resolvers.locate
-    resolvers.file = resolvers.filename
-    resolvers.path = resolvers.pathname
+function resolvers.find_file(filename,filetype,mustexist)
+    return (resolvers.find_files(filename,filetype,mustexist)[1] or "")
+end
 
-    local function resolve(str)
-        if type(str) == "table" then
-            for k, v in pairs(str) do
-                str[k] = resolve(v) or v
+function resolvers.find_given_files(filename)
+    local bname, result = file.basename(filename), { }
+    local hashes = instance.hashes
+    for k=1,#hashes do
+        local hash = hashes[k]
+        local files = instance.files[hash.tag]
+        local blist = files[bname]
+        if not blist then
+            local rname = "remap:"..bname
+            blist = files[rname]
+            if blist then
+                bname = files[rname]
+                blist = files[bname]
             end
-        elseif str and str ~= "" then
-            str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target)
-                if resolvers[method] then
-                    return resolvers[method](target)
-                else
-                    return method .. ":" .. target
-                end
-            end)
         end
-        return str
-    end
-
-    if os.uname then
-        for k, v in pairs(os.uname()) do
-            if not resolvers[k] then
-                resolvers[k] = function() return v end
+        if blist then
+            if type(blist) == 'string' then
+                result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or ""
+                if not instance.allresults then break end
+            else
+                for kk=1,#blist do
+                    local vv = blist[kk]
+                    result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or ""
+                    if not instance.allresults then break end
+                end
             end
         end
     end
-
-    input.resolve = resolve
-
+    return result
 end
 
-function input.boolean_variable(str,default)
-    local b = input.expansion(str)
-    if b == "" then
-        return default
-    else
-        b = toboolean(b)
-        return (b == nil and default) or b
-    end
+function resolvers.find_given_file(filename)
+    return (resolvers.find_given_files(filename)[1] or "")
 end
 
+local function doit(path,blist,bname,tag,kind,result,allresults)
+    local done = false
+    if blist and kind then
+        if type(blist) == 'string' then
+            -- make function and share code
+            if find(lower(blist),path) then
+                result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or ""
+                done = true
+            end
+        else
+            for kk=1,#blist do
+                local vv = blist[kk]
+                if find(lower(vv),path) then
+                    result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or ""
+                    done = true
+                    if not allresults then break end
+                end
+            end
+        end
+    end
+    return done
+end
 
-if not modules then modules = { } end modules ['luat-log'] = {
-    version   = 1.001,
-    comment   = "companion to luat-lib.tex",
-    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
-    copyright = "PRAGMA ADE / ConTeXt Development Team",
-    license   = "see context related readme files"
-}
-
---[[ldx--
-
This is a prelude to a more extensive logging module. For the sake
-of parsing log files, in addition to the standard logging we will
-provide an  structured file. Actually, any logging that
-is hooked into callbacks will be \XML\ by default.

---ldx]]--
-
--- input.logger -> special tracing, driven by log level (only input)
--- input.report -> goes to terminal, depends on verbose, has banner
--- logs.report  -> module specific tracing and reporting, no banner but class
-
-
-input = input or { }
-logs  = logs  or { }
-
---[[ldx--
-
This looks pretty ugly but we need to speed things up a bit.

---ldx]]--
-
-logs.levels = {
-    ['error']   = 1,
-    ['warning'] = 2,
-    ['info']    = 3,
-    ['debug']   = 4
-}
-
-logs.functions = {
-    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct'
-}
-
-logs.callbacks  = {
-    'start_page_number',
-    'stop_page_number',
-    'report_output_pages',
-    'report_output_log'
-}
-
-logs.tracers = {
-}
-
-logs.xml = logs.xml or { }
-logs.tex = logs.tex or { }
+function resolvers.find_wildcard_files(filename) -- todo: remap:
+    local result = { }
+    local bname, dname = file.basename(filename), file.dirname(filename)
+    local path = gsub(dname,"^*/","")
+    path = gsub(path,"*",".*")
+    path = gsub(path,"-","%%-")
+    if dname == "" then
+        path = ".*"
+    end
+    local name = bname
+    name = gsub(name,"*",".*")
+    name = gsub(name,"-","%%-")
+    path = lower(path)
+    name = lower(name)
+    local files, allresults, done = instance.files, instance.allresults, false
+    if find(name,"%*") then
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            for kk, hh in next, files[hash.tag] do
+                if not find(kk,"^remap:") then
+                    if find(lower(kk),name) then
+                        if doit(path,hh,kk,tag,kind,result,allresults) then done = true end
+                        if done and not allresults then break end
+                    end
+                end
+            end
+        end
+    else
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end
+            if done and not allresults then break end
+        end
+    end
+    -- we can consider also searching the paths not in the database, but then
+    -- we end up with a messy search (all // in all path specs)
+    return result
+end
 
-logs.level = 0
+function resolvers.find_wildcard_file(filename)
+    return (resolvers.find_wildcard_files(filename)[1] or "")
+end
 
-local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+-- main user functions
 
-if texlua then
-    write_nl = print
-    write    = io.write
+function resolvers.automount()
+    -- implemented later
 end
 
-function logs.xml.report(category,fmt,...) -- new
-    write_nl(format("%s",category,format(fmt,...)))
+function resolvers.load(option)
+    statistics.starttiming(instance)
+    resolvers.resetconfig()
+    resolvers.identify_cnf()
+    resolvers.load_lua()
+    resolvers.expand_variables()
+    resolvers.load_cnf()
+    resolvers.expand_variables()
+    if option ~= "nofiles" then
+        resolvers.load_hash()
+        resolvers.automount()
+    end
+    statistics.stoptiming(instance)
 end
-function logs.xml.line(fmt,...) -- new
-    write_nl(format("%s",format(fmt,...)))
+
+function resolvers.for_files(command, files, filetype, mustexist)
+    if files and #files > 0 then
+        local function report(str)
+            if trace_verbose then
+                logs.report("fileio",str) -- has already verbose
+            else
+                print(str)
+            end
+        end
+        if trace_verbose then
+            report('')
+        end
+        for _, file in ipairs(files) do
+            local result = command(file,filetype,mustexist)
+            if type(result) == 'string' then
+                report(result)
+            else
+                for _,v in ipairs(result) do
+                    report(v)
+                end
+            end
+        end
+    end
 end
 
-function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
-function logs.xml.stop () if logs.level > 0 then tw("") end end
-function logs.xml.push () if logs.level > 0 then tw("" ) end end
+-- strtab
 
-function logs.tex.report(category,fmt,...) -- new
- -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. .
-    write_nl(category .. " | " .. format(fmt,...))
-end
-function logs.tex.line(fmt,...) -- new
-    write_nl(format(fmt,...))
+resolvers.var_value  = resolvers.variable   -- output the value of variable $STRING.
+resolvers.expand_var = resolvers.expansion  -- output variable expansion of STRING.
+
+function resolvers.show_path(str)     -- output search path for file type NAME
+    return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str)))
 end
 
-function logs.set_level(level)
-    logs.level = logs.levels[level] or level
+-- resolvers.find_file(filename)
+-- resolvers.find_file(filename, filetype, mustexist)
+-- resolvers.find_file(filename, mustexist)
+-- resolvers.find_file(filename, filetype)
+
+function resolvers.register_file(files, name, path)
+    if files[name] then
+        if type(files[name]) == 'string' then
+            files[name] = { files[name], path }
+        else
+            files[name] = path
+        end
+    else
+        files[name] = path
+    end
 end
 
-function logs.set_method(method)
-    for _, v in pairs(logs.functions) do
-        logs[v] = logs[method][v] or function() end
+function resolvers.splitmethod(filename)
+    if not filename then
+        return { } -- safeguard
+    elseif type(filename) == "table" then
+        return filename -- already split
+    elseif not find(filename,"://") then
+        return { scheme="file", path = filename, original=filename } -- quick hack
+    else
+        return url.hashed(filename)
     end
-    if callback and input[method] then
-        for _, cb in pairs(logs.callbacks) do
-            callback.register(cb, input[method][cb])
-        end
+end
+
+function table.sequenced(t,sep) -- temp here
+    local s = { }
+    for k, v in pairs(t) do -- pairs?
+        s[#s+1] = k .. "=" .. v
     end
+    return concat(s, sep or " | ")
 end
 
-function logs.xml.start_page_number()
-    write_nl(format("
 %s -> %s',specification.original,what,table.sequenced(specification))
+        end
+        return resolvers[what][scheme](filename,filetype) -- todo: specification
+    else
+        return resolvers[what].tex(filename,filetype) -- todo: specification
+    end
 end
 
-function logs.xml.stop_page_number()
-    write("/>")
-    write_nl("")
+function resolvers.clean_path(str)
+    if str then
+        str = gsub(str,"\\","/")
+        str = gsub(str,"^!+","")
+        str = gsub(str,"^~",resolvers.homedir)
+        return str
+    else
+        return nil
+    end
 end
 
-function logs.xml.report_output_pages(p,b)
-    write_nl(format("", p))
-    write_nl(format("", b))
-    write_nl("")
+function resolvers.do_with_path(name,func)
+    for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs?
+        func("^"..resolvers.clean_path(v))
+    end
 end
 
-function logs.xml.report_output_log()
+function resolvers.do_with_var(name,func)
+    func(expanded_var(name))
 end
 
-function input.logger(...) -- assumes test for input.trace > n
-    if input.trace > 0 then
-        logs.report(...)
+function resolvers.with_files(pattern,handle)
+    for _, hash in ipairs(instance.hashes) do
+        local blobpath = hash.tag
+        local blobtype = hash.type
+        if blobpath then
+            local files = instance.files[blobpath]
+            if files then
+                for k,v in next, files do
+                    if find(k,"^remap:") then
+                        k = files[k]
+                        v = files[k] -- chained
+                    end
+                    if find(k,pattern) then
+                        if type(v) == "string" then
+                            handle(blobtype,blobpath,v,k)
+                        else
+                            for _,vv in pairs(v) do -- ipairs?
+                                handle(blobtype,blobpath,vv,k)
+                            end
+                        end
+                    end
+                end
+            end
+        end
     end
 end
 
-function input.report(fmt,...)
-    if input.verbose then
-        logs.report(input.banner or "report",format(fmt,...))
+function resolvers.locate_format(name)
+    local barename, fmtname = name:gsub("%.%a+$",""), ""
+    if resolvers.usecache then
+        local path = file.join(caches.setpath("formats")) -- maybe platform
+        fmtname = file.join(path,barename..".fmt") or ""
+    end
+    if fmtname == "" then
+        fmtname = resolvers.find_files(barename..".fmt")[1] or ""
     end
+    fmtname = resolvers.clean_path(fmtname)
+    if fmtname ~= "" then
+        local barename = file.removesuffix(fmtname)
+        local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui"
+        if lfs.isfile(luiname) then
+            return barename, luiname
+        elseif lfs.isfile(lucname) then
+            return barename, lucname
+        elseif lfs.isfile(luaname) then
+            return barename, luaname
+        end
+    end
+    return nil, nil
 end
 
-function input.reportlines(str) -- todo: 
-    for line in str:gmatch("(.-)[\n\r]") do
-        logs.report(input.banner or "report",line)
+function resolvers.boolean_variable(str,default)
+    local b = resolvers.expansion(str)
+    if b == "" then
+        return default
+    else
+        b = toboolean(b)
+        return (b == nil and default) or b
     end
 end
 
-input.moreinfo = [[
-more information about ConTeXt and the tools that come with it can be found at:
+texconfig.kpse_init = false
 
-maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
-webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
-wiki     : http://contextgarden.net
-]]
+kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } )
 
-function input.help(banner,message)
-    if not input.verbose then
-        input.verbose = true
-    --  input.report(banner,"\n")
-    end
-    input.report(banner,"\n")
-    input.report("")
-    input.reportlines(message)
-    if input.moreinfo and input.moreinfo ~= "" then
-        input.report("")
-        input.reportlines(input.moreinfo)
-    end
-end
+-- for a while
 
-logs.set_level('error')
-logs.set_method('tex')
+input = resolvers
+
+
+end -- of closure
 
+do -- create closure to overcome 200 locals limit
 
-if not modules then modules = { } end modules ['luat-tmp'] = {
+if not modules then modules = { } end modules ['data-tmp'] = {
     version   = 1.001,
     comment   = "companion to luat-lib.tex",
     author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
@@ -7417,17 +8171,16 @@ being written at the same time is small. We also need to extend
 luatools with a recache feature.

 --ldx]]--
 
-local format = string.format
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache = false  trackers.register("resolvers.cache", function(v) trace_cache = v end)
 
 caches = caches or { }
-dir    = dir    or { }
-texmf  = texmf  or { }
 
 caches.path     = caches.path or nil
 caches.base     = caches.base or "luatex-cache"
 caches.more     = caches.more or "context"
 caches.direct   = false -- true is faster but may need huge amounts of memory
-caches.trace    = false
 caches.tree     = false
 caches.paths    = caches.paths or nil
 caches.force    = false
@@ -7437,13 +8190,14 @@ function caches.temp()
     local cachepath = nil
     local function check(list,isenv)
         if not cachepath then
-            for _, v in ipairs(list) do
+            for k=1,#list do
+                local v = list[k]
                 cachepath = (isenv and (os.env[v] or "")) or v or ""
                 if cachepath == "" then
                     -- next
                 else
-                    cachepath = input.clean_path(cachepath)
-                     if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory"
+                    cachepath = resolvers.clean_path(cachepath)
+                    if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory"
                         break
                     elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
                         dir.mkdirs(cachepath)
@@ -7456,7 +8210,7 @@ function caches.temp()
             end
         end
     end
-    check(input.clean_path_list("TEXMFCACHE") or { })
+    check(resolvers.clean_path_list("TEXMFCACHE") or { })
     check(caches.defaults,true)
     if not cachepath then
         print("\nfatal error: there is no valid (writable) cache path defined\n")
@@ -7465,7 +8219,7 @@ function caches.temp()
         print(format("\nfatal error: cache path %s is not a directory\n",cachepath))
         os.exit()
     end
-    cachepath = input.normalize_name(cachepath)
+    cachepath = file.collapse_path(cachepath)
     function caches.temp()
         return cachepath
     end
@@ -7473,24 +8227,13 @@ function caches.temp()
 end
 
 function caches.configpath()
-    return table.concat(input.instance.cnffiles,";")
+    return table.concat(resolvers.instance.cnffiles,";")
 end
 
 function caches.hashed(tree)
-    return md5.hex((tree:lower()):gsub("[\\\/]+","/"))
+    return md5.hex(gsub(lower(tree),"[\\\/]+","/"))
 end
 
---~ tracing:
-
---~ function caches.hashed(tree)
---~     tree = (tree:lower()):gsub("[\\\/]+","/")
---~     local hash = md5.hex(tree)
---~     if input.verbose then -- temp message
---~         input.report("hashing %s => %s",tree,hash)
---~     end
---~     return hash
---~ end
-
 function caches.treehash()
     local tree = caches.configpath()
     if not tree or tree == "" then
@@ -7505,21 +8248,19 @@ function caches.setpath(...)
         if not caches.path then
             caches.path = caches.temp()
         end
-        caches.path = input.clean_path(caches.path) -- to be sure
-        if lfs then
-            caches.tree = caches.tree or caches.treehash()
-            if caches.tree then
-                caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
-            else
-                caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
-            end
+        caches.path = resolvers.clean_path(caches.path) -- to be sure
+        caches.tree = caches.tree or caches.treehash()
+        if caches.tree then
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
+        else
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
         end
     end
     if not caches.path then
         caches.path = '.'
     end
-    caches.path = input.clean_path(caches.path)
-    if lfs and not table.is_empty({...}) then
+    caches.path = resolvers.clean_path(caches.path)
+    if not table.is_empty({...}) then
         local pth = dir.mkdirs(caches.path,...)
         return pth
     end
@@ -7547,9 +8288,14 @@ function caches.loaddata(path,name)
     end
 end
 
-function caches.is_writable(filepath,filename)
+--~ function caches.loaddata(path,name)
+--~     local tmaname, tmcname = caches.setluanames(path,name)
+--~     return dofile(tmcname) or dofile(tmaname)
+--~ end
+
+function caches.iswritable(filepath,filename)
     local tmaname, tmcname = caches.setluanames(filepath,filename)
-    return file.is_writable(tmaname)
+    return file.iswritable(tmaname)
 end
 
 function caches.savedata(filepath,filename,data,raw)
@@ -7559,23 +8305,177 @@ function caches.savedata(filepath,filename,data,raw)
         reduce, simplify = false, false
     end
     if caches.direct then
-        file.savedata(tmaname, table.serialize(data,'return',true,true,false)) -- no hex
+        file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex
     else
-        table.tofile(tmaname, data,'return',true,true,false) -- maybe not the last true
+        table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true
     end
-    local cleanup = input.boolean_variable("PURGECACHE", false)
-    local strip = input.boolean_variable("LUACSTRIP", true)
+    local cleanup = resolvers.boolean_variable("PURGECACHE", false)
+    local strip = resolvers.boolean_variable("LUACSTRIP", true)
     utils.lua.compile(tmaname, tmcname, cleanup, strip)
 end
 
 -- here we use the cache for format loading (texconfig.[formatname|jobname])
 
 --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then
-if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and input.instance then
+if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then
     if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc
-    texconfig.formatname = caches.setpath("formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt")
+    texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt")
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-res'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex"))
+
+local upper, lower, gsub = string.upper, string.lower, string.gsub
+
+local prefixes = { }
+
+prefixes.environment = function(str)
+    return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "")
+end
+
+prefixes.relative = function(str,n)
+    if io.exists(str) then
+        -- nothing
+    elseif io.exists("./" .. str) then
+        str = "./" .. str
+    else
+        local p = "../"
+        for i=1,n or 2 do
+            if io.exists(p .. str) then
+                str = p .. str
+                break
+            else
+                p = p .. "../"
+            end
+        end
+    end
+    return resolvers.clean_path(str)
+end
+
+prefixes.locate = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path((fullname ~= "" and fullname) or str)
+end
+
+prefixes.filename = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str))
+end
+
+prefixes.pathname = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str))
+end
+
+prefixes.env  = prefixes.environment
+prefixes.rel  = prefixes.relative
+prefixes.loc  = prefixes.locate
+prefixes.kpse = prefixes.locate
+prefixes.full = prefixes.locate
+prefixes.file = prefixes.filename
+prefixes.path = prefixes.pathname
+
+local function _resolve_(method,target)
+    if prefixes[method] then
+        return prefixes[method](target)
+    else
+        return method .. ":" .. target
+    end
+end
+
+local function resolve(str)
+    if type(str) == "table" then
+        for k, v in pairs(str) do -- ipairs
+            str[k] = resolve(v) or v
+        end
+    elseif str and str ~= "" then
+        str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_)
+    end
+    return str
+end
+
+resolvers.resolve = resolve
+
+if os.uname then
+
+    for k, v in pairs(os.uname()) do
+        if not prefixes[k] then
+            prefixes[k] = function() return v end
+        end
+    end
+
 end
 
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+resolvers.finders = resolvers.finders or { }
+resolvers.openers = resolvers.openers or { }
+resolvers.loaders = resolvers.loaders or { }
+
+resolvers.finders.notfound  = { nil }
+resolvers.openers.notfound  = { nil }
+resolvers.loaders.notfound  = { false, nil, 0 }
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-out'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+outputs = outputs or { }
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-con'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache      = false  trackers.register("resolvers.cache",      function(v) trace_cache      = v end)
+local trace_containers = false  trackers.register("resolvers.containers", function(v) trace_containers = v end)
+local trace_storage    = false  trackers.register("resolvers.storage",    function(v) trace_storage    = v end)
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
 --[[ldx--
 
Once we found ourselves defining similar cache constructs
 several times, containers were introduced. Containers are used
@@ -7589,126 +8489,145 @@ table structures without bothering about the disk cache.

 
Examples of usage can be found in the font related code.

 --ldx]]--
 
-containers       = { }
-containers.trace = false
+containers = containers or { }
 
-do -- local report
+containers.usecache = true
 
-    local function report(container,tag,name)
-        if caches.trace or containers.trace or container.trace then
-            logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid')
-        end
+local function report(container,tag,name)
+    if trace_cache or trace_containers then
+        logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid')
     end
+end
 
-    local allocated = { }
+local allocated = { }
 
-    -- tracing
+-- tracing
 
-    function containers.define(category, subcategory, version, enabled)
-        return function()
-            if category and subcategory then
-                local c = allocated[category]
-                if not c then
-                    c  = { }
-                    allocated[category] = c
-                end
-                local s = c[subcategory]
-                if not s then
-                    s = {
-                        category = category,
-                        subcategory = subcategory,
-                        storage = { },
-                        enabled = enabled,
-                        version = version or 1.000,
-                        trace = false,
-                        path = caches.setpath(category,subcategory),
-                    }
-                    c[subcategory] = s
-                end
-                return s
-            else
-                return nil
-            end
+function containers.define(category, subcategory, version, enabled)
+    return function()
+        if category and subcategory then
+            local c = allocated[category]
+            if not c then
+                c  = { }
+                allocated[category] = c
+            end
+            local s = c[subcategory]
+            if not s then
+                s = {
+                    category = category,
+                    subcategory = subcategory,
+                    storage = { },
+                    enabled = enabled,
+                    version = version or 1.000,
+                    trace = false,
+                    path = caches and caches.setpath and caches.setpath(category,subcategory),
+                }
+                c[subcategory] = s
+            end
+            return s
+        else
+            return nil
         end
     end
+end
+
+function containers.is_usable(container, name)
+    return container.enabled and caches and caches.iswritable(container.path, name)
+end
 
-    function containers.is_usable(container, name)
-        return container.enabled and caches.is_writable(container.path, name)
+function containers.is_valid(container, name)
+    if name and name ~= "" then
+        local storage = container.storage[name]
+        return storage and not table.is_empty(storage) and storage.cache_version == container.version
+    else
+        return false
     end
+end
 
-    function containers.is_valid(container, name)
-        if name and name ~= "" then
-            local storage = container.storage[name]
-            return storage and not table.is_empty(storage) and storage.cache_version == container.version
+function containers.read(container,name)
+    if container.enabled and caches and not container.storage[name] and containers.usecache then
+        container.storage[name] = caches.loaddata(container.path,name)
+        if containers.is_valid(container,name) then
+            report(container,"loaded",name)
         else
-            return false
+            container.storage[name] = nil
         end
     end
-
-    function containers.read(container,name)
-        if container.enabled and not container.storage[name] then
-            container.storage[name] = caches.loaddata(container.path,name)
-            if containers.is_valid(container,name) then
-                report(container,"loaded",name)
-            else
-                container.storage[name] = nil
-            end
-        end
-        if container.storage[name] then
-            report(container,"reusing",name)
-        end
-        return container.storage[name]
+    if container.storage[name] then
+        report(container,"reusing",name)
     end
+    return container.storage[name]
+end
 
-    function containers.write(container, name, data)
-        if data then
-            data.cache_version = container.version
-            if container.enabled then
-                local unique, shared = data.unique, data.shared
-                data.unique, data.shared = nil, nil
-                caches.savedata(container.path, name, data)
-                report(container,"saved",name)
-                data.unique, data.shared = unique, shared
-            end
-            report(container,"stored",name)
-            container.storage[name] = data
+function containers.write(container, name, data)
+    if data then
+        data.cache_version = container.version
+        if container.enabled and caches then
+            local unique, shared = data.unique, data.shared
+            data.unique, data.shared = nil, nil
+            caches.savedata(container.path, name, data)
+            report(container,"saved",name)
+            data.unique, data.shared = unique, shared
         end
-        return data
+        report(container,"stored",name)
+        container.storage[name] = data
     end
+    return data
+end
 
-    function containers.content(container,name)
-        return container.storage[name]
-    end
+function containers.content(container,name)
+    return container.storage[name]
+end
 
+function containers.cleanname(name)
+    return (gsub(lower(name),"[^%w%d]+","-"))
 end
 
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-use'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
 -- since we want to use the cache instead of the tree, we will now
 -- reimplement the saver.
 
-local save_data = input.aux.save_data
-local load_data = input.aux.load_data
+local save_data = resolvers.save_data
+local load_data = resolvers.load_data
 
-input.cachepath = nil  -- public, for tracing
-input.usecache  = true -- public, for tracing
+resolvers.cachepath = nil  -- public, for tracing
+resolvers.usecache  = true -- public, for tracing
 
-function input.aux.save_data(dataname, check)
-    save_data(dataname, check, function(cachename,dataname)
-        input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true)
-        if input.usecache then
-            input.cachepath = input.cachepath or caches.definepath("trees")
-            return file.join(input.cachepath(),caches.hashed(cachename))
+function resolvers.save_data(dataname)
+    save_data(dataname, function(cachename,dataname)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(cachename))
         else
             return file.join(cachename,dataname)
         end
     end)
 end
 
-function input.aux.load_data(pathname,dataname,filename)
+function resolvers.load_data(pathname,dataname,filename)
     load_data(pathname,dataname,filename,function(dataname,filename)
-        input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true)
-        if input.usecache then
-            input.cachepath = input.cachepath or caches.definepath("trees")
-            return file.join(input.cachepath(),caches.hashed(pathname))
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(pathname))
         else
             if not filename or (filename == "") then
                 filename = dataname
@@ -7720,15 +8639,15 @@ end
 
 -- we will make a better format, maybe something xml or just text or lua
 
-input.automounted = input.automounted or { }
+resolvers.automounted = resolvers.automounted or { }
 
-function input.automount(usecache)
-    local mountpaths = input.clean_path_list(input.expansion('TEXMFMOUNT'))
+function resolvers.automount(usecache)
+    local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT'))
     if table.is_empty(mountpaths) and usecache then
         mountpaths = { caches.setpath("mount") }
     end
     if not table.is_empty(mountpaths) then
-        input.starttiming(input.instance)
+        statistics.starttiming(resolvers.instance)
         for k, root in pairs(mountpaths) do
             local f = io.open(root.."/url.tmi")
             if f then
@@ -7737,98 +8656,72 @@ function input.automount(usecache)
                         if line:find("^[%%#%-]") then -- or %W
                             -- skip
                         elseif line:find("^zip://") then
-                            input.report("mounting %s",line)
-                            table.insert(input.automounted,line)
-                            input.usezipfile(line)
+                            if trace_locating then
+                                logs.report("fileio","mounting %s",line)
+                            end
+                            table.insert(resolvers.automounted,line)
+                            resolvers.usezipfile(line)
                         end
                     end
                 end
                 f:close()
             end
         end
-        input.stoptiming(input.instance)
+        statistics.stoptiming(resolvers.instance)
     end
 end
 
--- store info in format
-
-input.storage            = { }
-input.storage.data       = { }
-input.storage.min        = 0 -- 500
-input.storage.max        = input.storage.min - 1
-input.storage.trace      = false -- true
-input.storage.done       = input.storage.done or 0
-input.storage.evaluators = { }
--- (evaluate,message,names)
+-- status info
 
-function input.storage.register(...)
-    input.storage.data[#input.storage.data+1] = { ... }
-end
+statistics.register("used config path", function() return caches.configpath()  end)
+statistics.register("used cache path",  function() return caches.temp() or "?" end)
 
-function input.storage.evaluate(name)
-    input.storage.evaluators[#input.storage.evaluators+1] = name
-end
+-- experiment (code will move)
 
-function input.storage.finalize() -- we can prepend the string with "evaluate:"
-    for _, t in ipairs(input.storage.evaluators) do
-        for i, v in pairs(t) do
-            if type(v) == "string" then
-                t[i] = loadstring(v)()
-            elseif type(v) == "table" then
-                for _, vv in pairs(v) do
-                    if type(vv) == "string" then
-                        t[i] = loadstring(vv)()
-                    end
-                end
-            end
-        end
+function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname
+    local enginebanner = status.list().banner
+    if formatbanner and enginebanner and sourcefile then
+        local luvname = file.replacesuffix(texname,"luv")
+        local luvdata = {
+            enginebanner = enginebanner,
+            formatbanner = formatbanner,
+            sourcehash   = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"),
+            sourcefile   = sourcefile,
+        }
+        io.savedata(luvname,table.serialize(luvdata,true))
     end
 end
 
-function input.storage.dump()
-    for name, data in ipairs(input.storage.data) do
-        local evaluate, message, original, target = data[1], data[2], data[3] ,data[4]
-        local name, initialize, finalize, code = nil, "", "", ""
-        for str in target:gmatch("([^%.]+)") do
-            if name then
-                name = name .. "." .. str
+function statistics.check_fmt_status(texname)
+    local enginebanner = status.list().banner
+    if enginebanner and texname then
+        local luvname = file.replacesuffix(texname,"luv")
+        if lfs.isfile(luvname) then
+            local luv = dofile(luvname)
+            if luv and luv.sourcefile then
+                local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown")
+                if luv.enginebanner and luv.enginebanner ~= enginebanner then
+                    return "engine mismatch"
+                end
+                if luv.sourcehash and luv.sourcehash ~= sourcehash then
+                    return "source mismatch"
+                end
             else
-                name = str
+                return "invalid status file"
             end
-            initialize = format("%s %s = %s or {} ", initialize, name, name)
-        end
-        if evaluate then
-            finalize = "input.storage.evaluate(" .. name .. ")"
-        end
-        input.storage.max = input.storage.max + 1
-        if input.storage.trace then
-            logs.report('storage','saving %s in slot %s',message,input.storage.max)
-            code =
-                initialize ..
-                format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) ..
-                table.serialize(original,name) ..
-                finalize
         else
-            code = initialize .. table.serialize(original,name) .. finalize
+            return "missing status file"
         end
-        lua.bytecode[input.storage.max] = loadstring(code)
     end
+    return true
 end
 
--- we also need to count at generation time (nicer for message)
 
-if lua.bytecode then -- from 0 upwards
-    local i = input.storage.min
-    while lua.bytecode[i] do
-        lua.bytecode[i]()
-        lua.bytecode[i] = nil
-        i = i + 1
-    end
-    input.storage.done = i
-end
+end -- of closure
 
+do -- create closure to overcome 200 locals limit
 
-if not modules then modules = { } end modules ['luat-log'] = {
+if not modules then modules = { } end modules ['data-zip'] = {
     version   = 1.001,
     comment   = "companion to luat-lib.tex",
     author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
@@ -7836,154 +8729,556 @@ if not modules then modules = { } end modules ['luat-log'] = {
     license   = "see context related readme files"
 }
 
---[[ldx--
-
This is a prelude to a more extensive logging module. For the sake
-of parsing log files, in addition to the standard logging we will
-provide an  structured file. Actually, any logging that
-is hooked into callbacks will be \XML\ by default.

---ldx]]--
+local format, find = string.format, string.find
 
--- input.logger -> special tracing, driven by log level (only input)
--- input.report -> goes to terminal, depends on verbose, has banner
--- logs.report  -> module specific tracing and reporting, no banner but class
+local trace_locating, trace_verbose  = false, false
 
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end)
 
-input = input or { }
-logs  = logs  or { }
+zip                 = zip or { }
+zip.archives        = zip.archives or { }
+zip.registeredfiles = zip.registeredfiles or { }
 
---[[ldx--
-
This looks pretty ugly but we need to speed things up a bit.

---ldx]]--
+local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
+local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators
 
-logs.levels = {
-    ['error']   = 1,
-    ['warning'] = 2,
-    ['info']    = 3,
-    ['debug']   = 4
-}
+local archives = zip.archives
 
-logs.functions = {
-    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct'
-}
+-- zip:///oeps.zip?name=bla/bla.tex
+-- zip:///oeps.zip?tree=tex/texmf-local
 
-logs.callbacks  = {
-    'start_page_number',
-    'stop_page_number',
-    'report_output_pages',
-    'report_output_log'
-}
+local function validzip(str) -- todo: use url splitter
+    if not find(str,"^zip://") then
+        return "zip:///" .. str
+    else
+        return str
+    end
+end
 
-logs.tracers = {
-}
+function zip.openarchive(name)
+    if not name or name == "" then
+        return nil
+    else
+        local arch = archives[name]
+        if not arch then
+           local full = resolvers.find_file(name) or ""
+           arch = (full ~= "" and zip.open(full)) or false
+           archives[name] = arch
+        end
+       return arch
+    end
+end
 
-logs.xml = logs.xml or { }
-logs.tex = logs.tex or { }
+function zip.closearchive(name)
+    if not name or (name == "" and archives[name]) then
+        zip.close(archives[name])
+        archives[name] = nil
+    end
+end
 
-logs.level = 0
+-- zip:///texmf.zip?tree=/tex/texmf
+-- zip:///texmf.zip?tree=/tex/texmf-local
+-- zip:///texmf-mine.zip?tree=/tex/texmf-projects
 
-local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+function locators.zip(specification) -- where is this used? startup zips (untested)
+    specification = resolvers.splitmethod(specification)
+    local zipfile = specification.path
+    local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree
+    if trace_locating then
+        if zfile then
+            logs.report("fileio",'! zip locator, found: %s',specification.original)
+        else
+            logs.report("fileio",'? zip locator, not found: %s',specification.original)
+        end
+    end
+end
 
-if texlua then
-    write_nl = print
-    write    = io.write
+function hashers.zip(tag,name)
+    if trace_verbose then
+        logs.report("fileio","loading zip file %s as %s",name,tag)
+    end
+    resolvers.usezipfile(format("%s?tree=%s",tag,name))
 end
 
-function logs.xml.report(category,fmt,...) -- new
-    write_nl(format("%s",category,format(fmt,...)))
+function concatinators.zip(tag,path,name)
+    if not path or path == "" then
+        return format('%s?name=%s',tag,name)
+    else
+        return format('%s?name=%s/%s',tag,path,name)
+    end
 end
-function logs.xml.line(fmt,...) -- new
-    write_nl(format("%s",format(fmt,...)))
+
+function resolvers.isreadable.zip(name)
+    return true
 end
 
-function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
-function logs.xml.stop () if logs.level > 0 then tw("") end end
-function logs.xml.push () if logs.level > 0 then tw("" ) end end
+function finders.zip(specification,filetype)
+    specification = resolvers.splitmethod(specification)
+    if specification.path then
+        local q = url.query(specification.query)
+        if q.name then
+            local zfile = zip.openarchive(specification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'! zip finder, path: %s',specification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    dfile = zfile:close()
+                    if trace_locating then
+                        logs.report("fileio",'+ zip finder, name: %s',q.name)
+                    end
+                    return specification.original
+                end
+            elseif trace_locating then
+                logs.report("fileio",'? zip finder, path %s',specification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip finder, name: %s',filename)
+    end
+    return unpack(finders.notfound)
+end
 
-function logs.tex.report(category,fmt,...) -- new
- -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. .
-    write_nl(category .. " | " .. format(fmt,...))
+function openers.zip(specification)
+    local zipspecification = resolvers.splitmethod(specification)
+    if zipspecification.path then
+        local q = url.query(zipspecification.query)
+        if q.name then
+            local zfile = zip.openarchive(zipspecification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'+ zip starter, path: %s',zipspecification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    logs.show_open(specification)
+                    return openers.text_opener(specification,dfile,'zip')
+                end
+            elseif trace_locating then
+                logs.report("fileio",'- zip starter, path %s',zipspecification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip opener, name: %s',filename)
+    end
+    return unpack(openers.notfound)
 end
-function logs.tex.line(fmt,...) -- new
-    write_nl(format(fmt,...))
+
+function loaders.zip(specification)
+    specification = resolvers.splitmethod(specification)
+    if specification.path then
+        local q = url.query(specification.query)
+        if q.name then
+            local zfile = zip.openarchive(specification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'+ zip starter, path: %s',specification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    logs.show_load(filename)
+                    if trace_locating then
+                        logs.report("fileio",'+ zip loader, name: %s',filename)
+                    end
+                    local s = dfile:read("*all")
+                    dfile:close()
+                    return true, s, #s
+                end
+            elseif trace_locating then
+                logs.report("fileio",'- zip starter, path: %s',specification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip loader, name: %s',filename)
+    end
+    return unpack(openers.notfound)
 end
 
-function logs.set_level(level)
-    logs.level = logs.levels[level] or level
+-- zip:///somefile.zip
+-- zip:///somefile.zip?tree=texmf-local -> mount
+
+function resolvers.usezipfile(zipname)
+    zipname = validzip(zipname)
+    if trace_locating then
+        logs.report("fileio",'! zip use, file: %s',zipname)
+    end
+    local specification = resolvers.splitmethod(zipname)
+    local zipfile = specification.path
+    if zipfile and not zip.registeredfiles[zipname] then
+        local tree = url.query(specification.query).tree or ""
+        if trace_locating then
+            logs.report("fileio",'! zip register, file: %s',zipname)
+        end
+        local z = zip.openarchive(zipfile)
+        if z then
+            local instance = resolvers.instance
+            if trace_locating then
+                logs.report("fileio","= zipfile, registering: %s",zipname)
+            end
+            statistics.starttiming(instance)
+            resolvers.prepend_hash('zip',zipname,zipfile)
+            resolvers.extend_texmf_var(zipname) -- resets hashes too
+            zip.registeredfiles[zipname] = z
+            instance.files[zipname] = resolvers.register_zip_file(z,tree or "")
+            statistics.stoptiming(instance)
+        elseif trace_locating then
+            logs.report("fileio","? zipfile, unknown: %s",zipname)
+        end
+    elseif trace_locating then
+        logs.report("fileio",'! zip register, no file: %s',zipname)
+    end
 end
 
-function logs.set_method(method)
-    for _, v in pairs(logs.functions) do
-        logs[v] = logs[method][v] or function() end
+function resolvers.register_zip_file(z,tree)
+    local files, filter = { }, ""
+    if tree == "" then
+        filter = "^(.+)/(.-)$"
+    else
+        filter = format("^%s/(.+)/(.-)$",tree)
     end
-    if callback and input[method] then
-        for _, cb in pairs(logs.callbacks) do
-            callback.register(cb, input[method][cb])
+    if trace_locating then
+        logs.report("fileio",'= zip filter: %s',filter)
+    end
+    local register, n = resolvers.register_file, 0
+    for i in z:files() do
+        local path, name = i.filename:match(filter)
+        if path then
+            if name and name ~= '' then
+                register(files, name, path)
+                n = n + 1
+            else
+                -- directory
+            end
+        else
+            register(files, i.filename, '')
+            n = n + 1
         end
     end
+    logs.report("fileio",'= zip entries: %s',n)
+    return files
 end
 
-function logs.xml.start_page_number()
-    write_nl(format("
")
-    write_nl("")
+function finders.curl(protocol,filename)
+    local foundname = curl.fetch(protocol, filename)
+    return finders.generic(protocol,foundname,filetype)
 end
 
-function logs.xml.report_output_pages(p,b)
-    write_nl(format("", p))
-    write_nl(format("", b))
-    write_nl("")
+function openers.curl(protocol,filename)
+    return openers.generic(protocol,filename)
 end
 
-function logs.xml.report_output_log()
+function loaders.curl(protocol,filename)
+    return loaders.generic(protocol,filename)
 end
 
-function input.logger(...) -- assumes test for input.trace > n
-    if input.trace > 0 then
-        logs.report(...)
+-- todo: metamethod
+
+function curl.install(protocol)
+    finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end
+    openers[protocol] = function (filename)          return openers.curl(protocol,filename) end
+    loaders[protocol] = function (filename)          return loaders.curl(protocol,filename) end
+end
+
+curl.install('http')
+curl.install('https')
+curl.install('ftp')
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-kps'] = {
+    version   = 1.001,
+    comment   = "companion to luatools.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This file is used when we want the input handlers to behave like
+kpsewhich. What to do with the following:

+
+
+{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}
+$SELFAUTOLOC    : /usr/tex/bin/platform
+$SELFAUTODIR    : /usr/tex/bin
+$SELFAUTOPARENT : /usr/tex
+
+
+
How about just forgetting about them?

+--ldx]]--
+
+local suffixes = resolvers.suffixes
+local formats  = resolvers.formats
+
+suffixes['gf']                       = { 'gf' }
+suffixes['pk']                       = { 'pk' }
+suffixes['base']                     = { 'base' }
+suffixes['bib']                      = { 'bib' }
+suffixes['bst']                      = { 'bst' }
+suffixes['cnf']                      = { 'cnf' }
+suffixes['mem']                      = { 'mem' }
+suffixes['mf']                       = { 'mf' }
+suffixes['mfpool']                   = { 'pool' }
+suffixes['mft']                      = { 'mft' }
+suffixes['mppool']                   = { 'pool' }
+suffixes['graphic/figure']           = { 'eps', 'epsi' }
+suffixes['texpool']                  = { 'pool' }
+suffixes['PostScript header']        = { 'pro' }
+suffixes['ist']                      = { 'ist' }
+suffixes['web']                      = { 'web', 'ch' }
+suffixes['cweb']                     = { 'w', 'web', 'ch' }
+suffixes['cmap files']               = { 'cmap' }
+suffixes['lig files']                = { 'lig' }
+suffixes['bitmap font']              = { }
+suffixes['MetaPost support']         = { }
+suffixes['TeX system documentation'] = { }
+suffixes['TeX system sources']       = { }
+suffixes['dvips config']             = { }
+suffixes['type42 fonts']             = { }
+suffixes['web2c files']              = { }
+suffixes['other text files']         = { }
+suffixes['other binary files']       = { }
+suffixes['opentype fonts']           = { 'otf' }
+
+suffixes['fmt']                      = { 'fmt' }
+suffixes['texmfscripts']             = { 'rb','lua','py','pl' }
+
+suffixes['pdftex config']            = { }
+suffixes['Troff fonts']              = { }
+
+suffixes['ls-R']                     = { }
+
+--[[ldx--
+
If you wondered abou tsome of the previous mappings, how about
+the next bunch:

+--ldx]]--
+
+formats['bib']                      = ''
+formats['bst']                      = ''
+formats['mft']                      = ''
+formats['ist']                      = ''
+formats['web']                      = ''
+formats['cweb']                     = ''
+formats['MetaPost support']         = ''
+formats['TeX system documentation'] = ''
+formats['TeX system sources']       = ''
+formats['Troff fonts']              = ''
+formats['dvips config']             = ''
+formats['graphic/figure']           = ''
+formats['ls-R']                     = ''
+formats['other text files']         = ''
+formats['other binary files']       = ''
+
+formats['gf']                       = ''
+formats['pk']                       = ''
+formats['base']                     = 'MFBASES'
+formats['cnf']                      = ''
+formats['mem']                      = 'MPMEMS'
+formats['mf']                       = 'MFINPUTS'
+formats['mfpool']                   = 'MFPOOL'
+formats['mppool']                   = 'MPPOOL'
+formats['texpool']                  = 'TEXPOOL'
+formats['PostScript header']        = 'TEXPSHEADERS'
+formats['cmap files']               = 'CMAPFONTS'
+formats['type42 fonts']             = 'T42FONTS'
+formats['web2c files']              = 'WEB2C'
+formats['pdftex config']            = 'PDFTEXCONFIG'
+formats['texmfscripts']             = 'TEXMFSCRIPTS'
+formats['bitmap font']              = ''
+formats['lig files']                = 'LIGFONTS'
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-aux'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+local trace_verbose = false  trackers.register("resolvers.verbose", function(v) trace_verbose = v end)
+
+function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix
+    local scriptpath = "scripts/context/lua"
+    newname = file.addsuffix(newname,"lua")
+    local oldscript = resolvers.clean_path(oldname)
+    if trace_verbose then
+        logs.report("fileio","to be replaced old script %s", oldscript)
+    end
+    local newscripts = resolvers.find_files(newname) or { }
+    if #newscripts == 0 then
+        if trace_verbose then
+            logs.report("fileio","unable to locate new script")
+        end
+    else
+        for i=1,#newscripts do
+            local newscript = resolvers.clean_path(newscripts[i])
+            if trace_verbose then
+                logs.report("fileio","checking new script %s", newscript)
+            end
+            if oldscript == newscript then
+                if trace_verbose then
+                    logs.report("fileio","old and new script are the same")
+                end
+            elseif not find(newscript,scriptpath) then
+                if trace_verbose then
+                    logs.report("fileio","new script should come from %s",scriptpath)
+                end
+            elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then
+                if trace_verbose then
+                    logs.report("fileio","invalid new script name")
+                end
+            else
+                local newdata = io.loaddata(newscript)
+                if newdata then
+                    if trace_verbose then
+                        logs.report("fileio","old script content replaced by new content")
+                    end
+                    io.savedata(oldscript,newdata)
+                    break
+                elseif trace_verbose then
+                    logs.report("fileio","unable to load new script")
+                end
+            end
+        end
     end
 end
 
-function input.report(fmt,...)
-    if input.verbose then
-        logs.report(input.banner or "report",format(fmt,...))
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- loads *.tmf files in minimal tree roots (to be optimized and documented)
+
+function resolvers.check_environment(tree)
+    logs.simpleline()
+    os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME'))
+    os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform()))
+    os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",''))
+    os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS'))
+    logs.simpleline()
+    logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH'))
+    logs.simple("preset : TEXOS   => %s", os.getenv('TEXOS'))
+    logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS'))
+    logs.simple("preset : TMP     => %s", os.getenv('TMP'))
+    logs.simple('')
+end
+
+function resolvers.load_environment(name) -- todo: key=value as well as lua
+    local f = io.open(name)
+    if f then
+        for line in f:lines() do
+            if line:find("^[%%%#]") then
+                -- skip comment
+            else
+                local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$")
+                if how then
+                    value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end)
+                        if how == "=" or how == "<<" then
+                            os.setenv(key,value)
+                    elseif how == "?" or how == "??" then
+                            os.setenv(key,os.getenv(key) or value)
+                    elseif how == "<" or how == "+=" then
+                        if os.getenv(key) then
+                            os.setenv(key,os.getenv(key) .. io.fileseparator .. value)
+                        else
+                            os.setenv(key,value)
+                        end
+                    elseif how == ">" or how == "=+" then
+                        if os.getenv(key) then
+                            os.setenv(key,value .. io.pathseparator .. os.getenv(key))
+                        else
+                            os.setenv(key,value)
+                        end
+                    end
+                end
+            end
+        end
+        f:close()
     end
 end
 
-function input.reportlines(str) -- todo: 
-    for line in str:gmatch("(.-)[\n\r]") do
-        logs.report(input.banner or "report",line)
+function resolvers.load_tree(tree)
+    if tree and tree ~= "" then
+        local setuptex = 'setuptex.tmf'
+        if lfs.attributes(tree, "mode") == "directory" then -- check if not nil
+            setuptex = tree .. "/" .. setuptex
+        else
+            setuptex = tree
+        end
+        if io.exists(setuptex) then
+            resolvers.check_environment(tree)
+            resolvers.load_environment(setuptex)
+        end
     end
 end
 
-input.moreinfo = [[
-more information about ConTeXt and the tools that come with it can be found at:
-
-maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
-webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
-wiki     : http://contextgarden.net
-]]
-
-function input.help(banner,message)
-    if not input.verbose then
-        input.verbose = true
-    --  input.report(banner,"\n")
-    end
-    input.report(banner,"\n")
-    input.report("")
-    input.reportlines(message)
-    if input.moreinfo and input.moreinfo ~= "" then
-        input.report("")
-        input.reportlines(input.moreinfo)
-    end
-end
 
-logs.set_level('error')
-logs.set_method('tex')
+end -- of closure
 
+do -- create closure to overcome 200 locals limit
 
 if not modules then modules = { } end modules ['luat-sta'] = {
     version   = 1.001,
@@ -7992,6 +9287,8 @@ if not modules then modules = { } end modules ['luat-sta'] = {
     license   = "see context related readme files"
 }
 
+-- this code is used in the updater
+
 states          = states          or { }
 states.data     = states.data     or { }
 states.hash     = states.hash     or { }
@@ -8018,27 +9315,32 @@ end
 function states.set_by_tag(tag,key,value,default,persistent)
     local d, h = states.data[tag], states.hash[tag]
     if d then
-        local dkey, hkey = key, key
-        local pre, post = key:match("(.+)%.([^%.]+)$")
-        if pre and post then
-            for k in pre:gmatch("[^%.]+") do
-                local dk = d[k]
-                if not dk then
-                    dk = { }
-                    d[k] = dk
+        if type(d) == "table" then
+            local dkey, hkey = key, key
+            local pre, post = key:match("(.+)%.([^%.]+)$")
+            if pre and post then
+                for k in pre:gmatch("[^%.]+") do
+                    local dk = d[k]
+                    if not dk then
+                        dk = { }
+                        d[k] = dk
+                    end
+                    d = dk
                 end
-                d = dk
+                dkey, hkey = post, key
             end
-            dkey, hkey = post, key
-        end
-        if type(value) == nil then
-            value = value or default
-        elseif persistent then
-            value = value or d[dkey] or default
-        else
-            value = value or default
+            if type(value) == nil then
+                value = value or default
+            elseif persistent then
+                value = value or d[dkey] or default
+            else
+                value = value or default
+            end
+            d[dkey], h[hkey] = value, value
+        elseif type(d) == "string" then
+            -- weird
+            states.data[tag], states.hash[tag] = value, value
         end
-        d[dkey], h[hkey] = value, value
     end
 end
 
@@ -8158,7 +9460,6 @@ end
 --~ 	},
 --~ }
 
-
 --~ states.save("teststate", "update")
 --~ states.load("teststate", "update")
 
@@ -8169,6 +9470,8 @@ end
 --~ states.load("teststate", "update")
 --~ print(states.get_by_tag("update","rsync.server","unknown"))
 
+
+end -- of closure
 -- end library merge
 
 own = { } -- not local
@@ -8178,27 +9481,43 @@ own.libs = { -- todo: check which ones are really needed
     'l-lpeg.lua',
     'l-table.lua',
     'l-io.lua',
-    'l-md5.lua',
     'l-number.lua',
     'l-set.lua',
     'l-os.lua',
     'l-file.lua',
+    'l-md5.lua',
     'l-dir.lua',
     'l-boolean.lua',
     'l-math.lua',
-    'l-xml.lua',
 --  'l-unicode.lua',
-    'l-utils.lua',
 --  'l-tex.lua',
-    'luat-lib.lua',
-    'luat-inp.lua',
-    'luat-log.lua',
---  'luat-zip.lua',
---  'luat-tex.lua',
---  'luat-kps.lua',
-    'luat-tmp.lua',
-    'luat-log.lua',
-    'luat-sta.lua',
+    'l-utils.lua',
+--  'l-xml.lua',
+    'lxml-tab.lua',
+    'lxml-pth.lua',
+    'lxml-ent.lua',
+    'lxml-mis.lua',
+    'trac-tra.lua',
+    'luat-env.lua',
+    'trac-inf.lua',
+    'trac-log.lua',
+    'data-res.lua',
+    'data-tmp.lua',
+    'data-pre.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-tex.lua',
+--  'data-bin.lua',
+    'data-zip.lua',
+    'data-crl.lua',
+--  'data-lua.lua',
+    'data-kps.lua', -- so that we can replace kpsewhich
+    'data-aux.lua', -- updater
+    'data-tmf.lua', -- tree files
+    -- needed ?
+    'luat-sta.lua', -- states
 }
 
 -- We need this hack till luatex is fixed.
@@ -8236,11 +9555,11 @@ local function locate_libs()
     end
 end
 
-if not input then
+if not resolvers then
     locate_libs()
 end
 
-if not input then
+if not resolvers then
     print("")
     print("Mtxrun is unable to start up due to lack of libraries. You may")
     print("try to run 'lua mtxrun.lua --selfmerge' in the path where this")
@@ -8249,216 +9568,12 @@ if not input then
     os.exit()
 end
 
-input.instance      = input.reset()
-input.banner        = 'MtxRun'
-utils.report        = input.report
-
-local instance = input.instance
-
-
--- use os.env or environment when available
-
-function input.check_environment(tree)
-    input.report('')
-    os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME'))
-    os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform()))
-    os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",''))
-    os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS'))
-    input.report('')
-    input.report("preset : TEXPATH => %s", os.getenv('TEXPATH'))
-    input.report("preset : TEXOS   => %s", os.getenv('TEXOS'))
-    input.report("preset : TEXMFOS => %s", os.getenv('TEXMFOS'))
-    input.report("preset : TMP     => %s", os.getenv('TMP'))
-    input.report('')
-end
-
-function input.load_environment(name) -- todo: key=value as well as lua
-    local f = io.open(name)
-    if f then
-        for line in f:lines() do
-            if line:find("^[%%%#]") then
-                -- skip comment
-            else
-                local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$")
-                if how then
-                    value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end)
-                        if how == "=" or how == "<<" then
-                            os.setenv(key,value)
-                    elseif how == "?" or how == "??" then
-                            os.setenv(key,os.getenv(key) or value)
-                    elseif how == "<" or how == "+=" then
-                        if os.getenv(key) then
-                            os.setenv(key,os.getenv(key) .. io.fileseparator .. value)
-                        else
-                            os.setenv(key,value)
-                        end
-                    elseif how == ">" or how == "=+" then
-                        if os.getenv(key) then
-                            os.setenv(key,value .. io.pathseparator .. os.getenv(key))
-                        else
-                            os.setenv(key,value)
-                        end
-                    end
-                end
-            end
-        end
-        f:close()
-    end
-end
-
-function input.load_tree(tree)
-    if tree and tree ~= "" then
-        local setuptex = 'setuptex.tmf'
-        if lfs.attributes(tree, "mode") == "directory" then -- check if not nil
-            setuptex = tree .. "/" .. setuptex
-        else
-            setuptex = tree
-        end
-        if io.exists(setuptex) then
-            input.check_environment(tree)
-            input.load_environment(setuptex)
-        end
-    end
-end
-
--- md5 extensions
-
--- maybe md.md5 md.md5hex md.md5HEX
-
-if not md5 then md5 = { } end
-
-if not md5.sum then
-    function md5.sum(k)
-        return string.rep("x",16)
-    end
-end
-
-function md5.hexsum(k)
-    return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02x", string.byte(c)) end))
-end
-
-function md5.HEXsum(k)
-    return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02X", string.byte(c)) end))
-end
-
--- file extensions
-
-file.needs_updating_threshold = 1
-
-function file.needs_updating(oldname,newname) -- size modification access change
-    local oldtime = lfs.attributes(oldname, modification)
-    local newtime = lfs.attributes(newname, modification)
-    if newtime >= oldtime then
-        return false
-    elseif oldtime - newtime < file.needs_updating_threshold then
-        return false
-    else
-        return true
-    end
-end
-
-function file.checksum(name)
-    if md5 then
-        local data = io.loaddata(name)
-        if data then
-            return md5.HEXsum(data)
-        end
-    end
-    return nil
-end
-
-function file.loadchecksum(name)
-    if md5 then
-        local data = io.loaddata(name .. ".md5")
-        return data and data:gsub("%s","")
-    end
-    return nil
-end
-
-function file.savechecksum(name, checksum)
-    if not checksum then checksum = file.checksum(name) end
-    if checksum then
-        io.savedata(name .. ".md5",checksum)
-        return checksum
-    end
-    return nil
-end
-
-os.arch = os.arch or function()
-    return os.resultof("uname -m") or "linux"
-end
-
-function os.currentplatform(name, default)
-    local name = os.name or os.platform or name -- os.name is built in, os.platform is mine
-    if name then
-        if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then
-            return "mswin"
-        elseif name == "linux" then
-            local architecture = os.arch()
-            if architecture:find("x86_64") then
-                return "linux-64"
-            elseif architecture:find("ppc") then
-                return "linux-ppc"
-            else
-                return "linux"
-            end
-        elseif name == "macosx" then
-            local architecture = os.arch()
-            if architecture:find("i386") then
-                return "osx-intel"
-            else
-                return "osx-ppc"
-            end
-        elseif name == "freebsd" then
-            return "freebsd"
-        end
-    end
-    return default or name
-end
-
--- it starts here
-
-input.runners              = { }
-input.runners.applications = { }
-
-input.runners.applications.lua = "luatex --luaonly"
-input.runners.applications.pl  = "perl"
-input.runners.applications.py  = "python"
-input.runners.applications.rb  = "ruby"
-
-input.runners.suffixes = {
-    'rb', 'lua', 'py', 'pl'
-}
-
-input.runners.registered = {
-    texexec      = { 'texexec.rb',      true  },
-    texutil      = { 'texutil.rb',      true  },
-    texfont      = { 'texfont.pl',      true  },
-    texshow      = { 'texshow.pl',      false },
-
-    makempy      = { 'makempy.pl',      true  },
-    mptopdf      = { 'mptopdf.pl',      true  },
-    pstopdf      = { 'pstopdf.rb',      true  },
-
---  examplex     = { 'examplex.rb',     false },
-    concheck     = { 'concheck.rb',     false },
-
-    runtools     = { 'runtools.rb',     true  },
-    textools     = { 'textools.rb',     true  },
-    tmftools     = { 'tmftools.rb',     true  },
-    ctxtools     = { 'ctxtools.rb',     true  },
-    rlxtools     = { 'rlxtools.rb',     true  },
-    pdftools     = { 'pdftools.rb',     true  },
-    mpstools     = { 'mpstools.rb',     true  },
-    exatools     = { 'exatools.rb',     true  },
-    xmltools     = { 'xmltools.rb',     true  },
-    luatools     = { 'luatools.lua',    true  },
-    mtxtools     = { 'mtxtools.rb',     true  },
+logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false)
 
-    pdftrimwhite = { 'pdftrimwhite.pl', false }
-}
+local instance = resolvers.reset()
 
-if not messages then messages = { } end
+runners  = runners  or { } -- global
+messages = messages or { }
 
 messages.help = [[
 --script              run an mtx script (--noquotes)
@@ -8488,43 +9603,78 @@ messages.help = [[
 --launch (--all)      launch files like manuals, assumes os support
 
 --intern              run script using built in libraries
+
+--usekpse            use kpse as fallback (when no mkiv and cache installed, often slower)
+--forcekpse          force using kpse (handy when no mkiv and cache installed but less functionality)
 ]]
 
-function input.runners.my_prepare_a()
-    input.resetconfig()
-    input.identify_cnf()
-    input.load_lua()
-    input.expand_variables()
-    input.load_cnf()
-    input.expand_variables()
-end
+runners.applications = {
+    ["lua"] = "luatex --luaonly",
+    ["luc"] = "luatex --luaonly",
+    ["pl"] = "perl",
+    ["py"] = "python",
+    ["rb"] = "ruby",
+}
 
-function input.runners.my_prepare_b()
-    input.runners.my_prepare_a()
-    input.load_hash()
-    input.automount()
-end
+runners.suffixes = {
+    'rb', 'lua', 'py', 'pl'
+}
+
+runners.registered = {
+    texexec      = { 'texexec.rb',      true  },  -- context mkii runner (only tool not to be luafied)
+    texutil      = { 'texutil.rb',      true  },  -- old perl based index sorter for mkii (old versions need it)
+    texfont      = { 'texfont.pl',      true  },  -- perl script that makes mkii font metric files
+    texfind      = { 'texfind.pl',      false },  -- perltk based tex searching tool, mostly used at pragma
+    texshow      = { 'texshow.pl',      false },  -- perltk based context help system, will be luafied
+ -- texwork      = { \texwork.pl',      false },  -- perltk based editing environment, only used at pragma
+
+    makempy      = { 'makempy.pl',      true  },
+    mptopdf      = { 'mptopdf.pl',      true  },
+    pstopdf      = { 'pstopdf.rb',      true  },  -- converts ps (and some more) images, does some cleaning (replaced)
+
+--  examplex     = { 'examplex.rb',     false },
+    concheck     = { 'concheck.rb',     false },
+
+    runtools     = { 'runtools.rb',     true  },
+    textools     = { 'textools.rb',     true  },
+    tmftools     = { 'tmftools.rb',     true  },
+    ctxtools     = { 'ctxtools.rb',     true  },
+    rlxtools     = { 'rlxtools.rb',     true  },
+    pdftools     = { 'pdftools.rb',     true  },
+    mpstools     = { 'mpstools.rb',     true  },
+--  exatools     = { 'exatools.rb',     true  },
+    xmltools     = { 'xmltools.rb',     true  },
+--  luatools     = { 'luatools.lua',    true  },
+    mtxtools     = { 'mtxtools.rb',     true  },
+
+    pdftrimwhite = { 'pdftrimwhite.pl', false }
+}
+
+runners.launchers = {
+    windows = { },
+    unix = { }
+}
 
-function input.runners.prepare()
+function runners.prepare()
     local checkname = environment.argument("ifchanged")
     if checkname and checkname ~= "" then
         local oldchecksum = file.loadchecksum(checkname)
         local newchecksum = file.checksum(checkname)
         if oldchecksum == newchecksum then
-            input.report("file '%s' is unchanged",checkname)
+            logs.simple("file '%s' is unchanged",checkname)
             return "skip"
         else
-            input.report("file '%s' is changed, processing started",checkname)
+            logs.simple("file '%s' is changed, processing started",checkname)
         end
         file.savechecksum(checkname)
     end
     local oldname, newname = string.split(environment.argument("iftouched") or "", ",")
     if oldname and newname and oldname ~= "" and newname ~= "" then
         if not file.needs_updating(oldname,newname) then
-            input.report("file '%s' and '%s' have same age",oldname,newname)
+            logs.simple("file '%s' and '%s' have same age",oldname,newname)
             return "skip"
         else
-            input.report("file '%s' is older than '%s'",oldname,newname)
+            logs.simple("file '%s' is older than '%s'",oldname,newname)
         end
     end
     local tree = environment.argument('tree') or ""
@@ -8532,28 +9682,27 @@ function input.runners.prepare()
         tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree
     end
     if tree and tree ~= "" then
-        input.load_tree(tree)
+        resolvers.load_tree(tree)
     end
     local env = environment.argument('environment') or ""
     if env and env ~= "" then
         for _,e in pairs(string.split(env)) do
             -- maybe force suffix when not given
-            input.load_tree(e)
+            resolvers.load_tree(e)
         end
     end
     local runpath = environment.argument("path")
-    if runpath and not dir.chdir(runpath) then
-        input.report("unable to change to path '%s'",runpath)
+    if runpath and not lfs.chdir(runpath) then
+        logs.simple("unable to change to path '%s'",runpath)
         return "error"
     end
     return "run"
 end
 
-function input.runners.execute_script(fullname,internal)
-    local instance = input.instance
+function runners.execute_script(fullname,internal)
     local noquote = environment.argument("noquotes")
     if fullname and fullname ~= "" then
-        local state = input.runners.prepare()
+        local state = runners.prepare()
         if state == 'error' then
             return false
         elseif state == 'skip' then
@@ -8570,41 +9719,44 @@ function input.runners.execute_script(fullname,internal)
                     return ""
                 end )
                 name = name:gsub("^script:","")
-                if suffix == "" and input.runners.registered[name] and input.runners.registered[name][1] then
-                    name = input.runners.registered[name][1]
+                if suffix == "" and runners.registered[name] and runners.registered[name][1] then
+                    name = runners.registered[name][1]
                     suffix = file.extname(name)
                 end
                 if suffix == "" then
                     -- loop over known suffixes
-                    for _,s in pairs(input.runners.suffixes) do
-                        result = input.find_file(name .. "." .. s, 'texmfscripts')
+                    for _,s in pairs(runners.suffixes) do
+                        result = resolvers.find_file(name .. "." .. s, 'texmfscripts')
                         if result ~= "" then
                             break
                         end
                     end
-                elseif input.runners.applications[suffix] then
-                    result = input.find_file(name, 'texmfscripts')
+                elseif runners.applications[suffix] then
+                    result = resolvers.find_file(name, 'texmfscripts')
                 else
                     -- maybe look on path
-                    result = input.find_file(name, 'other text files')
+                    result = resolvers.find_file(name, 'other text files')
                 end
             end
             if result and result ~= "" then
+                local before, after = environment.split_arguments(fullname) -- already done
+                environment.arguments_before, environment.arguments_after = before, after
                 if internal then
-                    local before, after = environment.split_arguments(fullname)
                     arg = { } for _,v in pairs(after) do arg[#arg+1] = v end
                     dofile(result)
                 else
-                    local binary = input.runners.applications[file.extname(result)]
+                    local binary = runners.applications[file.extname(result)]
                     if binary and binary ~= "" then
                         result = binary .. " " .. result
                     end
-                    local before, after = environment.split_arguments(fullname)
                     local command = result .. " " .. environment.reconstruct_commandline(after,noquote)
-                    input.report("")
-                    input.report("executing: %s",command)
-                    input.report("\n \n")
-                    io.flush()
+                    if logs.verbose then
+                        logs.simpleline()
+                        logs.simple("executing: %s",command)
+                        logs.simpleline()
+                        logs.simpleline()
+                        io.flush()
+                    end
                     local code = os.exec(command) -- maybe spawn
                     return code == 0
                 end
@@ -8614,10 +9766,10 @@ function input.runners.execute_script(fullname,internal)
     return false
 end
 
-function input.runners.execute_program(fullname)
+function runners.execute_program(fullname)
     local noquote = environment.argument("noquotes")
     if fullname and fullname ~= "" then
-        local state = input.runners.prepare()
+        local state = runners.prepare()
         if state == 'error' then
             return false
         elseif state == 'skip' then
@@ -8627,9 +9779,10 @@ function input.runners.execute_program(fullname)
             environment.initialize_arguments(after)
             fullname = fullname:gsub("^bin:","")
             local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "")
-            input.report("")
-            input.report("executing: %s",command)
-            input.report("\n \n")
+            logs.simpleline()
+            logs.simple("executing: %s",command)
+            logs.simpleline()
+            logs.simpleline()
             io.flush()
             local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn
             return code == 0
@@ -8638,8 +9791,13 @@ function input.runners.execute_program(fullname)
     return false
 end
 
-function input.runners.handle_stubs(create)
-    local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer supported
+-- the --usekpse flag will fallback on kpse
+
+local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010'
+local unix_stub    = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010'
+
+function runners.handle_stubs(create)
+    local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported
     local windows  = environment.argument('windows') or environment.argument('mswin') or false
     local unix     = environment.argument('unix') or environment.argument('linux') or false
     if not windows and not unix then
@@ -8649,77 +9807,77 @@ function input.runners.handle_stubs(create)
             windows = true
         end
     end
-    for _,v in pairs(input.runners.registered) do
+    for _,v in pairs(runners.registered) do
         local name, doit = v[1], v[2]
         if doit then
             local base = string.gsub(file.basename(name), "%.(.-)$", "")
             if create then
-                -- direct local command = input.runners.applications[file.extname(name)] .. " " .. name
-                local command = "luatex --luaonly mtxrun.lua " .. name
                 if windows then
-                    io.savedata(base..".bat", {"@echo off", command.." %*"}, "\013\010")
-                        input.report("windows stub for '%s' created",base)
+                    io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name))
+                    logs.simple("windows stub for '%s' created",base)
                 end
                 if unix then
-                    io.savedata(base, {"#!/bin/sh", command..' "$@"'}, "\010")
-                    input.report("unix stub for '%s' created",base)
+                    io.savedata(file.join(stubpath,base),string.format(unix_stub,name))
+                    logs.simple("unix stub for '%s' created",base)
                 end
             else
-                if windows and (os.remove(base..'.bat') or os.remove(base..'.cmd')) then
-                    input.report("windows stub for '%s' removed", base)
+                if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then
+                    logs.simple("windows stub for '%s' removed", base)
                 end
-                if unix and (os.remove(base) or os.remove(base..'.sh')) then
-                    input.report("unix stub for '%s' removed",base)
+                if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then
+                    logs.simple("unix stub for '%s' removed",base)
                 end
             end
         end
     end
 end
 
-function input.runners.resolve_string(filename)
+function runners.resolve_string(filename)
     if filename and filename ~= "" then
-        input.runners.report_location(input.resolve(filename))
+        runners.report_location(resolvers.resolve(filename))
     end
 end
 
-function input.runners.locate_file(filename)
+function runners.locate_file(filename)
+    -- differs from texmfstart where locate appends .com .exe .bat ... todo
     if filename and filename ~= "" then
-        input.runners.report_location(input.find_given_file(filename))
+        runners.report_location(resolvers.find_given_file(filename))
     end
 end
 
-function input.runners.locate_platform()
-    input.runners.report_location(os.currentplatform())
+function runners.locate_platform()
+    runners.report_location(os.currentplatform())
 end
 
-function input.runners.report_location(result)
-    if input.verbose then
-        input.report("")
+function runners.report_location(result)
+    if logs.verbose then
+        logs.simpleline()
         if result and result ~= "" then
-            input.report(result)
+            logs.simple(result)
         else
-            input.report("not found")
+            logs.simple("not found")
         end
     else
         io.write(result)
     end
 end
 
-function input.runners.edit_script(filename) -- we assume that vim is present on most systems
+function runners.edit_script(filename) -- we assume that vim is present on most systems
     local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim'
-    local rest = input.resolve(filename)
+    local rest = resolvers.resolve(filename)
     if rest ~= "" then
         local command = editor .. " " .. rest
-        if input.verbose then
-            input.report("")
-            input.report("starting editor: %s",command)
-            input.report("\n \n")
+        if logs.verbose then
+            logs.simpleline()
+            logs.simple("starting editor: %s",command)
+            logs.simple_line()
+            logs.simple_line()
         end
         os.launch(command)
     end
 end
 
-function input.runners.save_script_session(filename, list)
+function runners.save_script_session(filename, list)
     local t = { }
     for _, key in ipairs(list) do
         t[key] = environment.arguments[key]
@@ -8727,7 +9885,7 @@ function input.runners.save_script_session(filename, list)
     io.savedata(filename,table.serialize(t,true))
 end
 
-function input.runners.load_script_session(filename)
+function runners.load_script_session(filename)
     if lfs.isfile(filename) then
         local t = io.loaddata(filename)
         if t then
@@ -8740,15 +9898,10 @@ function input.runners.load_script_session(filename)
     end
 end
 
-input.runners.launchers = {
-    windows = { },
-    unix = { }
-}
-
-function input.launch(str)
+function resolvers.launch(str)
     -- maybe we also need to test on mtxrun.launcher.suffix environment
     -- variable or on windows consult the assoc and ftype vars and such
-    local launchers = input.runners.launchers[os.platform] if launchers then
+    local launchers = runners.launchers[os.platform] if launchers then
         local suffix = file.extname(str) if suffix then
             local runner = launchers[suffix] if runner then
                 str = runner .. " " .. str
@@ -8758,41 +9911,40 @@ function input.launch(str)
     os.launch(str)
 end
 
-function input.runners.launch_file(filename)
-    local instance = input.instance
+function runners.launch_file(filename)
     instance.allresults = true
-    input.verbose = true
+    logs.setverbose(true)
     local pattern = environment.arguments["pattern"]
     if not pattern or pattern == "" then
         pattern = filename
     end
     if not pattern or pattern == "" then
-        input.report("provide name or --pattern=")
+        logs.simple("provide name or --pattern=")
     else
-        local t = input.find_files(pattern)
+        local t = resolvers.find_files(pattern)
         if not t or #t == 0 then
-            t = input.aux.find_file("*/" .. pattern,true)
+            t = resolvers.find_files("*/" .. pattern)
         end
         if not t or #t == 0 then
-            t = input.aux.find_file("*/" .. pattern .. "*",true)
+            t = resolvers.find_files("*/" .. pattern .. "*")
         end
         if t and #t > 0 then
             if environment.arguments["all"] then
                 for _, v in pairs(t) do
-                    input.report("launching %s", v)
-                    input.launch(v)
+                    logs.simple("launching %s", v)
+                    resolvers.launch(v)
                 end
             else
-                input.report("launching %s", t[1])
-                input.launch(t[1])
+                logs.simple("launching %s", t[1])
+                resolvers.launch(t[1])
             end
         else
-            input.report("no match for %s", pattern)
+            logs.simple("no match for %s", pattern)
         end
     end
 end
 
-function input.runners.find_mtx_script(filename)
+function runners.find_mtx_script(filename)
     local function found(name)
         local path = file.dirname(name)
         if path and path ~= "" then
@@ -8803,10 +9955,10 @@ function input.runners.find_mtx_script(filename)
         end
     end
     filename = file.addsuffix(filename,"lua")
-    local basename = file.stripsuffix(file.basename(filename))
+    local basename = file.removesuffix(file.basename(filename))
     local suffix = file.extname(filename)
     -- qualified path, raw name
-    local fullname = input.aux.qualified_path(filename) and io.exists(filename) and filename
+    local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename
     if fullname and fullname ~= "" then
         return fullname
     end
@@ -8818,74 +9970,84 @@ function input.runners.find_mtx_script(filename)
     end
     -- context namespace, mtx-
     fullname = "mtx-" .. filename
-    fullname = found(fullname) or input.find_file(fullname)
+    fullname = found(fullname) or resolvers.find_file(fullname)
     if fullname and fullname ~= "" then
         return fullname
     end
     -- context namespace, mtx-s
     fullname = "mtx-" .. basename .. "s" .. "." .. suffix
-    fullname = found(fullname) or input.find_file(fullname)
+    fullname = found(fullname) or resolvers.find_file(fullname)
     if fullname and fullname ~= "" then
         return fullname
     end
     -- context namespace, mtx-
     fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix
-    fullname = found(fullname) or input.find_file(fullname)
+    fullname = found(fullname) or resolvers.find_file(fullname)
     if fullname and fullname ~= "" then
         return fullname
     end
     -- context namespace, just 
-    fullname = input.find_file(fullname)
+    fullname = resolvers.find_file(filename)
     return fullname
 end
 
-function input.runners.execute_ctx_script(filename,arguments)
-    local fullname = input.runners.find_mtx_script(filename)
+function runners.execute_ctx_script(filename,arguments)
+    local fullname = runners.find_mtx_script(filename) or ""
+    -- retyr after generate but only if --autogenerate
+    if fullname == "" and environment.argument("autogenerate") then -- might become the default
+        instance.renewcache = true
+        logs.setverbose(true)
+        resolvers.load()
+        --
+        fullname = runners.find_mtx_script(filename) or ""
+    end
     -- that should do it
-    if fullname and fullname ~= "" then
-        local state = input.runners.prepare()
+    if fullname ~= "" then
+        local state = runners.prepare()
         if state == 'error' then
             return false
         elseif state == 'skip' then
             return true
         elseif state == "run" then
             -- load and save ... kind of undocumented
-            arg = { } for _,v in pairs(arguments) do arg[#arg+1] = v end
+            arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end
             environment.initialize_arguments(arg)
             local loadname = environment.arguments['load']
             if loadname then
                 if type(loadname) ~= "string" then loadname = file.basename(fullname) end
                 loadname = file.replacesuffix(loadname,"cfg")
-                input.runners.load_script_session(loadname)
+                runners.load_script_session(loadname)
             end
             filename = environment.files[1]
-            if input.verbose then
-                input.report("using script: %s\n",fullname)
+            if logs.verbose then
+                logs.simple("using script: %s\n",fullname)
             end
             dofile(fullname)
             local savename = environment.arguments['save']
-            if savename and input.runners.save_list and not table.is_empty(input.runners.save_list or { }) then
+            if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then
                 if type(savename) ~= "string" then savename = file.basename(fullname) end
                 savename = file.replacesuffix(savename,"cfg")
-                input.runners.save_script_session(savename, input.runners.save_list)
+                runners.save_script_session(savename, runners.save_list)
             end
             return true
         end
     else
-        input.verbose = true
+        logs.setverbose(true)
         filename = file.addsuffix(filename,"lua")
         if filename == "" then
-            input.report("unknown script, no name given")
-        elseif input.aux.qualified_path(filename) then
-            input.report("unknown script '%s'",filename)
+            logs.simple("unknown script, no name given")
+        elseif file.is_qualified_path(filename) then
+            logs.simple("unknown script '%s'",filename)
         else
-            input.report("unknown script '%s' or 'mtx-%s'",filename,filename)
+            logs.simple("unknown script '%s' or 'mtx-%s'",filename,filename)
         end
         return false
     end
 end
 
-input.report("%s\n",banner)
+function runners.timed(action)
+    statistics.timed(action)
+end
 
 -- this is a bit dirty ... first we store the first filename and next we
 -- split the arguments so that we only see the ones meant for this script
@@ -8895,14 +10057,79 @@ local filename = environment.files[1] or ""
 local ok      = true
 
 local before, after = environment.split_arguments(filename)
+environment.arguments_before, environment.arguments_after = before, after
 environment.initialize_arguments(before)
 
 instance.engine   = environment.argument("engine")   or 'luatex'
 instance.progname = environment.argument("progname") or 'context'
 instance.lsrmode  = environment.argument("lsr")      or false
-input.verbose     = environment.argument("verbose")  or false
 
-input.runners.my_prepare_b()
+-- maybe the unset has to go to this level
+
+if environment.argument("usekpse") or environment.argument("forcekpse") then
+
+    os.setenv("engine","")
+    os.setenv("progname","")
+
+    local remapper = {
+        otf = "opentype fonts",
+        ttf = "truetype fonts",
+        ttc = "truetype fonts",
+        pfb = "type1 fonts",
+        other = "other text files",
+    }
+
+    local function kpse_initialized()
+        texconfig.kpse_init = true
+        local t = os.clock()
+        local k = kpse.original.new("luatex",instance.progname)
+        local dummy = k:find_file("mtxrun.lua") -- so that we're initialized
+        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t)
+        kpse_initialized = function() return k end
+        return k
+    end
+
+    local find_file = resolvers.find_file
+    local show_path = resolvers.show_path
+
+    if environment.argument("forcekpse") then
+
+        function resolvers.find_file(name,kind)
+            return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or ""
+        end
+        function resolvers.show_path(name)
+            return (kpse_initialized():show_path(name)) or ""
+        end
+
+    elseif environment.argument("usekpse") then
+
+        resolvers.load()
+
+        function resolvers.find_file(name,kind)
+            local found = find_file(name,kind) or ""
+            if found ~= "" then
+                return found
+            else
+                return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or ""
+            end
+        end
+        function resolvers.show_path(name)
+            local found = show_path(name) or ""
+            if found ~= "" then
+                return found
+            else
+                return (kpse_initialized():show_path(name)) or ""
+            end
+        end
+
+    end
+
+else
+
+    resolvers.load()
+
+end
+
 
 if environment.argument("selfmerge") then
     -- embed used libraries
@@ -8911,47 +10138,47 @@ elseif environment.argument("selfclean") then
     -- remove embedded libraries
     utils.merger.selfclean(own.name)
 elseif environment.argument("selfupdate") then
-    input.verbose = true
-    input.update_script(own.name,"mtxrun")
+    logs.setverbose(true)
+    resolvers.update_script(own.name,"mtxrun")
 elseif environment.argument("ctxlua") or environment.argument("internal") then
     -- run a script by loading it (using libs)
-    ok = input.runners.execute_script(filename,true)
-elseif environment.argument("script") then
+    ok = runners.execute_script(filename,true)
+elseif environment.argument("script") or environment.argument("s") then
     -- run a script by loading it (using libs), pass args
-    ok = input.runners.execute_ctx_script(filename,after)
+    ok = runners.execute_ctx_script(filename,after)
 elseif environment.argument("execute") then
     -- execute script
-    ok = input.runners.execute_script(filename)
+    ok = runners.execute_script(filename)
 elseif environment.argument("direct") then
     -- equals bin:
-    ok = input.runners.execute_program(filename)
+    ok = runners.execute_program(filename)
 elseif environment.argument("edit") then
     -- edit file
-    input.runners.edit_script(filename)
+    runners.edit_script(filename)
 elseif environment.argument("launch") then
-    input.runners.launch_file(filename)
+    runners.launch_file(filename)
 elseif environment.argument("make") then
     -- make stubs
-    input.runners.handle_stubs(true)
+    runners.handle_stubs(true)
 elseif environment.argument("remove") then
     -- remove stub
-    input.runners.handle_stubs(false)
+    runners.handle_stubs(false)
 elseif environment.argument("resolve") then
     -- resolve string
-    input.runners.resolve_string(filename)
+    runners.resolve_string(filename)
 elseif environment.argument("locate") then
     -- locate file
-    input.runners.locate_file(filename)
+    runners.locate_file(filename)
 elseif environment.argument("platform")then
     -- locate platform
-    input.runners.locate_platform()
+    runners.locate_platform()
 elseif environment.argument("help") or filename=='help' or filename == "" then
-    input.help(banner,messages.help)
+    logs.help(messages.help)
     -- execute script
 elseif filename:find("^bin:") then
-    ok = input.runners.execute_program(filename)
+    ok = runners.execute_program(filename)
 else
-    ok = input.runners.execute_script(filename)
+    ok = runners.execute_script(filename)
 end
 
 if os.platform == "unix" then
diff --git a/scripts/context/lua/x-ldx.lua b/scripts/context/lua/x-ldx.lua
index 67d5f925c..af5c9c0c8 100644
--- a/scripts/context/lua/x-ldx.lua
+++ b/scripts/context/lua/x-ldx.lua
@@ -335,10 +335,10 @@ The next function wraps it all in one call:
 --ldx]]--
 
 function ldx.convert(luaname,ldxname)
-    if not file.is_readable(luaname) then
+    if not file.isreadable(luaname) then
         luaname = luaname .. ".lua"
     end
-    if file.is_readable(luaname) then
+    if file.isreadable(luaname) then
         if not ldxname then
             ldxname = file.replacesuffix(luaname,"ldx")
         end
diff --git a/scripts/context/ruby/base/exa.rb b/scripts/context/ruby/base/exa.rb
index 5a094351e..7ba990cf9 100644
--- a/scripts/context/ruby/base/exa.rb
+++ b/scripts/context/ruby/base/exa.rb
@@ -3,8 +3,9 @@
 # tex.setup.setuplayout.width.[integer|real|dimension|string|key]
 # tex.[mp]var.whatever.width.[integer|real|dimension|string|key]
 
-require 'ftools'
-require 'md5'
+require 'fileutils'
+# require 'ftools'
+require 'digest/md5'
 
 # this can become a lua thing
 
@@ -40,7 +41,7 @@ module ExaEncrypt
                         pre, password, post = $1, $2, $3
                         unless password =~ /MD5:/i then
                             done = true
-                            password = "MD5:" + MD5.new(password).hexdigest.upcase
+                            password = "MD5:" + Digest::MD5.hexdigest(password).upcase
                         end
                         "#{pre}#{password}#{post}"
                     end
@@ -49,7 +50,7 @@ module ExaEncrypt
                         attributes, password = $1, $2
                         unless password =~ /^([0-9A-F][0-9A-F])+$/ then
                             done = true
-                            password = MD5.new(password).hexdigest.upcase
+                            password = Digest::MD5.hexdigest(password).upcase
                             attributes = " encryption='md5'#{attributes}"
                         end
                         "#{password}"
diff --git a/scripts/context/ruby/base/file.rb b/scripts/context/ruby/base/file.rb
index 39bb7d467..1aeac5fd6 100644
--- a/scripts/context/ruby/base/file.rb
+++ b/scripts/context/ruby/base/file.rb
@@ -8,7 +8,8 @@
 # info      : j.hagen@xs4all.nl
 # www       : www.pragma-ade.com
 
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 
 class File
 
@@ -110,7 +111,7 @@ class File
 
     def File.silentcopy(oldname,newname)
         return if File.expand_path(oldname) == File.expand_path(newname)
-        File.makedirs(File.dirname(newname)) rescue false
+        FileUtils.makedirs(File.dirname(newname)) rescue false
         File.copy(oldname,newname) rescue false
     end
 
@@ -123,7 +124,7 @@ class File
         begin
             File.rename(oldname,newname)
         rescue
-            File.makedirs(File.dirname(newname)) rescue false
+            FileUtils.makedirs(File.dirname(newname)) rescue false
             File.copy(oldname,newname) rescue false
         end
     end
diff --git a/scripts/context/ruby/base/kpse.rb b/scripts/context/ruby/base/kpse.rb
index 0e185b5b8..0f9868784 100644
--- a/scripts/context/ruby/base/kpse.rb
+++ b/scripts/context/ruby/base/kpse.rb
@@ -13,6 +13,7 @@
 # todo: web2c vs miktex module and include in kpse
 
 require 'rbconfig'
+require 'fileutils'
 
 # beware $engine is lowercase in kpse
 #
@@ -185,6 +186,7 @@ module Kpse
         return results
     end
 
+
     def Kpse.formatpaths
         # maybe we should check for writeability
         unless @@paths.key?('formatpaths') then
@@ -272,7 +274,7 @@ module Kpse
                     unless done then
                         formatpaths.each do |fp|
                             fpp = fp.sub(/#{engine}\/*$/o,'')
-                            File.makedirs(fpp) rescue false # maybe we don't have an path yet
+                            FileUtils.makedirs(fpp) rescue false # maybe we don't have an path yet
                             if FileTest.directory?(fpp) && FileTest.writable?(fpp) then
                                 # use this path
                                 formatpath, done = fp.dup, true
@@ -285,12 +287,12 @@ module Kpse
                     end
                 end
                 # needed !
-                File.makedirs(formatpath) rescue false
+                FileUtils.makedirs(formatpath) rescue false
                 # fall back to current path
                 formatpath = '.' if formatpath.empty? || ! FileTest.writable?(formatpath)
                 # append engine but prevent duplicates
                 formatpath = File.join(formatpath.sub(/\/*#{engine}\/*$/,''), engine) if enginepath
-                File.makedirs(formatpath) rescue false
+                FileUtils.makedirs(formatpath) rescue false
                 setpath(engine,formatpath)
                 # ENV['engine'] = savedengine
             end
diff --git a/scripts/context/ruby/base/state.rb b/scripts/context/ruby/base/state.rb
index 4b2088128..76ef50b25 100644
--- a/scripts/context/ruby/base/state.rb
+++ b/scripts/context/ruby/base/state.rb
@@ -1,4 +1,4 @@
-require "md5"
+require 'digest/md5'
 
 # todo: register omissions per file
 
@@ -57,7 +57,7 @@ class FileState
         begin
             if FileTest.file?(filename) && (data = IO.read(filename)) then
                 data.gsub!(/\n.*?(#{[omit].flatten.join('|')}).*?\n/) do "\n" end if omit
-                sum = MD5.new(data).hexdigest.upcase
+                sum = Digest::MD5.hexdigest(data).upcase
             end
         rescue
             sum = ''
diff --git a/scripts/context/ruby/base/tex.rb b/scripts/context/ruby/base/tex.rb
index 4f3e51299..c14cb840f 100644
--- a/scripts/context/ruby/base/tex.rb
+++ b/scripts/context/ruby/base/tex.rb
@@ -16,6 +16,8 @@
 
 # report ?
 
+require 'fileutils'
+
 require 'base/variables'
 require 'base/kpse'
 require 'base/system'
@@ -128,11 +130,12 @@ class TEX
     ['cont-ro','ro','romanian']                    .each do |f| @@texformats[f] = 'cont-ro'   end
     ['cont-gb','gb','cont-uk','uk','british']      .each do |f| @@texformats[f] = 'cont-gb'   end
     ['cont-pe','pe','persian']                     .each do |f| @@texformats[f] = 'cont-pe'   end
+    ['cont-xp','xp','experimental']                .each do |f| @@texformats[f] = 'cont-xp'   end
     ['mptopdf']                                    .each do |f| @@texformats[f] = 'mptopdf'   end
 
     ['latex']                                      .each do |f| @@texformats[f] = 'latex.ltx' end
 
-    ['plain','mpost']                              .each do |f| @@mpsformats[f] = 'plain'     end
+    ['plain','mpost']                              .each do |f| @@mpsformats[f] = 'mpost'     end
     ['metafun','context','standard']               .each do |f| @@mpsformats[f] = 'metafun'   end
 
     ['pdftex','pdfetex','aleph','omega','petex',
@@ -143,7 +146,7 @@ class TEX
     ['plain','default','standard','mptopdf']       .each do |f| @@texmethods[f] = 'plain'     end
     ['cont-en','cont-nl','cont-de','cont-it',
      'cont-fr','cont-cs','cont-ro','cont-gb',
-     'cont-pe']                                    .each do |f| @@texmethods[f] = 'context'   end
+     'cont-pe','cont-xp']                          .each do |f| @@texmethods[f] = 'context'   end
     ['latex','pdflatex']                           .each do |f| @@texmethods[f] = 'latex'     end
 
     ['plain','default','standard']                 .each do |f| @@mpsmethods[f] = 'plain'     end
@@ -154,7 +157,7 @@ class TEX
 
     ['cont-en','cont-nl','cont-de','cont-it',
      'cont-fr','cont-cs','cont-ro','cont-gb',
-     'cont-pe']                                    .each do |f| @@texprocstr[f] = @@platformslash + "emergencyend"  end
+     'cont-pe','cont-xp']                         .each do |f| @@texprocstr[f] = @@platformslash + "emergencyend"  end
 
     @@runoptions['aleph']   = ['--8bit']
     @@runoptions['luatex']  = ['--file-line-error']
@@ -166,7 +169,7 @@ class TEX
 
     @@tcxflag['aleph']   = true
     @@tcxflag['luatex']  = false
-    @@tcxflag['mpost']   = true
+    @@tcxflag['mpost']   = false
     @@tcxflag['pdfetex'] = true
     @@tcxflag['pdftex']  = true
     @@tcxflag['petex']   = false
@@ -174,8 +177,8 @@ class TEX
 
     @@draftoptions['pdftex'] = ['--draftmode']
 
-    @@booleanvars = [
-        'batchmode', 'nonstopmode', 'fast', 'fastdisabled', 'silentmode', 'final',
+    @@mainbooleanvars = [
+        'batchmode', 'nonstopmode', 'fast', 'final',
         'paranoid', 'notparanoid', 'nobanner', 'once', 'allpatterns', 'draft',
         'nompmode', 'nomprun', 'automprun', 'combine',
         'nomapfiles', 'local',
@@ -187,19 +190,19 @@ class TEX
         'globalfile', 'autopath',
         'purge', 'purgeall', 'keep', 'autopdf', 'xpdf', 'simplerun', 'verbose',
         'nooptionfile', 'nobackend', 'noctx', 'utfbom',
-        'mkii',
+        'mkii','mkiv',
     ]
-    @@stringvars = [
+    @@mainstringvars = [
         'modefile', 'result', 'suffix', 'response', 'path',
         'filters', 'usemodules', 'environments', 'separation', 'setuppath',
         'arguments', 'input', 'output', 'randomseed', 'modes', 'mode', 'filename',
         'ctxfile', 'printformat', 'paperformat', 'paperoffset',
         'timeout', 'passon'
     ]
-    @@standardvars = [
+    @@mainstandardvars = [
         'mainlanguage', 'bodyfont', 'language'
     ]
-    @@knownvars = [
+    @@mainknownvars = [
         'engine', 'distribution', 'texformats', 'mpsformats', 'progname', 'interface',
         'runs', 'backend'
     ]
@@ -208,29 +211,31 @@ class TEX
     @@extrastringvars = []
 
     def booleanvars
-        [@@booleanvars,@@extrabooleanvars].flatten.uniq
+        [@@mainbooleanvars,@@extrabooleanvars].flatten.uniq
     end
     def stringvars
-        [@@stringvars,@@extrastringvars].flatten.uniq
+        [@@mainstringvars,@@extrastringvars].flatten.uniq
     end
     def standardvars
-        [@@standardvars].flatten.uniq
+        [@@mainstandardvars].flatten.uniq
     end
     def knownvars
-        [@@knownvars].flatten.uniq
+        [@@mainknownvars].flatten.uniq
     end
     def allbooleanvars
-        [@@booleanvars,@@extrabooleanvars].flatten.uniq
+        [@@mainbooleanvars,@@extrabooleanvars].flatten.uniq
     end
     def allstringvars
-        [@@stringvars,@@extrastringvars,@@standardvars,@@knownvars].flatten.uniq
+        [@@mainstringvars,@@extrastringvars,@@mainstandardvars,@@mainknownvars].flatten.uniq
     end
 
     def setextrastringvars(vars)
-        @@extrastringvars << vars
+        # @@extrastringvars << vars -- problems in 1.9
+        @@extrastringvars = [@@extrastringvars,vars].flatten
     end
     def setextrabooleanvars(vars)
-        @@extrabooleanvars << vars
+        # @@extrabooleanvars << vars -- problems in 1.9
+        @@extrabooleanvars = [@@extrabooleanvars,vars].flatten
     end
 
     # def jobvariables(names=nil)
@@ -580,7 +585,7 @@ class TEX
         if data = (IO.readlines(@@luafiles) rescue nil) then
             report("compiling lua files (using #{File.expand_path(@@luafiles)})")
             begin
-                File.makedirs(@@luatarget) rescue false
+                FileUtils.makedirs(@@luatarget) rescue false
                 data.each do |line|
                     luafile = line.chomp
                     lucfile = File.basename(luafile).gsub(/\..*?$/,'') + ".luc"
@@ -686,7 +691,8 @@ class TEX
                 mpsformats.each do |mpsformat|
                     report("generating mps format #{mpsformat}")
                     progname = validprogname([getvariable('progname'),mpsformat,mpsengine])
-                    if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,tcxflag(mpsengine),runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then
+                #   if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,tcxflag(mpsengine),runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then
+                    if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then
                         setvariable('error','no format made')
                     end
                 end
@@ -1009,7 +1015,11 @@ end
             tmp << "\\starttext\n"
             if forcexml then
                 # tmp << checkxmlfile(rawname)
-                tmp << "\\processXMLfilegrouped{#{rawname}}\n"
+                if getvariable('mkiv') then
+                    tmp << "\\xmlprocess{\\xmldocument}{#{rawname}}{}\n"
+                else
+                    tmp << "\\processXMLfilegrouped{#{rawname}}\n"
+                end
             else
                 tmp << "\\processfile{#{rawname}}\n"
             end
@@ -1329,12 +1339,9 @@ class TEX
                 # local handies
                 opt << "\% #{topname}\n"
                 opt << "\\unprotect\n"
-                if getvariable('utfbom') then
-                    opt << "\\enableregime[utf]"
-                end
-                opt << "\\setupsystem[\\c!n=#{kindofrun},\\c!m=#{currentrun}]\n"
-                progname = validprogname(['metafun']) # [getvariable('progname'),mpsformat,mpsengine]
-                opt << "\\def\\MPOSTformatswitch\{#{prognameflag(progname)} #{formatflag('mpost')}=\}\n"
+                #
+                # feedback and basic control
+                #
                 if getvariable('batchmode') then
                     opt << "\\batchmode\n"
                 end
@@ -1344,6 +1351,21 @@ class TEX
                 if getvariable('paranoid') then
                     opt << "\\def\\maxreadlevel{1}\n"
                 end
+                if getvariable('nomapfiles') then
+                    opt << "\\disablemapfiles\n"
+                end
+                if getvariable('nompmode') || getvariable('nomprun') || getvariable('automprun') then
+                    opt << "\\runMPgraphicsfalse\n"
+                end
+                if getvariable('utfbom') then
+                    opt << "\\enableregime[utf]"
+                end
+                progname = validprogname(['metafun']) # [getvariable('progname'),mpsformat,mpsengine]
+                opt << "\\def\\MPOSTformatswitch\{#{prognameflag(progname)} #{formatflag('mpost')}=\}\n"
+                #
+                # process info
+                #
+                opt << "\\setupsystem[\\c!n=#{kindofrun},\\c!m=#{currentrun}]\n"
                 if (str = File.unixfied(getvariable('modefile'))) && ! str.empty? then
                     opt << "\\readlocfile{#{str}}{}{}\n"
                 end
@@ -1360,6 +1382,35 @@ class TEX
                 if (str = getvariable('mainlanguage').downcase) && ! str.empty? && ! str.standard? then
                     opt << "\\setuplanguage[#{str}]\n"
                 end
+                if (str = getvariable('arguments')) && ! str.empty? then
+                    opt << "\\setupenv[#{str}]\n"
+                end
+                if (str = getvariable('setuppath')) && ! str.empty? then
+                    opt << "\\setupsystem[\\c!directory=\{#{str}\}]\n"
+                end
+                if (str = getvariable('randomseed')) && ! str.empty? then
+                    report("using randomseed #{str}")
+                    opt << "\\setupsystem[\\c!random=#{str}]\n"
+                end
+                if (str = getvariable('input')) && ! str.empty? then
+                    opt << "\\setupsystem[inputfile=#{str}]\n"
+                else
+                    opt << "\\setupsystem[inputfile=#{rawname}]\n"
+                end
+                #
+                # modes
+                #
+                # we handle both "--mode" and "--modes", else "--mode" is mapped onto "--modefile"
+                if (str = getvariable('modes')) && ! str.empty? then
+                    opt << "\\enablemode[#{str}]\n"
+                end
+                if (str = getvariable('mode')) && ! str.empty? then
+                    opt << "\\enablemode[#{str}]\n"
+                end
+                #
+                # options
+                #
+                opt << "\\startsetups *runtime:options\n"
                 if str = validbackend(getvariable('backend')) then
                     opt << "\\setupoutput[#{str}]\n"
                 elsif str = validbackend(getvariable('output')) then
@@ -1368,21 +1419,9 @@ class TEX
                 if getvariable('color') then
                     opt << "\\setupcolors[\\c!state=\\v!start]\n"
                 end
-                if getvariable('nompmode') || getvariable('nomprun') || getvariable('automprun') then
-                    opt << "\\runMPgraphicsfalse\n"
-                end
-                if getvariable('fast') && ! getvariable('fastdisabled') then
-                    opt << "\\fastmode\n"
-                end
-                if getvariable('silentmode') then
-                    opt << "\\silentmode\n"
-                end
                 if (str = getvariable('separation')) && ! str.empty? then
                     opt << "\\setupcolors[\\c!split=#{str}]\n"
                 end
-                if (str = getvariable('setuppath')) && ! str.empty? then
-                    opt << "\\setupsystem[\\c!directory=\{#{str}\}]\n"
-                end
                 if (str = getvariable('paperformat')) && ! str.empty? && ! str.standard? then
                     if str =~ /^([a-z]+\d+)([a-z]+\d+)$/io then # A5A4 A4A3 A2A1 ...
                         opt << "\\setuppapersize[#{$1.upcase}][#{$2.upcase}]\n"
@@ -1399,9 +1438,6 @@ class TEX
                 if getvariable('centerpage') then
                     opt << "\\setuplayout[\\c!location=\\v!middle,\\c!marking=\\v!on]\n"
                 end
-                if getvariable('nomapfiles') then
-                    opt << "\\disablemapfiles\n"
-                end
                 if getvariable('noarrange') then
                     opt << "\\setuparranging[\\v!disable]\n"
                 elsif getvariable('arrange') then
@@ -1419,26 +1455,6 @@ class TEX
                     end
                     opt << "\\setuparranging[#{arrangement.flatten.join(',')}]\n" if arrangement.size > 0
                 end
-                # we handle both "--mode" and "--modes", else "--mode" is
-                # mapped onto "--modefile"
-                if (str = getvariable('modes')) && ! str.empty? then
-                    opt << "\\enablemode[#{str}]\n"
-                end
-                if (str = getvariable('mode')) && ! str.empty? then
-                    opt << "\\enablemode[#{str}]\n"
-                end
-                if (str = getvariable('arguments')) && ! str.empty? then
-                    opt << "\\setupenv[#{str}]\n"
-                end
-                if (str = getvariable('randomseed')) && ! str.empty? then
-                    report("using randomseed #{str}")
-                    opt << "\\setupsystem[\\c!random=#{str}]\n"
-                end
-                if (str = getvariable('input')) && ! str.empty? then
-                    opt << "\\setupsystem[inputfile=#{str}]\n"
-                else
-                    opt << "\\setupsystem[inputfile=#{rawname}]\n"
-                end
                 if (str = getvariable('pages')) && ! str.empty? then
                     if str.downcase == 'odd' then
                         opt << "\\chardef\\whichpagetoshipout=1\n"
@@ -1459,14 +1475,18 @@ class TEX
                         opt << "\\def\\pagestoshipout\{#{pagelist.join(',')}\}\n";
                     end
                 end
-                opt << "\\protect\n";
-              # begin getvariable('modes'       ).split(',').uniq.each do |e| opt << "\\enablemode  [#{e}]\n" end ; rescue ; end
+                opt << "\\stopsetups\n"
+                #
+                # styles and modules
+                #
+                opt << "\\startsetups *runtime:modules\n"
                 begin getvariable('filters'     ).split(',').uniq.each do |f| opt << "\\useXMLfilter[#{f}]\n" end ; rescue ; end
                 begin getvariable('usemodules'  ).split(',').uniq.each do |m| opt << "\\usemodule   [#{m}]\n" end ; rescue ; end
                 begin getvariable('environments').split(',').uniq.each do |e| opt << "\\environment  #{e} \n" end ; rescue ; end
-              # this will become:
-              # begin getvariable('environments').split(',').uniq.each do |e| opt << "\\useenvironment[#{e}]\n" end ; rescue ; end
-                opt << "\\endinput\n"
+                opt << "\\stopsetups\n"
+                #
+                opt << "\\protect \\endinput\n"
+                #
                 opt.close
            else
                 report("unable to write option file #{topname}")
@@ -1579,7 +1599,9 @@ end
         if mpsengine && mpsformat then
             ENV["MPXCOMMAND"] = "0" unless mpx
             progname = validprogname([getvariable('progname'),mpsformat,mpsengine])
-            runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),tcxflag(mpsengine),runoptions(mpsengine),mpname,mpsprocextras(mpsformat)])
+            mpname.gsub!(/\.mp$/,"") # temp bug in mp
+        #   runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),tcxflag(mpsengine),runoptions(mpsengine),mpname,mpsprocextras(mpsformat)])
+            runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),runoptions(mpsengine),mpname,mpsprocextras(mpsformat)])
             true
         else
             false
@@ -1589,7 +1611,7 @@ end
     def runtexmp(filename,filetype='',purge=true)
         checktestversion
         mpname = File.suffixed(filename,filetype,'mp')
-        if File.atleast?(mpname,25) then
+        if File.atleast?(mpname,10) then
             # first run needed
             File.silentdelete(File.suffixed(mpname,'mpt'))
             doruntexmp(mpname,nil,true,purge)
@@ -1627,7 +1649,7 @@ end
     end
 
     def runtexutil(filename=[], options=['--ref','--ij','--high'], old=false)
-        filename.each do |fname|
+        [filename].flatten.each do |fname|
             if old then
                 Kpse.runscript('texutil',fname,options)
             else
@@ -2065,7 +2087,7 @@ end
         setvariable('mp.line','')
         setvariable('mp.error','')
         if mpdata = File.silentread(mpname) then
-            mpdata.gsub!(/^\%.*\n/o,'')
+        #   mpdata.gsub!(/^\%.*\n/o,'')
             File.silentrename(mpname,mpcopy)
             texfound = mergebe || (mpdata =~ /btex .*? etex/mo)
             if mp = openedfile(mpname) then
@@ -2082,10 +2104,11 @@ end
                         mp << mergebe['0'] if mergebe.key?('0')
                     end
                 end
-                mp << MPTools::splitmplines(mpdata)
-                mp << "\n"
-                mp << "end"
+        #       mp << MPTools::splitmplines(mpdata)
+                mp << mpdata
                 mp << "\n"
+        #        mp << "end"
+        #        mp << "\n"
                 mp.close
             end
             processmpx(mpname,true,true,purge) if texfound
@@ -2097,7 +2120,10 @@ end
                 options = ''
             end
             # todo plain|mpost|metafun
-            ok = runmp(mpname)
+            begin
+                ok = runmp(mpname)
+            rescue
+            end
             if f = File.silentopen(File.suffixed(mpname,'log')) then
                 while str = f.gets do
                     if str =~ /^l\.(\d+)\s(.*?)\n/o then
diff --git a/scripts/context/ruby/base/texutil.rb b/scripts/context/ruby/base/texutil.rb
index 9e66aecff..3775469ed 100644
--- a/scripts/context/ruby/base/texutil.rb
+++ b/scripts/context/ruby/base/texutil.rb
@@ -475,24 +475,37 @@ class TeXUtil
                 @@debug = false
 
                 def initialize(t, c, k, d)
-                    @type, @command, @key, @sortkey, @data = t, c, k, k, d
+                    @type, @command, @key, @sortkey, @data = t, c, k, c, d
                 end
 
                 attr_reader :type, :command, :key, :data
                 attr_reader :sortkey
                 attr_writer :sortkey
 
+                # def build(sorter)
+                    # if @key then
+                        # @sortkey = sorter.normalize(sorter.tokenize(@sortkey))
+                        # @sortkey = sorter.remap(sorter.simplify(@key.downcase)) # ??
+                        # if @sortkey.empty? then
+                            # @sortkey = sorter.remap(@command.downcase)
+                        # end
+                    # else
+                        # @key = ""
+                        # @sortkey = ""
+                    # end
+                # end
+
                 def build(sorter)
-                    if @key then
+                    if @sortkey and not @sortkey.empty? then
                         @sortkey = sorter.normalize(sorter.tokenize(@sortkey))
-                        @sortkey = sorter.remap(sorter.simplify(@key.downcase)) # ??
-                        if @sortkey.empty? then
-                            @sortkey = sorter.remap(@command.downcase)
-                        end
-                    else
-                        @key = ""
-                        @sortkey = ""
-                        # weird
+                        @sortkey = sorter.remap(sorter.simplify(@sortkey.downcase)) # ??
+                    end
+                    if not @sortkey or @sortkey.empty? then
+                        @sortkey = sorter.normalize(sorter.tokenize(@key))
+                        @sortkey = sorter.remap(sorter.simplify(@sortkey.downcase)) # ??
+                    end
+                    if not @sortkey or @sortkey.empty? then
+                        @sortkey = @key.dup
                     end
                 end
 
diff --git a/scripts/context/ruby/base/tool.rb b/scripts/context/ruby/base/tool.rb
index 5ccedfec1..abf0d5ed0 100644
--- a/scripts/context/ruby/base/tool.rb
+++ b/scripts/context/ruby/base/tool.rb
@@ -24,10 +24,15 @@ module Tool
                     t = Time.now
                     u = t.usec.to_s % [1..2] [0..3]
                     pth = t.strftime("#{mainpath}%Y%m%d-%H%M%S-#{u}-#{Process.pid}")
-                    if pth == $constructedtempdir
-                        # sleep(0.01)
-                        retry
-                    end
+                #
+                #    problems with 1.9
+                #
+                #    if pth == $constructedtempdir
+                #        # sleep(0.01)
+                #        retry
+                #    end
+                    pth == $constructedtempdir
+                #
                     Dir.mkdir(pth) if create
                     $constructedtempdir = pth
                     return pth
@@ -216,7 +221,7 @@ module Tool
 
     def Tool.checksuffix(old)
 
-        return old unless test(?f,old)
+        return old unless FileTest.file?(old)
 
         new = old
 
diff --git a/scripts/context/ruby/ctxtools.rb b/scripts/context/ruby/ctxtools.rb
index b5e231e27..ecdc4c128 100644
--- a/scripts/context/ruby/ctxtools.rb
+++ b/scripts/context/ruby/ctxtools.rb
@@ -56,7 +56,8 @@ require 'base/file'
 
 require 'rexml/document'
 require 'net/http'
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 require 'kconv'
 
 exit if defined?(REQUIRE2LIB)
@@ -2279,6 +2280,7 @@ class TexDeps
         report("loading files")
         report('')
         n = 0
+# try tex and mkiv
         @files.each do |filename|
             if File.file?(filename) and f = File.open(filename) then
                 defs, uses, l = 0, 0, 0
diff --git a/scripts/context/ruby/fcd_start.rb b/scripts/context/ruby/fcd_start.rb
index 348ac75ba..28f407c76 100644
--- a/scripts/context/ruby/fcd_start.rb
+++ b/scripts/context/ruby/fcd_start.rb
@@ -260,6 +260,7 @@ class FastCD
                 puts(Dir.pwd.gsub(/\\/o, '/'))
             end
         rescue
+            puts("some error")
         end
     end
 
@@ -272,6 +273,7 @@ class FastCD
                     else
                         f.puts("cd #{dir.gsub("\\",'/')}")
                     end
+                    f.close
                 end
                 @result = dir
                 report("changing to #{dir}",true)
@@ -283,6 +285,7 @@ class FastCD
     end
 
     def choose(args=[])
+        offset = 97
         unless @pattern.empty? then
             begin
                 case @result.size
@@ -301,7 +304,7 @@ class FastCD
                                         return
                                     end
                                 else
-                                    index = answer[0] - ?a
+                                    index = answer[0] - offset
                                     if dir = list[index] then
                                         chdir(dir)
                                         return
@@ -309,21 +312,27 @@ class FastCD
                                 end
                             end
                         rescue
+                            puts("some error")
                         end
                         loop do
                             print("\n")
                             list.each_index do |i|
+begin
                                 if i < @@maxlength then
-                                    puts("#{(i+?a).chr}  #{list[i]}")
+                                    # puts("#{(i+?a).chr}  #{list[i]}")
+                                    puts("#{(i+offset).chr}  #{list[i]}")
                                 else
                                     puts("\n   there are #{list.length-@@maxlength} entries more")
                                     break
                                 end
+rescue
+                            puts("some error")
+end
                             end
                             print("\n>> ")
                             if answer = wait then
-                                if answer >= ?a and answer <= ?z then
-                                    index = answer - ?a
+                                if answer >= offset and answer <= offset+25 then
+                                    index = answer - offset
                                     if dir = list[index] then
                                         print("#{answer.chr} ")
                                         chdir(dir)
@@ -350,7 +359,7 @@ class FastCD
                         end
                     end
             rescue
-                # report($!)
+                 report($!)
             end
         end
     end
diff --git a/scripts/context/ruby/graphics/gs.rb b/scripts/context/ruby/graphics/gs.rb
index cb3d016f4..6143c8812 100644
--- a/scripts/context/ruby/graphics/gs.rb
+++ b/scripts/context/ruby/graphics/gs.rb
@@ -13,7 +13,8 @@
 
 require 'base/variables'
 require 'base/system'
-require 'ftools'
+require 'fileutils'
+# Require 'ftools'
 
 class GhostScript
 
@@ -218,15 +219,15 @@ class GhostScript
         rescue
             report("job aborted due to some error: #{$!}")
             begin
-                File.delete(resultfile) if test(?e,resultfile)
+                File.delete(resultfile) if FileTest.file?(resultfile)
             rescue
                 report("unable to delete faulty #{resultfile}")
             end
             ok = false
         ensure
             deleteprofile(getvariable('profile'))
-            File.delete(@@pstempfile)  if test(?e,@@pstempfile)
-            File.delete(@@pdftempfile) if test(?e,@@pdftempfile)
+            File.delete(@@pstempfile)  if FileTest.file?(@@pstempfile)
+            File.delete(@@pdftempfile) if FileTest.file?(@@pdftempfile)
         end
         return ok
     end
@@ -243,13 +244,14 @@ class GhostScript
 
     def pdfmethod? (str)
         case method(str).to_i
-            when 3, 4, 5 then return true
+            when 1, 3, 4, 5 then return true
         end
         return false
     end
 
     def pdfprefix (str)
         case method(str).to_i
+            when 1 then return 'raw-'
             when 4 then return 'lowres-'
             when 5 then return 'normal-'
         end
@@ -383,7 +385,7 @@ class GhostScript
             debug('piping data')
             unless pipebounded(tmp,eps) then
                 debug('something went wrong in the pipe')
-                File.delete(outfile) if test(?e,outfile)
+                File.delete(outfile) if FileTest.file?(outfile)
             end
             debug('closing pipe')
             eps.close_write
@@ -412,7 +414,7 @@ class GhostScript
             unless ok then
                 begin
                     report('no output file due to error')
-                    File.delete(outfile) if test(?e,outfile)
+                    File.delete(outfile) if FileTest.file?(outfile)
                 rescue
                     # debug("fatal error: #{$!}")
                     debug('file',outfile,'may be invalid')
@@ -421,7 +423,7 @@ class GhostScript
 
             debug('deleting temp file')
             begin
-                File.delete(@@pstempfile) if test(?e,@@pstempfile)
+                File.delete(@@pstempfile) if FileTest.file?(@@pstempfile)
             rescue
             end
 
@@ -467,7 +469,7 @@ class GhostScript
     # def convertcropped (inpfile, outfile)
         # report("converting #{inpfile} cropped")
         # do_convertbounded(inpfile, @@pdftempfile)
-        # return unless test(?e,@@pdftempfile)
+        # return unless FileTest.file?(@@pdftempfile)
         # arguments = " --offset=#{@offset} #{@@pdftempfile} #{outfile}"
         # report("calling #{@@pdftrimwhite}")
         # unless ok = System.run(@@pdftrimwhite,arguments) then
diff --git a/scripts/context/ruby/pdftools.rb b/scripts/context/ruby/pdftools.rb
index 23edfeca2..8ad74ec4f 100644
--- a/scripts/context/ruby/pdftools.rb
+++ b/scripts/context/ruby/pdftools.rb
@@ -20,7 +20,8 @@ $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.u
 require 'base/switch'
 require 'base/logger'
 
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 
 class File
 
diff --git a/scripts/context/ruby/rlxtools.rb b/scripts/context/ruby/rlxtools.rb
index 1617fcad4..36bc9f790 100644
--- a/scripts/context/ruby/rlxtools.rb
+++ b/scripts/context/ruby/rlxtools.rb
@@ -19,7 +19,8 @@ require 'base/logger'
 require 'base/system'
 require 'base/kpse'
 
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 require 'rexml/document'
 
 class Commands
diff --git a/scripts/context/ruby/rsfiltool.rb b/scripts/context/ruby/rsfiltool.rb
index f3abfdfc7..6d7c7aba0 100644
--- a/scripts/context/ruby/rsfiltool.rb
+++ b/scripts/context/ruby/rsfiltool.rb
@@ -18,7 +18,8 @@ end
 
 # todo : split session stuff from xmpl/base into an xmpl/session module and "include xmpl/session" into base and here and ...
 
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 require 'xmpl/base'
 require 'xmpl/switch'
 require 'xmpl/request'
diff --git a/scripts/context/ruby/runtools.rb b/scripts/context/ruby/runtools.rb
index 9c504845a..5565748e2 100644
--- a/scripts/context/ruby/runtools.rb
+++ b/scripts/context/ruby/runtools.rb
@@ -1,5 +1,6 @@
 require 'timeout'
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 require 'rbconfig'
 
 class File
diff --git a/scripts/context/ruby/texexec.rb b/scripts/context/ruby/texexec.rb
index a09572c6c..a549659ef 100644
--- a/scripts/context/ruby/texexec.rb
+++ b/scripts/context/ruby/texexec.rb
@@ -1,8 +1,9 @@
-banner = ['TeXExec', 'version 6.2.0', '1997-2006', 'PRAGMA ADE/POD']
+banner = ['TeXExec', 'version 6.2.1', '1997-2009', 'PRAGMA ADE/POD']
 
 $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq!
 
-require 'ftools'     # needed ?
+require 'fileutils'
+# require 'ftools'     # needed ?
 
 require 'base/switch'
 require 'base/logger'
@@ -277,6 +278,7 @@ class Commands
                     info = `pdfinfo #{filename}`
                     if info =~ /Pages:\s*(\d+)/ then
                         nofpages = $1.to_i
+                        result = @commandline.checkedoption('result','texexec')
                         nofpages.times do |i|
                             if f = File.open(tempfile,"w") then
                                 n = i + 1
@@ -285,8 +287,10 @@ class Commands
                                 f << "\\externalfigure[#{filename}][object=no,page=#{n}]\n"
                                 f << "\\stopTEXpage\\stoptext\n"
                                 f.close
+                                job.setvariable('result',"#{result}-#{n}")
                                 job.setvariable('interface','english') # redundant
                                 job.setvariable('simplerun',true)
+                                job.setvariable('purge',true)
                                 job.setvariable('files',[tempfile])
                                 job.processtex
                             end
diff --git a/scripts/context/ruby/texmfstart.rb b/scripts/context/ruby/texmfstart.rb
index 388bef85b..4976f7fd0 100644
--- a/scripts/context/ruby/texmfstart.rb
+++ b/scripts/context/ruby/texmfstart.rb
@@ -1,5 +1,9 @@
 #!/usr/bin/env ruby
 
+# We have removed the fast, server and client variants and no longer
+# provide the distributed 'serve trees' option. After all, we're now
+# using luatex.
+
 # program   : texmfstart
 # copyright : PRAGMA Advanced Document Engineering
 # version   : 1.9.0 - 2003/2006
@@ -18,8 +22,6 @@
 # Of couse I can make this into a nice class, which i'll undoubtely will
 # do when I feel the need. In that case it will be part of a bigger game.
 
-# turning this into a service would be nice, so some day ...
-
 # --locate        => provides location
 # --exec          => exec instead of system
 # --iftouched=a,b => only if timestamp a<>b
@@ -34,30 +36,17 @@
 $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq!
 
 require "rbconfig"
-require "md5"
+require "fileutils"
 
-# funny, selfmergs was suddenly broken to case problems
-
-# kpse_merge_done: require 'base/kpseremote'
-# kpse_merge_done: require 'base/kpsedirect'
-# kpse_merge_done: require 'base/kpsefast'
-# kpse_merge_done: require 'base/merge'
+require "digest/md5"
 
 # kpse_merge_start
 
-# kpse_merge_file: 't:/ruby/base/kpsefast.rb'
-
-# module    : base/kpsefast
-# copyright : PRAGMA Advanced Document Engineering
-# version   : 2005
-# author    : Hans Hagen
-#
-# project   : ConTeXt / eXaMpLe
-# concept   : Hans Hagen
-# info      : j.hagen@xs4all.nl
-
-# todo: multiple cnf files
-#
+class File
+    def File::makedirs(*x)
+        FileUtils.makedirs(x)
+    end
+end
 
 class String
 
@@ -106,1147 +95,6 @@ class File
 
 end
 
-module KpseUtil
-
-    # to be adapted, see loading cnf file
-
-    @@texmftrees = ['texmf-local','texmf.local','../..','texmf'] # '../..' is for gwtex
-    @@texmfcnf   = 'texmf.cnf'
-
-    def KpseUtil::identify
-        # we mainly need to identify the local tex stuff and wse assume that
-        # the texmfcnf variable is set; otherwise we need to expand the
-        # TEXMF variable and that takes time since it may involve more
-        ownpath = File.expand_path($0)
-        if ownpath.gsub!(/texmf.*?$/o, '') then
-            ENV['SELFAUTOPARENT'] = ownpath
-        else
-            ENV['SELFAUTOPARENT'] = '.' # fall back
-            # may be too tricky:
-            #
-            # (ENV['PATH'] ||'').split_path.each do |p|
-                # if p.gsub!(/texmf.*?$/o, '') then
-                    # ENV['SELFAUTOPARENT'] = p
-                    # break
-                # end
-            # end
-        end
-        filenames = Array.new
-        if ENV['TEXMFCNF'] && ! ENV['TEXMFCNF'].empty? then
-            ENV['TEXMFCNF'].to_s.split_path.each do |path|
-                filenames << File.join(path,@@texmfcnf)
-            end
-        elsif ENV['SELFAUTOPARENT'] == '.' then
-            filenames << File.join('.',@@texmfcnf)
-        else
-            @@texmftrees.each do |tree|
-                filenames << File.join(ENV['SELFAUTOPARENT'],tree,'web2c',@@texmfcnf)
-            end
-        end
-        loop do
-            busy = false
-            filenames.collect! do |f|
-                f.gsub(/\$([a-zA-Z0-9\_\-]+)/o) do
-                    if (! ENV[$1]) || (ENV[$1] == $1) then
-                        "$#{$1}"
-                    else
-                        busy = true
-                        ENV[$1]
-                    end
-                end
-            end
-            break unless busy
-        end
-        filenames.delete_if do |f|
-            ! FileTest.file?(f)
-        end
-        return filenames
-    end
-
-    def KpseUtil::environment
-        Hash.new.merge(ENV)
-    end
-
-end
-
-class KpseFast
-
-    # formats are an incredible inconsistent mess
-
-    @@suffixes  = Hash.new
-    @@formats   = Hash.new
-    @@suffixmap = Hash.new
-
-    @@texmfcnf  = 'texmf.cnf'
-
-    @@suffixes['gf']                       = ['.gf'] # todo
-    @@suffixes['pk']                       = ['.pk'] # todo
-    @@suffixes['tfm']                      = ['.tfm']
-    @@suffixes['afm']                      = ['.afm']
-    @@suffixes['base']                     = ['.base']
-    @@suffixes['bib']                      = ['.bib']
-    @@suffixes['bst']                      = ['.bst']
-    @@suffixes['cnf']                      = ['.cnf']
-    @@suffixes['ls-R']                     = ['ls-R', 'ls-r']
-    @@suffixes['fmt']                      = ['.fmt', '.efmt', '.efm', '.ofmt', '.ofm', '.oft', '.eofmt', '.eoft', '.eof', '.pfmt', '.pfm', '.epfmt', '.epf', '.xpfmt', '.xpf', '.afmt', '.afm']
-    @@suffixes['map']                      = ['.map']
-    @@suffixes['mem']                      = ['.mem']
-    @@suffixes['mf']                       = ['.mf']
-    @@suffixes['mfpool']                   = ['.pool']
-    @@suffixes['mft']                      = ['.mft']
-    @@suffixes['mp']                       = ['.mp']
-    @@suffixes['mppool']                   = ['.pool']
-    @@suffixes['ocp']                      = ['.ocp']
-    @@suffixes['ofm']                      = ['.ofm', '.tfm']
-    @@suffixes['opl']                      = ['.opl']
-    @@suffixes['otp']                      = ['.otp']
-    @@suffixes['ovf']                      = ['.ovf']
-    @@suffixes['ovp']                      = ['.ovp']
-    @@suffixes['graphic/figure']           = ['.eps', '.epsi']
-    @@suffixes['tex']                      = ['.tex']
-    @@suffixes['texpool']                  = ['.pool']
-    @@suffixes['PostScript header']        = ['.pro']
-    @@suffixes['type1 fonts']              = ['.pfa', '.pfb']
-    @@suffixes['vf']                       = ['.vf']
-    @@suffixes['ist']                      = ['.ist']
-    @@suffixes['truetype fonts']           = ['.ttf', '.ttc']
-    @@suffixes['web']                      = ['.web', '.ch']
-    @@suffixes['cweb']                     = ['.w', '.web', '.ch']
-    @@suffixes['enc files']                = ['.enc']
-    @@suffixes['cmap files']               = ['.cmap']
-    @@suffixes['subfont definition files'] = ['.sfd']
-    @@suffixes['lig files']                = ['.lig']
-    @@suffixes['bitmap font']              = []
-    @@suffixes['MetaPost support']         = []
-    @@suffixes['TeX system documentation'] = []
-    @@suffixes['TeX system sources']       = []
-    @@suffixes['Troff fonts']              = []
-    @@suffixes['dvips config']             = []
-    @@suffixes['type42 fonts']             = []
-    @@suffixes['web2c files']              = []
-    @@suffixes['other text files']         = []
-    @@suffixes['other binary files']       = []
-    @@suffixes['misc fonts']               = []
-    @@suffixes['opentype fonts']           = []
-    @@suffixes['pdftex config']            = []
-    @@suffixes['texmfscripts']             = []
-
-    # replacements
-
-    @@suffixes['fmt']                      = ['.fmt']
-    @@suffixes['type1 fonts']              = ['.pfa', '.pfb', '.pfm']
-    @@suffixes['tex']                      = ['.tex', '.xml']
-    @@suffixes['texmfscripts']             = ['rb','lua','py','pl']
-
-    @@suffixes.keys.each do |k| @@suffixes[k].each do |s| @@suffixmap[s] = k end end
-
-    # TTF2TFMINPUTS
-    # MISCFONTS
-    # TEXCONFIG
-    # DVIPDFMINPUTS
-    # OTFFONTS
-
-    @@formats['gf']                       = ''
-    @@formats['pk']                       = ''
-    @@formats['tfm']                      = 'TFMFONTS'
-    @@formats['afm']                      = 'AFMFONTS'
-    @@formats['base']                     = 'MFBASES'
-    @@formats['bib']                      = ''
-    @@formats['bst']                      = ''
-    @@formats['cnf']                      = ''
-    @@formats['ls-R']                     = ''
-    @@formats['fmt']                      = 'TEXFORMATS'
-    @@formats['map']                      = 'TEXFONTMAPS'
-    @@formats['mem']                      = 'MPMEMS'
-    @@formats['mf']                       = 'MFINPUTS'
-    @@formats['mfpool']                   = 'MFPOOL'
-    @@formats['mft']                      = ''
-    @@formats['mp']                       = 'MPINPUTS'
-    @@formats['mppool']                   = 'MPPOOL'
-    @@formats['ocp']                      = 'OCPINPUTS'
-    @@formats['ofm']                      = 'OFMFONTS'
-    @@formats['opl']                      = 'OPLFONTS'
-    @@formats['otp']                      = 'OTPINPUTS'
-    @@formats['ovf']                      = 'OVFFONTS'
-    @@formats['ovp']                      = 'OVPFONTS'
-    @@formats['graphic/figure']           = ''
-    @@formats['tex']                      = 'TEXINPUTS'
-    @@formats['texpool']                  = 'TEXPOOL'
-    @@formats['PostScript header']        = 'TEXPSHEADERS'
-    @@formats['type1 fonts']              = 'T1FONTS'
-    @@formats['vf']                       = 'VFFONTS'
-    @@formats['ist']                      = ''
-    @@formats['truetype fonts']           = 'TTFONTS'
-    @@formats['web']                      = ''
-    @@formats['cweb']                     = ''
-    @@formats['enc files']                = 'ENCFONTS'
-    @@formats['cmap files']               = 'CMAPFONTS'
-    @@formats['subfont definition files'] = 'SFDFONTS'
-    @@formats['lig files']                = 'LIGFONTS'
-    @@formats['bitmap font']              = ''
-    @@formats['MetaPost support']         = ''
-    @@formats['TeX system documentation'] = ''
-    @@formats['TeX system sources']       = ''
-    @@formats['Troff fonts']              = ''
-    @@formats['dvips config']             = ''
-    @@formats['type42 fonts']             = 'T42FONTS'
-    @@formats['web2c files']              = 'WEB2C'
-    @@formats['other text files']         = ''
-    @@formats['other binary files']       = ''
-    @@formats['misc fonts']               = ''
-    @@formats['opentype fonts']           = 'OPENTYPEFONTS'
-    @@formats['pdftex config']            = 'PDFTEXCONFIG'
-    @@formats['texmfscripts']             = 'TEXMFSCRIPTS'
-
-    attr_accessor :progname, :engine, :format, :rootpath, :treepath,
-        :verbose, :remember, :scandisk, :diskcache, :renewcache
-
-    @@cacheversion = '1'
-
-    def initialize
-        @rootpath    = ''
-        @treepath    = ''
-        @progname    = 'kpsewhich'
-        @engine      = 'pdftex'
-        @variables   = Hash.new
-        @expansions  = Hash.new
-        @files       = Hash.new
-        @found       = Hash.new
-        @kpsevars    = Hash.new
-        @lsrfiles    = Array.new
-        @cnffiles    = Array.new
-        @verbose     = true
-        @remember    = true
-        @scandisk    = true
-        @diskcache   = true
-        @renewcache  = false
-        @isolate     = false
-
-        @diskcache   = false
-        @cachepath   = nil
-        @cachefile   = 'tmftools.log'
-
-        @environment = ENV
-    end
-
-    def set(key,value)
-        case key
-            when 'progname' then @progname = value
-            when 'engine'   then @engine   = value
-            when 'format'   then @format   = value
-        end
-    end
-
-    def push_environment(env)
-        @environment = env
-    end
-
-    # {$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}
-    #
-    # $SELFAUTOLOC    : /usr/tex/bin/platform
-    # $SELFAUTODIR    : /usr/tex/bin
-    # $SELFAUTOPARENT : /usr/tex
-    #
-    # since we live in scriptpath we need a slightly different method
-
-    def load_cnf(filenames=nil)
-        unless filenames then
-            ownpath = File.expand_path($0)
-            if ownpath.gsub!(/texmf.*?$/o, '') then
-                @environment['SELFAUTOPARENT'] = ownpath
-            else
-                @environment['SELFAUTOPARENT'] = '.'
-            end
-            unless @treepath.empty? then
-                unless @rootpath.empty? then
-                    @treepath = @treepath.split(',').collect do |p| File.join(@rootpath,p) end.join(',')
-                end
-                @environment['TEXMF'] = @treepath
-                # only the first one
-                @environment['TEXMFCNF'] = File.join(@treepath.split(',').first,'texmf/web2c')
-            end
-            unless @rootpath.empty? then
-                @environment['TEXMFCNF'] = File.join(@rootpath,'texmf/web2c')
-                @environment['SELFAUTOPARENT'] = @rootpath
-                @isolate = true
-            end
-            filenames = Array.new
-            if @environment['TEXMFCNF'] and not @environment['TEXMFCNF'].empty? then
-                @environment['TEXMFCNF'].to_s.split_path.each do |path|
-                    filenames << File.join(path,@@texmfcnf)
-                end
-            elsif @environment['SELFAUTOPARENT'] == '.' then
-                filenames << File.join('.',@@texmfcnf)
-            else
-                ['texmf-local','texmf'].each do |tree|
-                    filenames << File.join(@environment['SELFAUTOPARENT'],tree,'web2c',@@texmfcnf)
-                end
-            end
-        end
-        # /texmf/web2c/texmf.cnf
-        filenames = _expanded_path_(filenames)
-        @rootpath = filenames.first
-        3.times do
-            @rootpath = File.dirname(@rootpath)
-        end
-        filenames.collect! do |f|
-            f.gsub("\\", '/')
-        end
-        filenames.each do |fname|
-            if FileTest.file?(fname) and f = File.open(fname) then
-                @cnffiles << fname
-                while line = f.gets do
-                    loop do
-                        # concatenate lines ending with \
-                        break unless line.sub!(/\\\s*$/o) do
-                            f.gets || ''
-                        end
-                    end
-                    case line
-                        when /^[\%\#]/o then
-                            # comment
-                        when /^\s*(.*?)\s*\=\s*(.*?)\s*$/o then
-                            key, value = $1, $2
-                            unless @variables.key?(key) then
-                                value.sub!(/\%.*$/,'')
-                                value.sub!(/\~/, "$HOME")
-                                @variables[key] = value
-                            end
-                            @kpsevars[key] = true
-                    end
-                end
-                f.close
-            end
-        end
-    end
-
-    def load_lsr
-        @lsrfiles = []
-        simplified_list(expansion('TEXMF')).each do |p|
-            ['ls-R','ls-r'].each do |f|
-                filename = File.join(p,f)
-                if FileTest.file?(filename) then
-                    @lsrfiles << [filename,File.size(filename)]
-                    break
-                end
-            end
-        end
-        @files = Hash.new
-        if @diskcache then
-            ['HOME','TEMP','TMP','TMPDIR'].each do |key|
-                if @environment[key] then
-                    if FileTest.directory?(@environment[key]) then
-                        @cachepath = @environment[key]
-                        @cachefile = [@rootpath.gsub(/[^A-Z0-9]/io, '-').gsub(/\-+/,'-'),File.basename(@cachefile)].join('-')
-                        break
-                    end
-                end
-            end
-            if @cachepath and not @renewcache and FileTest.file?(File.join(@cachepath,@cachefile)) then
-                begin
-                    if f = File.open(File.join(@cachepath,@cachefile)) then
-                        cacheversion = Marshal.load(f)
-                        if cacheversion == @@cacheversion then
-                            lsrfiles = Marshal.load(f)
-                            if lsrfiles == @lsrfiles then
-                                @files = Marshal.load(f)
-                            end
-                        end
-                        f.close
-                    end
-                rescue
-                   @files = Hash.new
-               end
-            end
-        end
-        return if @files.size > 0
-        @lsrfiles.each do |filedata|
-            filename, filesize = filedata
-            filepath = File.dirname(filename)
-            begin
-                path = '.'
-                data = IO.readlines(filename)
-                if data[0].chomp =~ /% ls\-R \-\- filename database for kpathsea\; do not change this line\./io then
-                    data.each do |line|
-                        case line
-                            when /^[a-zA-Z0-9]/o then
-                                line.chomp!
-                                if @files[line] then
-                                    @files[line] << path
-                                else
-                                    @files[line] = [path]
-                                end
-                            when /^\.\/(.*?)\:$/o then
-                                path = File.join(filepath,$1)
-                        end
-                    end
-                end
-            rescue
-                # sorry
-            end
-        end
-        if @diskcache and @cachepath and f = File.open(File.join(@cachepath,@cachefile),'wb') then
-            f << Marshal.dump(@@cacheversion)
-            f << Marshal.dump(@lsrfiles)
-            f << Marshal.dump(@files)
-            f.close
-        end
-    end
-
-    def expand_variables
-        @expansions = Hash.new
-        if @isolate then
-            @variables['TEXMFCNF'] = @environment['TEXMFCNF'].dup
-            @variables['SELFAUTOPARENT'] = @environment['SELFAUTOPARENT'].dup
-        else
-            @environment.keys.each do |e|
-                if e =~ /^([a-zA-Z]+)\_(.*)\s*$/o then
-                    @expansions["#{$1}.#{$2}"] = (@environment[e] ||'').dup
-                else
-                    @expansions[e] = (@environment[e] ||'').dup
-                end
-            end
-        end
-        @variables.keys.each do |k|
-            @expansions[k] = @variables[k].dup unless @expansions[k]
-        end
-        loop do
-            busy = false
-            @expansions.keys.each do |k|
-                @expansions[k].gsub!(/\$([a-zA-Z0-9\_\-]*)/o) do
-                    busy = true
-                    @expansions[$1] || ''
-                end
-                @expansions[k].gsub!(/\$\{([a-zA-Z0-9\_\-]*)\}/o) do
-                    busy = true
-                    @expansions[$1] || ''
-                end
-            end
-            break unless busy
-        end
-        @expansions.keys.each do |k|
-            @expansions[k] = @expansions[k].gsub("\\", '/')
-        end
-    end
-
-    def variable(name='')
-        (name and not name.empty? and @variables[name.sub('$','')]) or  ''
-    end
-
-    def expansion(name='')
-        (name and not name.empty? and @expansions[name.sub('$','')]) or ''
-    end
-
-    def variable?(name='')
-        name and not name.empty? and @variables.key?(name.sub('$',''))
-    end
-
-    def expansion?(name='')
-        name and not name.empty? and @expansions.key?(name.sub('$',''))
-    end
-
-    def simplified_list(str)
-        lst = str.gsub(/^\{/o,'').gsub(/\}$/o,'').split(",")
-        lst.collect do |l|
-            l.sub(/^[\!]*/,'').sub(/[\/\\]*$/o,'')
-        end
-    end
-
-    def original_variable(variable)
-        if variable?("#{@progname}.#{variable}") then
-            variable("#{@progname}.#{variable}")
-        elsif variable?(variable) then
-            variable(variable)
-        else
-            ''
-        end
-    end
-
-    def expanded_variable(variable)
-        if expansion?("#{variable}.#{@progname}") then
-            expansion("#{variable}.#{@progname}")
-        elsif expansion?(variable) then
-            expansion(variable)
-        else
-            ''
-        end
-    end
-
-    def original_path(filename='')
-        _expanded_path_(original_variable(var_of_format_or_suffix(filename)).split(";"))
-    end
-
-    def expanded_path(filename='')
-        _expanded_path_(expanded_variable(var_of_format_or_suffix(filename)).split(";"))
-    end
-
-    def _expanded_path_(pathlist)
-        i, n = 0, 0
-        pathlist.collect! do |mainpath|
-            mainpath.gsub(/([\{\}])/o) do
-                if $1 == "{" then
-                    i += 1 ; n = i if i > n ; "<#{i}>"
-                else
-                    i -= 1 ; ""
-                end
-            end
-        end
-        n.times do |i|
-            loop do
-                more = false
-                newlist = []
-                pathlist.each do |path|
-                    unless path.sub!(/^(.*?)<(#{n-i})>(.*?)<\/\2>(.*?)$/) do
-                        pre, mid, post = $1, $3, $4
-                        mid.gsub!(/\,$/,',.')
-                        mid.split(',').each do |m|
-                            more = true
-                            if m == '.' then
-                                newlist << "#{pre}#{post}"
-                            else
-                                newlist << "#{pre}#{m}#{post}"
-                            end
-                        end
-                    end then
-                        newlist << path
-                    end
-                end
-                if more then
-                    pathlist = [newlist].flatten # copy -)
-                else
-                    break
-                end
-            end
-        end
-        pathlist = pathlist.uniq.collect do |path|
-            p = path
-            # p.gsub(/^\/+/o) do '' end
-            # p.gsub!(/(.)\/\/(.)/o) do "#{$1}/#{$2}" end
-            # p.gsub!(/\/\/+$/o) do '//' end
-            p.gsub!(/\/\/+/o) do '//' end
-            p
-        end
-        pathlist
-    end
-
-    # todo: ignore case
-
-    def var_of_format(str)
-        @@formats[str] || ''
-    end
-
-    def var_of_suffix(str) # includes .
-        if @@suffixmap.key?(str) then @@formats[@@suffixmap[str]] else '' end
-    end
-
-    def var_of_format_or_suffix(str)
-        if @@formats.key?(str) then
-            @@formats[str]
-        elsif @@suffixmap.key?(File.extname(str)) then # extname includes .
-            @@formats[@@suffixmap[File.extname(str)]]  # extname includes .
-        else
-            ''
-        end
-    end
-
-end
-
-class KpseFast
-
-    # test things
-
-    def list_variables(kpseonly=true)
-        @variables.keys.sort.each do |k|
-            if kpseonly then
-                puts("#{k} = #{@variables[k]}") if @kpsevars[k]
-            else
-                puts("#{if @kpsevars[k] then 'K' else 'E' end} #{k} = #{@variables[k]}")
-            end
-        end
-    end
-
-    def list_expansions(kpseonly=true)
-        @expansions.keys.sort.each do |k|
-            if kpseonly then
-                puts("#{k} = #{@expansions[k]}") if @kpsevars[k]
-            else
-                puts("#{if @kpsevars[k] then 'K' else 'E' end} #{k} = #{@expansions[k]}")
-            end
-        end
-    end
-
-    def list_lsr
-        puts("files = #{@files.size}")
-    end
-
-    def set_test_patterns
-        @variables["KPSE_TEST_PATTERN_A"] = "foo/{1,2}/bar//"
-        @variables["KPSE_TEST_PATTERN_B"] = "!!x{A,B{1,2}}y"
-        @variables["KPSE_TEST_PATTERN_C"] = "x{A,B//{1,2}}y"
-        @variables["KPSE_TEST_PATTERN_D"] = "x{A,B//{1,2,}}//y"
-    end
-
-    def show_test_patterns
-        ['A','B','D'].each do |i|
-            puts ""
-            puts @variables ["KPSE_TEST_PATTERN_#{i}"]
-            puts ""
-            puts expand_path("KPSE_TEST_PATTERN_#{i}").split_path
-            puts ""
-        end
-    end
-
-end
-
-class KpseFast
-
-    # kpse stuff
-
-    def expand_braces(str) # output variable and brace expansion of STRING.
-        _expanded_path_(original_variable(str).split_path).join_path
-    end
-
-    def expand_path(str)   # output complete path expansion of STRING.
-        _expanded_path_(expanded_variable(str).split_path).join_path
-    end
-
-    def expand_var(str)    # output variable expansion of STRING.
-        expanded_variable(str)
-    end
-
-    def show_path(str)     # output search path for file type NAME
-        expanded_path(str).join_path
-    end
-
-    def var_value(str)     # output the value of variable $STRING.
-        original_variable(str)
-    end
-
-end
-
-class KpseFast
-
-    def _is_cnf_?(filename)
-        filename == File.basename((@cnffiles.first rescue @@texmfcnf) || @@texmfcnf)
-    end
-
-    def find_file(filename)
-        if _is_cnf_?(filename) then
-            @cnffiles.first rescue ''
-        else
-            [find_files(filename,true)].flatten.first || ''
-        end
-    end
-
-    def find_files(filename,first=false)
-        if _is_cnf_?(filename) then
-            result = @cnffiles.dup
-        else
-            if @remember then
-                # stamp = "#{filename}--#{@format}--#{@engine}--#{@progname}"
-                stamp = "#{filename}--#{@engine}--#{@progname}"
-                return @found[stamp] if @found.key?(stamp)
-            end
-            pathlist = expanded_path(filename)
-            result = []
-            filelist = if @files.key?(filename) then @files[filename].uniq else nil end
-            done = false
-            if pathlist.size == 0 then
-                if FileTest.file?(filename) then
-                    done = true
-                    result << '.'
-                end
-            else
-                pathlist.each do |path|
-                    doscan = if path =~ /^\!\!/o then false else true end
-                    recurse = if path =~ /\/\/$/o then true else false end
-                    pathname = path.dup
-                    pathname.gsub!(/^\!+/o, '')
-                    done = false
-                    if not done and filelist then
-                        # checking for exact match
-                        if filelist.include?(pathname) then
-                            result << pathname
-                            done = true
-                        end
-                        if not done and recurse then
-                            # checking for fuzzy //
-                            pathname.gsub!(/\/+$/o, '/.*')
-                            # pathname.gsub!(/\/\//o,'/[\/]*/')
-                            pathname.gsub!(/\/\//o,'/.*?/')
-                            re = /^#{pathname}/
-                            filelist.each do |f|
-                                if re =~ f then
-                                    result << f # duplicates will be filtered later
-                                    done = true
-                                end
-                                break if done
-                            end
-                        end
-                    end
-                    if not done and doscan then
-                        # checking for path itself
-                        pname = pathname.sub(/\.\*$/,'')
-                        if not pname =~ /\*/o and FileTest.file?(File.join(pname,filename)) then
-                            result << pname
-                            done = true
-                        end
-                    end
-                    break if done and first
-                end
-            end
-            if not done and @scandisk then
-                pathlist.each do |path|
-                    pathname = path.dup
-                    unless pathname.gsub!(/^\!+/o, '') then # !! prevents scan
-                        recurse = pathname.gsub!(/\/+$/o, '')
-                        complex = pathname.gsub!(/\/\//o,'/*/')
-                        if recurse then
-                            if complex then
-                                if ok = File.glob_file("#{pathname}/**/#{filename}") then
-                                    result << File.dirname(ok)
-                                    done = true
-                                end
-                            elsif ok = File.locate_file(pathname,filename) then
-                                result << File.dirname(ok)
-                                done = true
-                            end
-                        elsif complex then
-                            if ok = File.glob_file("#{pathname}/#{filename}") then
-                                result << File.dirname(ok)
-                                done = true
-                            end
-                        elsif FileTest.file?(File.join(pathname,filename)) then
-                            result << pathname
-                            done = true
-                        end
-                        break if done and first
-                    end
-                end
-            end
-            result = result.uniq.collect do |pathname|
-                File.join(pathname,filename)
-            end
-            @found[stamp] = result if @remember
-        end
-        return result # redundant
-    end
-
-end
-
-class KpseFast
-
-    class FileData
-        attr_accessor :tag, :name, :size, :date
-        def initialize(tag=0,name=nil,size=nil,date=nil)
-            @tag, @name, @size, @date = tag, name, size, date
-        end
-        def FileData.sizes(a)
-            a.collect do |aa|
-                aa.size
-            end
-        end
-        def report
-            case @tag
-                when 1 then "deleted  | #{@size.to_s.rjust(8)} | #{@date.strftime('%m/%d/%Y %I:%M')} | #{@name}"
-                when 2 then "present  | #{@size.to_s.rjust(8)} | #{@date.strftime('%m/%d/%Y %I:%M')} | #{@name}"
-                when 3 then "obsolete | #{' '*8} | #{' '*16} | #{@name}"
-            end
-        end
-    end
-
-    def analyze_files(filter='',strict=false,sort='',delete=false)
-        puts("command line     = #{ARGV.join(' ')}")
-        puts("number of files  = #{@files.size}")
-        puts("filter pattern   = #{filter}")
-        puts("loaded cnf files = #{@cnffiles.join(' ')}")
-        puts('')
-        if filter.gsub!(/^not:/,'') then
-            def the_same(filter,filename)
-                not filter or filter.empty? or /#{filter}/ !~ filename
-            end
-        else
-            def the_same(filter,filename)
-                not filter or filter.empty? or /#{filter}/ =~ filename
-            end
-        end
-        @files.keys.each do |name|
-            if @files[name].size > 1 then
-                data = Array.new
-                @files[name].each do |path|
-                    filename = File.join(path,name)
-                    # if not filter or filter.empty? or /#{filter}/ =~ filename then
-                    if the_same(filter,filename) then
-                        if FileTest.file?(filename) then
-                            if delete then
-                                data << FileData.new(1,filename,File.size(filename),File.mtime(filename))
-                                begin
-                                    File.delete(filename) if delete
-                                rescue
-                                end
-                            else
-                                data << FileData.new(2,filename,File.size(filename),File.mtime(filename))
-                            end
-                        else
-                            # data << FileData.new(3,filename)
-                        end
-                    end
-                end
-                if data.length > 1 then
-                    if strict then
-                        # if data.collect do |d| d.size end.uniq! then
-                            # data.sort! do |a,b| b.size <=> a.size end
-                            # data.each do |d| puts d.report end
-                            # puts ''
-                        # end
-                        data.sort! do |a,b|
-                            if a.size and b.size then
-                                b.size <=> a.size
-                            else
-                                0
-                            end
-                        end
-                        bunch = Array.new
-                        done = false
-                        data.each do |d|
-                            if bunch.size == 0 then
-                                bunch << d
-                            elsif bunch[0].size == d.size then
-                                bunch << d
-                            else
-                                if bunch.size > 1 then
-                                    bunch.each do |b|
-                                        puts b.report
-                                    end
-                                    done = true
-                                end
-                                bunch = [d]
-                            end
-                        end
-                        puts '' if done
-                    else
-                        case sort
-                            when 'size'    then data.sort! do |a,b| a.size <=> b.size end
-                            when 'revsize' then data.sort! do |a,b| b.size <=> a.size end
-                            when 'date'    then data.sort! do |a,b| a.date <=> b.date end
-                            when 'revdate' then data.sort! do |a,b| b.date <=> a.date end
-                        end
-                        data.each do |d| puts d.report end
-                        puts ''
-                    end
-                end
-            end
-        end
-    end
-
-end
-
-
-    # k = KpseFast.new # (root)
-    # k.set_test_patterns
-    # k.load_cnf
-    # k.expand_variables
-    # k.load_lsr
-
-    # k.show_test_patterns
-
-    # puts k.list_variables
-    # puts k.list_expansions
-    # k.list_lsr
-    # puts k.expansion("$TEXMF")
-    # puts k.expanded_path("TEXINPUTS","context")
-
-    # k.progname, k.engine, k.format = 'context', 'pdftex', 'tfm'
-    # k.scandisk = false # == must_exist
-    # k.expand_variables
-
-    # 10.times do |i| puts k.find_file('texnansi-lmr10.tfm') end
-
-    # puts "expand braces $TEXMF"
-    # puts k.expand_braces("$TEXMF")
-    # puts "expand path $TEXMF"
-    # puts k.expand_path("$TEXMF")
-    # puts "expand var $TEXMF"
-    # puts k.expand_var("$TEXMF")
-    # puts "expand path $TEXMF"
-    # puts k.show_path('tfm')
-    # puts "expand value $TEXINPUTS"
-    # puts k.var_value("$TEXINPUTS")
-    # puts "expand value $TEXINPUTS.context"
-    # puts k.var_value("$TEXINPUTS.context")
-
-    # exit
-
-
-
-# kpse_merge_file: 't:/ruby/base/kpse/trees.rb'
-
-require 'monitor'
-# kpse_merge_done: require 'base/kpsefast'
-
-class KpseTrees < Monitor
-
-    def initialize
-        @trees = Hash.new
-    end
-
-    def pattern(filenames)
-        filenames.join('|').gsub(/\\+/o,'/').downcase
-    end
-
-    def choose(filenames,environment)
-        current = pattern(filenames)
-        load(filenames,environment) unless @trees[current]
-        puts "enabling tree #{current}"
-        current
-    end
-
-    def fetch(filenames,environment) # will send whole object !
-        current = pattern(filenames)
-        load(filenames,environment) unless @trees[current]
-        puts "fetching tree #{current}"
-        @trees[current]
-    end
-
-    def load(filenames,environment)
-        current = pattern(filenames)
-        puts "loading tree #{current}"
-        @trees[current] = KpseFast.new
-        @trees[current].push_environment(environment)
-        @trees[current].load_cnf(filenames)
-        @trees[current].expand_variables
-        @trees[current].load_lsr
-    end
-
-    def set(tree,key,value)
-        case key
-            when 'progname' then @trees[tree].progname = value
-            when 'engine'   then @trees[tree].engine   = value
-            when 'format'   then @trees[tree].format   = value
-        end
-    end
-    def get(tree,key)
-        case key
-            when 'progname' then @trees[tree].progname
-            when 'engine'   then @trees[tree].engine
-            when 'format'   then @trees[tree].format
-        end
-    end
-
-    def load_cnf(tree)
-        @trees[tree].load_cnf
-    end
-    def load_lsr(tree)
-        @trees[tree].load_lsr
-    end
-    def expand_variables(tree)
-        @trees[tree].expand_variables
-    end
-    def expand_braces(tree,str)
-        @trees[tree].expand_braces(str)
-    end
-    def expand_path(tree,str)
-        @trees[tree].expand_path(str)
-    end
-    def expand_var(tree,str)
-        @trees[tree].expand_var(str)
-    end
-    def show_path(tree,str)
-        @trees[tree].show_path(str)
-    end
-    def var_value(tree,str)
-        @trees[tree].var_value(str)
-    end
-    def find_file(tree,filename)
-        @trees[tree].find_file(filename)
-    end
-    def find_files(tree,filename,first)
-        @trees[tree].find_files(filename,first)
-    end
-
-end
-
-
-# kpse_merge_file: 't:/ruby/base/kpse/drb.rb'
-
-require 'drb'
-# kpse_merge_done: require 'base/kpse/trees'
-
-class KpseServer
-
-    attr_accessor :port
-
-    def initialize(port=7000)
-        @port = port
-    end
-
-    def start
-        puts "starting drb service at port #{@port}"
-        DRb.start_service("druby://localhost:#{@port}", KpseTrees.new)
-        trap(:INT) do
-            DRb.stop_service
-        end
-        DRb.thread.join
-    end
-
-    def stop
-        # todo
-    end
-
-end
-
-class KpseClient
-
-    attr_accessor :port
-
-    def initialize(port=7000)
-        @port = port
-        @kpse = nil
-    end
-
-    def start
-        # only needed when callbacks are used / slow, due to Socket::getaddrinfo
-        # DRb.start_service
-    end
-
-    def object
-        @kpse = DRbObject.new(nil,"druby://localhost:#{@port}")
-    end
-
-end
-
-
-# SERVER_URI="druby://localhost:8787"
-#
-#   # Start a local DRbServer to handle callbacks.
-#   #
-#   # Not necessary for this small example, but will be required
-#   # as soon as we pass a non-marshallable object as an argument
-#   # to a dRuby call.
-#   DRb.start_service
-#
-
-
-# kpse_merge_file: 't:/ruby/base/kpseremote.rb'
-
-# kpse_merge_done: require 'base/kpsefast'
-
-case ENV['KPSEMETHOD']
-    when /soap/o then # kpse_merge_done: require 'base/kpse/soap'
-    when /drb/o  then # kpse_merge_done: require 'base/kpse/drb'
-    else              # kpse_merge_done: require 'base/kpse/drb'
-end
-
-class KpseRemote
-
-    @@port   = ENV['KPSEPORT']   || 7000
-    @@method = ENV['KPSEMETHOD'] || 'drb'
-
-    def KpseRemote::available?
-        @@method && @@port
-    end
-
-    def KpseRemote::start_server(port=nil)
-        kpse = KpseServer.new(port || @@port)
-        kpse.start
-    end
-
-    def KpseRemote::start_client(port=nil) # keeps object in server
-        kpseclient = KpseClient.new(port || @@port)
-        kpseclient.start
-        kpse = kpseclient.object
-        tree = kpse.choose(KpseUtil::identify, KpseUtil::environment)
-        [kpse, tree]
-    end
-
-    def KpseRemote::fetch(port=nil) # no need for defining methods but slower, send whole object
-        kpseclient = KpseClient.new(port || @@port)
-        kpseclient.start
-        kpseclient.object.fetch(KpseUtil::identify, KpseUtil::environment) rescue nil
-    end
-
-    def initialize(port=nil)
-        if KpseRemote::available? then
-            begin
-                @kpse, @tree = KpseRemote::start_client(port)
-            rescue
-                @kpse, @tree = nil, nil
-            end
-        else
-            @kpse, @tree = nil, nil
-        end
-    end
-
-    def progname=(value)
-        @kpse.set(@tree,'progname',value)
-    end
-    def format=(value)
-        @kpse.set(@tree,'format',value)
-    end
-    def engine=(value)
-        @kpse.set(@tree,'engine',value)
-    end
-
-    def progname
-        @kpse.get(@tree,'progname')
-    end
-    def format
-        @kpse.get(@tree,'format')
-    end
-    def engine
-        @kpse.get(@tree,'engine')
-    end
-
-    def load
-        @kpse.load(KpseUtil::identify, KpseUtil::environment)
-    end
-    def okay?
-        @kpse && @tree
-    end
-    def set(key,value)
-        @kpse.set(@tree,key,value)
-    end
-    def load_cnf
-        @kpse.load_cnf(@tree)
-    end
-    def load_lsr
-        @kpse.load_lsr(@tree)
-    end
-    def expand_variables
-        @kpse.expand_variables(@tree)
-    end
-    def expand_braces(str)
-        clean_name(@kpse.expand_braces(@tree,str))
-    end
-    def expand_path(str)
-        clean_name(@kpse.expand_path(@tree,str))
-    end
-    def expand_var(str)
-        clean_name(@kpse.expand_var(@tree,str))
-    end
-    def show_path(str)
-        clean_name(@kpse.show_path(@tree,str))
-    end
-    def var_value(str)
-        clean_name(@kpse.var_value(@tree,str))
-    end
-    def find_file(filename)
-        clean_name(@kpse.find_file(@tree,filename))
-    end
-    def find_files(filename,first=false)
-        # dodo: each filename
-        @kpse.find_files(@tree,filename,first)
-    end
-
-    private
-
-    def clean_name(str)
-        str.gsub(/\\/,'/')
-    end
-
-end
-
-
 # kpse_merge_file: 't:/ruby/base/kpsedirect.rb'
 
 class KpseDirect
@@ -1284,14 +132,11 @@ class KpseDirect
 
 end
 
-
 # kpse_merge_stop
 
-
-
 $mswindows = Config::CONFIG['host_os'] =~ /mswin/
 $separator = File::PATH_SEPARATOR
-$version   = "2.0.3"
+$version   = "2.1.0"
 $ownpath   = File.dirname($0)
 
 if $mswindows then
@@ -1335,7 +180,7 @@ $predefined['ctxtools'] = 'ctxtools.rb'
 $predefined['rlxtools'] = 'rlxtools.rb'
 $predefined['pdftools'] = 'pdftools.rb'
 $predefined['mpstools'] = 'mpstools.rb'
-$predefined['exatools'] = 'exatools.rb'
+# $predefined['exatools'] = 'exatools.rb'
 $predefined['xmltools'] = 'xmltools.rb'
 # $predefined['luatools'] = 'luatools.lua'
 # $predefined['mtxtools'] = 'mtxtools.rb'
@@ -1412,22 +257,7 @@ def check_kpse
     if $kpse then
         # already done
     else
-        begin
-            if KpseRemote::available? then
-                $kpse = KpseRemote.new
-                if $kpse.okay? then
-                    puts("kpse     : remote") if $verbose
-                else
-                    $kpse = KpseDirect.new
-                    puts("kpse     : direct (forced)") if $verbose
-                end
-            else
-                $kpse = KpseDirect.new
-                puts("kpse     : direct") if $verbose
-            end
-        rescue
-            puts("kpse     : direct (fallback)") if $verbose
-        end
+        $kpse = KpseDirect.new
     end
 end
 
@@ -1488,18 +318,6 @@ end
 
 class File
 
-    # def File.needsupdate(oldname,newname)
-        # begin
-            # if $mswindows then
-                # return File.stat(oldname).mtime > File.stat(newname).mtime
-            # else
-                # return File.stat(oldname).mtime != File.stat(newname).mtime
-            # end
-        # rescue
-            # return true
-        # end
-    # end
-
     @@update_eps = 1
 
     def File.needsupdate(oldname,newname)
@@ -1543,33 +361,6 @@ class File
 
 end
 
-# def hashed (arr=[])
-    # arg = if arr.class == String then arr.split(' ') else arr.dup end
-    # hsh = Hash.new
-    # if arg.length > 0
-        # hsh['arguments'] = ''
-        # done = false
-        # arg.each do |s|
-            # if done then
-                # if s =~ / / then
-                   # hsh['arguments'] += " \"#{s}\"" # maybe split on =
-                # else
-                   # hsh['arguments'] += " #{s}"
-                # end
-            # else
-                # kvl = s.split('=')
-                # if kvl[0].sub!(/^\-+/,'') then
-                    # hsh[kvl[0]] = if kvl.length > 1 then kvl[1] else true end
-                # else
-                    # hsh['file'] = s
-                    # done = true
-                # end
-            # end
-        # end
-    # end
-    # return hsh
-# end
-
 def hashed (arr=[])
     arg = if arr.class == String then arr.split(' ') else arr.dup end
     hsh = Hash.new
@@ -2115,12 +906,6 @@ end
 
 def make(filename,windows=false,linux=false,remove=false)
     basename = File.basename(filename).gsub(/\.[^.]+?$/, '')
-# if @kpse.find_file(@tree,filename+".lua") or @kpse.find_file(@tree,filename+".rb") or @kpse.find_file(@tree,filename+".pl") then
-    # make stub indeed
-# else
-    # report("no stub needed for '#{basename}'")
-    # return
-# end
     if $stubpath == 'auto' then
         basename = File.dirname($0) + '/' + basename
     else
@@ -2203,7 +988,7 @@ def process(&block)
         checkname = filename + ".md5"
         oldchecksum, newchecksum = "old", "new"
         begin
-            newchecksum = MD5.new(IO.read(filename)).hexdigest.upcase
+            newchecksum = Digest::MD5.hexdigest(IO.read(filename)).upcase
         rescue
             newchecksum = "new"
         else
@@ -2423,22 +1208,6 @@ def execute(arguments) # br global
     elsif $selfcleanup then
         output("ruby libraries are cleaned up") if SelfMerge::cleanup
         return true
-    elsif $serve then
-        if ENV['KPSEMETHOD'] && ENV['KPSEPORT'] then
-            # # kpse_merge_done: require 'base/kpseremote'
-            begin
-                KpseRemote::start_server
-            rescue
-                return false
-            else
-                return true
-            end
-        else
-            usage
-            puts("")
-            puts("message  : set 'KPSEMETHOD' and 'KPSEPORT' variables")
-            return false
-        end
     elsif $help || ! $filename || $filename.empty? then
         usage
         loadtree($tree)
diff --git a/scripts/context/ruby/textools.rb b/scripts/context/ruby/textools.rb
index 442dc1924..a5858c5ca 100644
--- a/scripts/context/ruby/textools.rb
+++ b/scripts/context/ruby/textools.rb
@@ -20,7 +20,8 @@ $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.u
 require 'base/switch'
 require 'base/logger'
 
-require 'ftools'
+require 'fileutils'
+# require 'ftools'
 
 # Remark
 #
diff --git a/scripts/context/ruby/www/admin.rb b/scripts/context/ruby/www/admin.rb
deleted file mode 100644
index 8983ca2b7..000000000
--- a/scripts/context/ruby/www/admin.rb
+++ /dev/null
@@ -1,215 +0,0 @@
-require 'fileutils'
-
-require 'www/lib'
-require 'www/dir'
-require 'www/common'
-
-class WWW
-
-    include Common
-
-    # klopt nog niet, twee keer task met een verschillend doel
-
-    def handle_exatask
-        # case @session.check('task', request_variable('task'))
-        task, options, option = @session.get('task'), @session.get('option').split(@@re_bar), request_variable('option')
-        option = (options.first || '') if option.empty?
-        case task
-            when 'exaadmin'
-                @session.set('status', 'admin') # admin: status|dir
-                touch_session(@session.get('id'))
-                if options.include?(option) then
-                    case option
-                        when 'status' then handle_exaadmin_status
-                        when 'dir'    then handle_exaadmin_dir
-                        else               handle_exaadmin_status
-                        end
-                elsif option.empty? then
-                    message('Status', "unknown option")
-                else
-                    message('Status', "option '#{option}' not permitted #{options.inspect}")
-                end
-            else
-                message('Status', "unknown task '#{task}'")
-        end
-    end
-
-    def handle_exaadmin
-        if id = valid_session() then
-            handle_exatask
-        else
-            message('Status', 'no login')
-        end
-    end
-
-    def handle_exaadmin_dir
-        check_template_file('exalogin','exalogin-template.htm')
-        @interface.set('path:docroot', work_root)
-        @interface.set('dir:uri', 'exaadmin')   # forces the dir handler into cgi mode
-        @interface.set('dir:task', 'exaadmin')  # forces the dir handler into cgi mode
-        @interface.set('dir:option', 'dir')     # forces the dir handler into cgi mode
-        filename = "#{@@session_prefix}#{request_variable('path')}"
-        fullname = File.join(work_root,filename)
-        if request_variable('path').empty? then
-            handle_exaadmin_status
-        elsif FileTest.directory?(fullname) then
-            handle_dir(filename, [], false)
-        elsif File.zero?(fullname) then
-            message('Error', "The file '#{filename}' is empty")
-        elsif File.size?(fullname) > (4 * 1024 * 1024) then
-            if FileTest.file?(File.expand_path(File.join(cache_root,filename))) then
-                str = "

Cached alternative: #{File.basename(filename)}"
-            else
-                str = ''
-            end
-            message('Error', "The file '#{filename}' is too big to serve over cgi." + str)
-        else
-            send_file(fullname)
-        end
-    end
-
-    def handle_exaadmin_status
-        check_template_file('exalogin','exalogin-template.htm')
-        begin
-            n, str, lines, list, start, most, least, cached = 0, '', '', Hash.new, Time.now, 0, 0, false
-            filename = File.join(tmp_path(dirname),'sessions.rbd')
-            begin
-                File.open(filename) do |f|
-                    list = Marshal.load(f)
-                end
-            rescue
-                cached, list = false, Hash.new
-            else
-                cached = true
-            end
-            files = Dir.glob("{#{work_roots.join(',')}}/#{@@session_prefix}*.ses")
-            list.keys.each do |l|
-                list.delete(l) unless files.include?(l) # slow
-            end
-            files.each do |f|
-                ctime = File.ctime(f)
-                stime = list[f][0] == ctime rescue 0
-                unless ctime == stime then
-                    begin
-                        hash = load_session_file(f)
-                    rescue
-                    else
-                        list[f] = [ctime,hash]
-                    end
-                end
-            end
-            begin
-                File.open(filename,'w') do |f|
-                    f << Marshal.dump(list)
-                end
-            rescue
-                # no save
-            end
-            begin
-                keys = list.keys.sort do |a,b|
-                    case list[b][0] <=> list[a][0]
-                        when -1 then -1
-                        when +1 then +1
-                    else
-                        a <=> b
-                    end
-                end
-            rescue
-                keys = list.keys.sort
-            end
-            totaltime, totaldone = 0.0, 0
-            if keys.length > 0 then
-                keys.each do |entry|
-                    s, t, session = entry, list[entry][0], list[entry][1]
-                    status = session['status'] || ''
-                    runtime = (session['runtime'] || '').to_f rescue 0
-                    starttime = (start.to_i-session['starttime'].to_i).to_s rescue ''
-                    requesttime = session['endtime'].to_i-session['starttime'].to_i rescue 0
-                    requesttime = if requesttime > 0 then requesttime.to_s else '' end
-                    if runtime > 0.0 then
-                        totaltime += runtime
-                        totaldone += 1
-                        if least > 0 then
-                            if runtime < least then least = runtime end
-                        else
-                            least = runtime
-                        end
-                        if most > 0 then
-                            if runtime > most then most = runtime end
-                        else
-                            most = runtime
-                        end
-                    end
-                    if status.empty? then
-                        # skip, garbage
-                    elsif status =~ /^(|exa)admin/o then
-                        # skip, useless
-                    else
-                        begin
-                            lines << "\n"
-                            lines << td("#{session['id']}")
-                            lines << td(status)
-                            lines << td(session['timeout'])
-                            lines << td(starttime)
-                            lines << td(session['runtime'])
-                            lines << td(requesttime)
-                            lines << td(t.strftime("%H:%M:%S %Y-%m-%d"))
-                            lines << td(session['domain'])
-                            lines << td(session['project'])
-                            lines << td(session['username'])
-                            lines << td(File.basename(File.dirname(s)))
-                            lines << "\n"
-                        rescue
-                        else
-                            n += 1
-                        end
-                    end
-                end
-                if n > 0 then
-                    str = "\n"
-                    str << "\n"
-                    str << th('session identifier')
-                    str << th('status')
-                    str << th('timeout')
-                    str << th('time')
-                    str << th('runtime')
-                    str << th('total')
-                    str << th('modification time')
-                    str << th('domain')
-                    str << th('project')
-                    str << th('username')
-                    str << th('process')
-                    str << "\n"
-                    str << lines
-                    str << "
\n"
-                end
-            end
-        rescue
-            message('Status', "#{$!} There is currently no status available.", false, @@admin_refresh, 'exaadmin')
-        else
-            if n > 0 then
-                # r = if n > 100 then 60 else @@admin_refresh.to_i end # scanning takes long
-                r = @@admin_refresh
-                average  = "average = #{if totaldone > 0 then sprintf('%.02f',totaltime/totaldone) else '0' end} (#{sprintf('%.02f',least)} .. #{sprintf('%.02f',most)})"
-                sessions = "sessions = #{n}"
-                refresh  = "refresh = #{r.to_s} sec"
-                loadtime = "loadtime = #{sprintf('%.04f',Time.now-start)} sec"
-                cached   = if cached then "cached" else "not cached" end
-                message("Status | #{sessions} | #{refresh} | #{loadtime} - #{cached} | #{average} |", str, false, r, 'exaadmin')
-            else
-                message('Status', "There are no sessions registered.", false, @@admin_refresh, 'exaadmin')
-            end
-        end
-    end
-
-    private
-
-    def th(str)
-        "#{str}   \n"
-    end
-
-    def td(str)
-        "#{str || ''}   \n"
-    end
-
-end
diff --git a/scripts/context/ruby/www/common.rb b/scripts/context/ruby/www/common.rb
deleted file mode 100644
index 9c3832294..000000000
--- a/scripts/context/ruby/www/common.rb
+++ /dev/null
@@ -1,80 +0,0 @@
-# We cannot chdir in threads because it is something
-# process wide, so we will run into problems with the
-# other threads. The same is true for the global ENV
-# pseudo hash, so we cannot communicate the runpath
-# via an anvironment either. This leaves texmfstart
-# in combination with a path directive and an tmf file.
-
-module Common # can be a mixin
-
-    # we assume that the hash.subset method is defined
-
-    @@re_texmfstart = /^(texmfstart|ruby\s*texmfstart.rb)\s*(.*)$/
-    @@re_texmfpath  = /^\-\-path\=/
-
-    def command_string(path,command,log='')
-        runner = "texmfstart --path=#{File.expand_path(path)}"
-        if command =~ @@re_texmfstart then
-            cmd, arg = $1, $2
-            if arg =~ @@re_texmfpath then
-                # there is already an --path (first switch)
-            else
-                command = "#{runner} #{arg}"
-            end
-        else
-            command = "#{runner} bin:#{command}"
-        end
-        if log && ! log.empty? then
-            return "#{command} 2>&1 > #{File.expand_path(File.join(path,log))}"
-        else
-            return command
-        end
-    end
-
-    def set_os_vars
-        begin
-            ENV['TEXOS'] = ENV['TEXOS'] || platform
-        rescue
-            ENV['TEXOS'] = 'texmf-linux'
-        else
-            ENV['TEXOS'] = 'texmf-' + ENV['TEXOS'] unless ENV['TEXOS'] =~ /^texmf\-/
-        ensure
-            ENV['EXA:TEXOS'] = ENV['TEXOS']
-        end
-    end
-
-    def set_environment(hash)
-        set_os_vars
-        paths = ENV['PATH'].split(File::PATH_SEPARATOR)
-        hash.subset('binpath:').keys.each do |key|
-            begin
-                paths << File.expand_path(hash[key])
-            rescue
-            end
-        end
-        ENV['PATH'] = paths.uniq.join(File::PATH_SEPARATOR)
-        hash.subset('path:').keys.each do |path|
-            key, value = "EXA:#{path.upcase}", File.expand_path(hash[path])
-            ENV[key] = value
-        end
-    end
-
-    def save_environment(hash,path,filename='request.tmf')
-        begin
-            File.open(File.join(path,filename),'w') do |f|
-                set_os_vars
-                ['EXA:TEXOS','TEXOS'].each do |key|
-                    f.puts("#{key} = #{ENV[key]}")
-                end
-                hash.subset('binpath:').keys.each do |key|
-                    f.puts("PATH < #{File.expand_path(@interface.get(key))}")
-                end
-                hash.subset('path:').keys.each do |path|
-                    f.puts("EXA:#{path.upcase} = #{File.expand_path(@interface.get(path))}")
-                end
-            end
-        rescue
-        end
-    end
-
-end
diff --git a/scripts/context/ruby/www/dir.rb b/scripts/context/ruby/www/dir.rb
deleted file mode 100644
index 115fd8911..000000000
--- a/scripts/context/ruby/www/dir.rb
+++ /dev/null
@@ -1,155 +0,0 @@
-require 'www/lib'
-
-# dir handling
-
-class WWW
-
-    # borrowed code from webrick demo, patched
-
-    @@dir_name_width = 25
-
-    def handle_dir(dirpath=@variables.get('path'),hidden=[],showdirs=true)
-        check_template_file('dir','text-template.htm')
-        docroot = @interface.get('path:docroot')
-        dirpath = dirpath || ''
-        hidden = [] unless hidden
-        local_path = dirpath.dup
-        title, str = "Index of #{escaped(dirpath)}", ''
-        begin
-            local_path.gsub!(/[\/\\]+/,'/')
-            local_path.gsub!(/\/$/, '')
-            if local_path !~ /^(\.|\.\.|\/|[a-zA-Z]\:)$/io then # maybe also /...
-                full_path = File.join(docroot,local_path)
-                @interface.set('log:dir', full_path)
-                begin
-                    list = Dir::entries(full_path)
-                rescue
-                    str << "unable to parse #{local_path}"
-                else
-                    if list then
-                        list.collect! do |name|
-                            if name =~ /^\.+/o then
-                                nil # no . and ..
-                            else
-                                st = (File::stat(File.join(docroot,local_path,name)) rescue nil)
-                                if st.nil? then
-                                    [name, nil, -1, false]
-                                elsif st.directory? then
-                                    if showdirs then [name + "/", st.mtime, -1, true] else nil end
-                                elsif hidden.length > 0 then
-                                    if hidden.include?(name) then nil else [name, st.mtime, st.size, false] end
-                                else
-                                    [name, st.mtime, st.size, false]
-                                end
-                            end
-                        end
-                        list.compact!
-                        n, m, s = @variables.get('n'), @variables.get('m'), @variables.get('s')
-                        if    ! n.empty? then
-                            idx, d0 = 0, n
-                        elsif ! m.empty? then
-                            idx, d0 = 1, m
-                        elsif ! s.empty? then
-                            idx, d0 = 2, s
-                        else
-                            idx, d0 = 0, 'a'
-                        end
-                        d1 = if d0 == 'a' then 'd' else 'a' end
-                        if d0 == 'a' then
-                            list.sort! do |a,b| a[idx] <=> b[idx] end
-                        else
-                            list.sort! do |a,b| b[idx] <=> a[idx] end
-                        end
-                        u = dir_uri(@variables.get('path') || '.')
-                        str << "
\n
\n"
-                        str << "name".ljust(49+u.length)
-                        str << "last modified".ljust(41+u.length)
-                        str << "size".rjust(31+u.length) << "\n" << "\n"
-                        # parent path
-                        if showdirs && ! hidden.include?('..') then
-                            dname = "parent directory"
-                            fname = "#{File.dirname(dirpath)}"
-                            time = File::mtime(File.join(docroot,local_path,"/.."))
-                            str << dir_entry(fname,dname,time,-1,true)
-                            str << "\n"
-                        end
-                        # directories
-                        done = false
-                        list.each do |name, time, size, dir|
-                            if dir then
-                                if name.size > @@dir_name_width then
-                                    dname = name.sub(/^(.#{@@dir_name_width-2})(.*)/) do $1 + ".." end
-                                else
-                                    dname = name
-                                end
-                                fname = "#{escaped(dirpath)}/#{escaped(name)}"
-                                str << dir_entry(fname,dname,time,size,dir)
-                                done = true
-                            end
-                        end
-                        str << "\n" if done
-                        # files
-                        list.each do |name, time, size, dir|
-                            unless dir then
-                                if name.size > @@dir_name_width then
-                                    dname = name.sub(/^(.#{@@dir_name_width-2})(.*)/) do $1 + ".." end
-                                else
-                                    dname = name
-                                end
-                                fname = "#{escaped(dirpath)}/#{escaped(name)}"
-                                str << dir_entry(fname,dname,time,size,dir)
-                            end
-                        end
-                        str << "\n"
-                        str << '
'
-                    else
-                        str << 'no info'
-                    end
-                end
-            else
-                str << 'no access'
-            end
-        rescue
-            str << "error #{$!}
"
-            str << $@.join("\n")
-            str << "
"
-        end
-        message(title,str)
-    end
-    def dir_uri(f='.')
-        u, t, o = @interface.get('dir:uri'), @interface.get('dir:task'), @interface.get('dir:option') # takes precedence, in case we run under cgi control
-        if u.empty? then
-            u, t, o = @interface.get('process:uri'), '', ''
-        elsif ! t.empty? then
-            t = "task=#{t}&"
-            o = "option=#{o}&"
-        end
-        if u && ! u.empty? then
-            u = u.sub(/\?.*$/,'') # frozen string
-            if f =~ /^\.+$/ then
-                "#{u}?#{t}#{o}path="
-            else
-                "#{u}?#{t}#{o}path=#{f}"
-            end
-        else
-            ''
-        end
-    end
-
-    def dir_entry(fname,dname,time,size,dir=false)
-        if dir then
-            f = fname.sub(/\/+$/,'').sub(/^\/+/,'')
-            s = "#{dname}"
-        elsif ! @interface.get('dir:uri').empty? then # takes precedence, in case we run under cgi control
-            s = "#{dname}"
-        else
-            s = "#{dname}"
-        end
-        # s << " " * (30 - dname.size)
-        s << " " * (@@dir_name_width + 5 - dname.size)
-        s << (time ? time.strftime("%Y/%m/%d %H:%M      ") : " " * 22)
-        s << (size >= 0 ? size.to_s : "-").rjust(12) << "\n"
-        return s
-    end
-
-end
diff --git a/scripts/context/ruby/www/exa.rb b/scripts/context/ruby/www/exa.rb
deleted file mode 100644
index 20a40fc7b..000000000
--- a/scripts/context/ruby/www/exa.rb
+++ /dev/null
@@ -1,387 +0,0 @@
-require 'fileutils'
-require 'www/lib'
-require 'www/dir'
-require 'www/common'
-require 'www/admin'
-
-class WWW
-
-    include Common
-
-    def handle_exadefault
-        check_template_file('exalogin','exalogin-template.htm')
-        if id = logged_in_session(true) then
-            finish_login
-        else
-            message('Error', 'No default login permitted.')
-        end
-    end
-
-    def handle_exalogin
-        check_template_file('exalogin','exalogin-template.htm')
-        if id = logged_in_session(false) then
-            finish_login
-        else
-            message('Error', 'No default login permitted.')
-        end
-    end
-
-    def finish_login
-        get_gui()
-        filename, path, task = @session.get('gui'), @session.checked('path','.'), @session.get('task')
-        if ! task.empty? then
-            save_session
-            handle_exatask
-        elsif filename and not filename.empty? then
-            save_session
-            fullname = filename.gsub(/\.\./,'')
-            fullname = File.join(path,filename)                                     unless FileTest.file?(fullname)
-            fullname = File.join(@interface.get('path:interfaces'), filename)       unless FileTest.file?(fullname)
-            fullname = File.join(@interface.get('path:interfaces'), path, filename) unless FileTest.file?(fullname)
-            if FileTest.file?(fullname) then
-                send_file(fullname,true)
-            else
-                message('Interface', 'Invalid interface request, no valid interface file.' )
-            end
-        else
-            message('Interface', 'Invalid interface request, no default interface file.')
-        end
-    end
-
-    def handle_exainterface()
-        check_template_file('text','exalogin-template.htm')
-        if id = valid_session() then
-            filename = @interface.get('process:uri').to_s # kind of dup
-            if ! filename.empty? && filename.sub!(/^.*\//,'') then
-                path = @session.checked('path', '.')
-                fullname = filename.gsub(/\.\./,'')
-                fullname = File.join(path,filename) unless FileTest.file?(fullname)
-                fullname = File.join(@interface.get('path:interfaces'),filename) unless FileTest.file?(fullname)
-                fullname = File.join(@interface.get('path:interfaces'),path,filename) unless FileTest.file?(fullname)
-                if FileTest.file?(fullname) then
-                    save_session
-                    send_file(fullname,true)
-                else
-                    get_file(filename)
-                    filename, path = @session.get('gui'), @session.checked('path','.')
-                    if filename and not filename.empty? then
-                        save_session
-                        fullname = filename.gsub(/\.\./,'')
-                        fullname = File.join(path,filename) unless FileTest.file?(fullname)
-                        fullname = File.join(@interface.get('path:interfaces'),filename) unless FileTest.file?(fullname)
-                        fullname = File.join(@interface.get('path:interfaces'),path,filename) unless FileTest.file?(fullname)
-                        send_file(fullname,true) if FileTest.file?(fullname)
-                    else
-                        message('Interface', 'Invalid interface request, no interface file.')
-                    end
-                end
-            else
-                message('Interface', 'Invalid interface request, no resource file.')
-            end
-        else
-            message('Interface', 'Invalid interface request, no login.')
-        end
-    end
-
-    def handle_exarequest() # todo: check if request is 'command'
-        check_template_file('exalogin','exalogin-template.htm')
-        if id = client_session() then
-            client = true
-            @interface.set('log:kind', "remote client request: #{id}")
-        elsif id = valid_session() then
-            client = false
-            @interface.set('log:kind', "remote browser request: #{id}")
-        else
-            client, id = false, nil
-            @interface.set('log:kind', 'unknown kind of request')
-        end
-        if id then
-            dir, tmp = dirname, tmp_path(dirname)
-            requestname, replyname = 'request.exa', 'reply.exa'
-            requestfile, replyfile = File.join(tmp,requestname), File.join(tmp,replyname)
-            lockfile = File.join(dirname,lckname)
-            action, filename, command, url, req = '', '', '', '', ''
-            extract_sent_files(tmp)
-            @variables.each do |key, value|
-                case key
-                    when 'exa:request' then
-                        req = value.dup
-                    when 'exa:action' then
-                        action = value.dup
-                    # when 'exa:command' then
-                        # command = value.dup
-                    # when 'exa:url' then
-                        # url = value.dup
-                    when 'exa:filename' then
-                        filename = value.dup
-                    when 'exa:threshold' then
-                        @interface.set('process:threshold', value.dup)
-                    when /^fakename/o then
-                        @variables.set(key, File.basename(value))
-                    when /^filename\-/o then
-                        @variables.set(key, filename = File.basename(value))
-                    when /^dataname\-/o then
-                        @variables.set(key)
-                    else # remove varname- prefix from value
-                        @variables.set(key, @variables.get(key).sub(/#{key}\-/,''))
-                end
-            end
-            @variables.check('exa:filename', filename)
-            @variables.check('exa:action', action)
-            if @variables.empty?('exa:filename') then
-                @variables.set('exa:filename', @interface.get('log:attachments').split('|').first || '')
-            end
-            req.gsub!(//i, '')
-            dat = "\n"
-            @variables.each do |key, value|
-                if ['password','exa:request'].include?(key) then
-                    # skip
-                elsif ! value || value.empty? then
-                    dat << "\n"
-                else # todo: escape 'm
-                    dat << "#{value}\n"
-                end
-            end
-            dat << "\n"
-            if req.empty? then
-                req << "\n"
-                req << "\n"
-                req << "\n"
-                req << "'#{action}\n"     unless action.empty?
-                # req << "'#{command}\n"  unless command.empty?
-                # req << "'#{url}\n"              unless url.empty?
-                req << "\n"
-                req << "constructed request\n"
-                req << dat
-                req << "\n"
-            else
-                # better use rexml but slower
-                if req =~ /]*>.*?\s*\s*(.*?)\s*<\/exa:threshold>\s*.*?<\/exa:request>/moi then
-                    threshold = $1
-                    unless threshold.empty? then
-                        @interface.set('process:threshold', threshold)
-                        @session.set('threshold', threshold)
-                    end
-                end
-                req.sub!(/(]*>.*?)\s*\s*\-\-action\=(.*?)\s*<\/exa:option>\s*(.*?<\/exa:request>)/moi) do
-                    pre, act, pos = $1, $2, $3
-                    action = act.sub(/\.exa$/,'') if action.empty?
-                    str = "#{pre}#{action}#{pos}"
-                    str.sub(/\s*.*?<\/exa:command>\s*/moi ,'')
-                end
-                req.sub!(/(]*>.*?)\s*(.*?)\s*<\/exa:action>(.*?<\/exa:request>)/moi) do
-                    pre, act, pos = $1, $2, $3
-                    action = act.sub(/\.exa$/,'') if action.empty?
-                    str = "#{pre}#{action}#{pos}"
-                    str.sub(/\s*.*?<\/exa:command>\s*/moi ,'')
-                end
-                unless req =~ /(.*?)<\/exa:data>/moi then
-                    req.sub!(/(<\/exa:request>)/) do dat + $1 end
-                end
-            end
-            req.sub!(/.*?<\/exa:filename>/moi, '')
-            unless @variables.empty?('exa:filename') then
-                req.sub!(/(<\/exa:application>)/moi) do
-                    "#{@variables.get('exa:filename')}<\/exa:filename>" + $1
-                end
-            end
-            @variables.set('exa:action', action)
-            @interface.set("log:#{requestname}", req)
-            begin
-                File.open(requestfile,'w') do |f|
-                    f << req
-                end
-            rescue
-                message('Error', 'There is a problem in handling this request (working path access).')
-                return
-            end
-            File.delete(replyfile) rescue false
-            @interface.set('log:action',action)
-            get_command(action)
-            logdata = ''
-            begin
-                command = @session.get('command')
-                @interface.set('log:command',if command.empty? then '[no command]' else command end)
-                if ! command.empty? then
-                    @session.set('starttime', Time.now.to_i.to_s) # can be variables and in save list
-                    if @interface.true?('process:background') then
-                        # background
-                        @session.set('status',  'running: background')
-                        @session.set('maxtime', @interface.get('process:timeout'))
-                        @session.set('threshold', @interface.get('process:threshold'))
-                        save_session
-                        timeout(@@watch_delay) do
-                            save_environment(@interface,tmp)
-                            begin
-                                starttime = File.mtime(@session_file)
-                                # crap
-                                loop do
-                                    sleep(1)
-                                    if starttime != File.mtime(@session_file) then
-                                        break unless FileTest.file?(lockfile)
-                                    end
-                                end
-                            rescue TimeoutError
-                                if client then
-                                    send_reply()
-                                else
-                                    message('Status', 'Processing your request takes a while',true,5,'exastatus')
-                                end
-                                return
-                            rescue
-                            end
-                        end
-                        if client then send_reply() else send_result() end
-                    else
-                        # foreground
-                        status = 'running: foreground'
-                        @session.set('status',  status)
-                        @session.set('maxtime', @interface.get('process:timeout'))
-                        @session.set('threshold', @interface.get('process:threshold'))
-                        save_session
-                        timeout(@interface.get('process:timeout').to_i) do
-                            begin
-                                status = 'running: foreground'
-                                set_environment(@interface)
-                                save_environment(@interface,tmp)
-                                command = command_string(tmp,command)
-                                logdata = `#{command}`
-                            rescue TimeoutError
-                                status = 'running: timeout'
-                                logdata = "timeout: #{@interface.get('process:timeout')} seconds"
-                            rescue
-                                status = 'running: aborted'
-                                logdata = 'fatal runtime error'
-                            else
-                                @session.set('endtime', Time.now.to_i.to_s)
-                                status = 'running: finished'
-                            end
-                        end
-                        @session.set('status', status)
-                        save_session
-                        case @session.get('status')
-                            when 'running: finished' then
-                                if client then send_reply(logdata) else send_result(logdata) end
-                            when 'running: timeout' then
-                                message('Error', 'There is a problem in handling this request (timeout).')
-                            when 'running: aborted' then
-                                message('Error', 'There is a problem in handling this request (aborted).')
-                            else
-                                message('Error', 'There is a problem in handling this request (unknown).')
-                        end
-                    end
-                else
-                    message('Error', 'There is a problem in handling this request (no runner).')
-                end
-            rescue
-                message('Error', 'There is a problem in handling this request (no run).' + $!)
-            end
-        else
-            message('Error', 'Invalid session.')
-        end
-    end
-
-    def handle_exacommand() # shares code with exarequest
-        check_template_file('exalogin','exalogin-template.htm')
-        if id = client_session() then
-            client = true
-            @interface.set('log:kind', "remote client request: #{id}")
-        elsif id = valid_session() then
-            client = false
-            @interface.set('log:kind', "remote browser request: #{id}")
-        else
-            client, id = false, nil
-            @interface.set('log:kind', 'unknown kind of request')
-        end
-        if id then
-            dir, tmp = dirname, tmp_path(dirname)
-            requestname, replyname = 'request.exa', 'reply.exa'
-            requestfile, replyfile = File.join(tmp,requestname), File.join(tmp,replyname)
-            req, command, url = '', '', ''
-            @variables.each do |key, value|
-                case key
-                    when 'exa:request' then
-                        req = value.dup
-                    when 'exa:command' then
-                        command = value.dup
-                    when 'exa:threshold' then
-                        @interface.set('process:threshold', value.dup)
-                    when 'exa:url' then
-                        url = value.dup
-                end
-            end
-            unless req.empty? then
-                # better use rexml but slower / reuse these : command = filter_from_request('exa:command')
-                if req =~ /]*>.*?\s*\s*(.*?)\s*<\/exa:command>\s*.*?<\/exa:request>/moi then
-                    command = $1
-                end
-                if req =~ /]*>.*?\s*\s*(.*?)\s*<\/exa:url>\s*.*?<\/exa:request>/moi then
-                    url = $1
-                end
-                if req =~ /]*>.*?\s*\s*(.*?)\s*<\/exa:threshold>\s*.*?<\/exa:request>/moi then
-                    threshold = $1
-                    unless threshold.empty? then
-                        @interface.set('process:threshold', threshold)
-                        @session.set('threshold', threshold)
-                    end
-                end
-            end
-            @variables.check('exa:command', command)
-            @variables.check('exa:url', url)
-            File.delete(replyfile) rescue false
-            case @variables.get('exa:command')
-                when 'fetch' then
-                    if @variables.empty?('exa:url') then
-                        message('Error', "Problems with fetching, no file given")
-                    else
-                        # the action starts here
-                        filename = @variables.get('exa:url').to_s # kind of dup
-                        unless filename.empty? then
-                            get_path(filename) # also registers filename as url
-                            path = @session.checked('path', '')
-                            fullname = filename.gsub(/\.\./,'')
-                            fullname = File.join(path,fullname) unless path.empty?
-                            if FileTest.file?(fullname) then
-                                if client then
-                                    send_url(fullname)
-                                else
-                                    send_file(fullname,true)
-                                end
-                                @session.set('threshold', @interface.get('process:threshold'))
-                                @session.set('url',filename)
-                                save_session
-                            else
-                                message('Error', "Problems with fetching, unknown file #{fullname}.")
-                                # message('Error', "Problems with fetching, unknown file #{filename}.")
-                            end
-                        else
-                           message('Error', "Problems with fetching, invalid file #{filename}.")
-                        end
-                        # and ends here
-                    end
-            else
-                message('Error', "Invalid command #{command}.")
-            end
-        else
-            message('Error', 'Invalid session.')
-        end
-    end
-
-    def handle_exastatus
-        get_cfg() # weird, needed for apache, but not for wwwserver
-        if request_variable('id').empty? then
-            if id = valid_session() then
-                send_result()
-            else
-                message('Error', 'Invalid session.')
-            end
-        else
-            if id = valid_session() then
-                send_reply()
-            else
-                send_reply('invalid session')
-            end
-        end
-    end
-
-end
diff --git a/scripts/context/ruby/www/lib.rb b/scripts/context/ruby/www/lib.rb
deleted file mode 100644
index b9a44c9f6..000000000
--- a/scripts/context/ruby/www/lib.rb
+++ /dev/null
@@ -1,1405 +0,0 @@
-#!/usr/bin/env ruby
-
-# This is just a simple environment for remote processing of context
-# files. It's not a framework, nor an example of how that should be done.
-# Nowadays there are environments like Rails or Nitro. Maybe some day I'll
-# give one of them a try.
-
-# 
-# 
-# 
-
-# we make limited use of cgi methods because we also need to handle webrick
-
-# %var% as well as $(var) are supported
-
-# paths need to be expanded before they enter apache, since .. is not
-# handled by default
-
-require 'base/variables'
-
-require 'ftools'
-require 'fileutils'
-require 'tempfile'
-require 'timeout'
-require 'md5'
-require 'digest/md5'
-require 'cgi' # we also need escaping for webrick (could move it here)
-
-# beware, namespaces have to match !
-
-module XML
-
-    def XML::element(tag,attributes=nil)
-        if attributes.class == Hash then
-            if block_given? then
-                XML::element(tag,XML::attributes(attributes)) do yield end
-            else
-                XML::element(tag,XML::attributes(attributes))
-            end
-        else
-            if block_given? then
-                "<#{tag}#{if attributes && ! attributes.empty? then ' ' + attributes end}>#{yield}"
-            else
-                "<#{tag}#{if attributes && ! attributes.empty? then ' ' + attributes end}/>"
-            end
-        end
-    end
-
-    def XML::attributes(hash)
-        str = ''
-        hash.each do |k,v|
-            str << ' ' unless str.empty?
-            if v =~ /\'/ then
-                str << "#{k}=\"#{v}\""
-            else
-                str << "#{k}=\'#{v}\'"
-            end
-        end
-        return str
-    end
-
-    def XML::create(version='1.0')
-        "#{yield || ''}"
-    end
-
-    def XML::line
-        "\n"
-    end
-
-end
-
-# str =
-    # XML::create do
-        # XML::element('test') do
-            # XML::element('test') do
-                # 'text a'
-            # end +
-            # XML::element('test',XML::attributes({'a'=>'b'})) do
-                # XML::element('nested',XML::attributes({'a'=>'b'})) do
-                    # 'text b-1'
-                # end +
-                # XML::element('nested',XML::attributes({'a'=>'b'})) do
-                    # 'text b-2'
-                # end
-            # end +
-            # XML::element('nested',{'a'=>'b'}) do
-                # 'text c'
-            # end
-       # end
-    # end
-
-class ExtendedHash
-
-    DEFAULT = 'default'
-
-    @@re_default = /^(default|)$/i
-
-    def default?(key)
-        self[key] =~ @@re_default rescue true # unset, empty or 'default'
-    end
-
-    def default(key)
-        self[key] = DEFAULT
-    end
-
-    def match?(key,value)
-        value == '*' || value == self[key]
-    end
-
-
-end
-
-class WWW
-
-    @@session_prefix  = ''
-    @@data_file       = 'example.cfg'
-    @@session_max_age = 60*60
-    @@watch_delay     = 30
-    @@send_threshold  = 2*1024*1024
-    @@admin_refresh   = 10
-    @@namespace       = "http://www.pragma-ade.com/schemas/example.rng"
-
-    @@re_bar   = /\s*\|\s*/
-    @@re_lst   = /\s*\,\s*/
-    @@re_var_a = /\%(.*?)\%/
-    @@re_var_b = /\$\((.*?)\)/
-
-    attr_reader :variables
-    attr_writer :variables
-
-    @@paths = [
-        'configurations',
-        'data',
-        'distributions',
-        'documents',
-        'interfaces',
-        'logs',
-        'resources',
-        'runners',
-        'scripts',
-        'templates',
-        'work']
-
-    @@re_true  = /^\s*(YES|ON|TRUE|1)\s*$/io
-    @@re_false = /^\s*(NO|OFF|FALSE|0)\s*$/io
-
-    def initialize(webrick_daemon=nil,webrick_request=nil,webrick_response=nil)
-        @session_id, @session_file = '', ''
-        @cgi, @cgi_cookie = nil, nil
-        @webrick_daemon, @webrick_request, @webrick_response = webrick_daemon, webrick_request, webrick_response
-
-        @interface = ExtendedHash.new
-        @variables = ExtendedHash.new
-        @session   = ExtendedHash.new
-
-        @checked = false
-
-        analyze_request()
-        update_interface()
-
-        @interface.set('template:message'  , 'exalogin-template.htm')
-        @interface.set('template:status'   , 'exalogin-template.htm')
-        @interface.set('template:login'    , 'exalogin.htm')
-        @interface.set('process:timeout'   ,  @@session_max_age)
-        @interface.set('process:threshold' ,  @@send_threshold)
-        @interface.set('process:background', 'yes') # this demands a watchdog being active
-        @interface.set('process:indirect'  , 'no')  # indirect download, no direct feed
-        @interface.set('process:autologin' , 'yes') # provide default interface when applicable
-        @interface.set('process:exaurl'    , '')    # this one will be used as replacement in templates
-        @interface.set('trace:run'         , 'no')
-        @interface.set('trace:errors'      , 'no')
-        @interface.set('process:os'        , platform)
-        @interface.set('process:texos'     , 'texmf-' + platform)
-
-        @interface.set('trace:run'         , 'yes') if (ENV['EXA_TRACE_RUN']    || '') =~ @@re_true
-        @interface.set('trace:errors'      , 'yes') if (ENV['EXA_TRACE_ERRORS'] || '') =~ @@re_true
-
-        yield self if block_given?
-    end
-
-    def set(key,value)
-        @interface.set(key,value)
-    end
-    def get(key)
-        @interface.get(key,value)
-    end
-
-    def platform
-        case RUBY_PLATFORM
-            when /(mswin|bccwin|mingw|cygwin)/i then 'mswin'
-            when /(linux)/i                     then 'linux'
-            when /(netbsd|unix)/i               then 'unix'
-            when /(darwin|rhapsody|nextstep)/i  then 'macosx'
-            else                                     'unix'
-        end
-    end
-
-    def check_cgi
-        # when mod_ruby is used, we need to close
-        # the cgi session explicitly
-        unless @webrick_request then
-            unless @cgi then
-                @cgi = CGI.new('html4')
-                at_exit do
-                    begin
-                        @cgi.close
-                    rescue
-                    end
-                end
-            end
-        end
-    end
-
-    def request_variable(key)
-        begin
-            if @webrick_request then
-                [@webrick_request.query[key]].flatten.first.to_s
-            else
-                check_cgi
-                [@cgi.params[key]].flatten.first.to_s
-            end
-        rescue
-            ''
-        end
-    end
-
-    def request_cookie(key)
-        begin
-            if @cgi then
-                if str = @cgi.cookies[key] then
-                    return str.first || ''
-                end
-            elsif @webrick_request then
-                @webrick_request.cookies.flatten.each do |cookie|
-                    if cookie.name == key then
-                        return cookie.value unless cookie.value.empty?
-                    end
-                end
-            end
-        rescue
-        end
-        return ''
-    end
-
-    def analyze_request
-        if @webrick_request then
-            @interface.set('path:docroot', @webrick_daemon.config[:DocumentRoot] || './documents')
-            @interface.set('process:uri', @webrick_request.request_uri.to_s)
-            # @interface.set('process.url', [@webrick_request.host,@webrick_request.request_port].join(':'))
-            @cgi = nil
-            @webrick_request.query.each do |key, value|
-                # todo: filename
-                @variables.set(key, [value].flatten.first)
-            end
-        else
-            @interface.set('path:docroot', ENV['DOCUMENT_ROOT'] || './documents')
-            @interface.set('process:uri', ENV['REQUEST_URI'] || '')
-            # @interface.set('process.url', [ENV['SERVER_NAME'],ENV['SERVER:PORT']].join(':'))
-            ARGV[0] = '' # get rid of terminal mode
-            check_cgi
-            # quite fragile, due to changes between 1.6 and 1.8
-            @cgi.params.keys.each do |p|
-                if @cgi[p].respond_to?(:original_filename) then
-                    @interface.set('log:method','post')
-                    if @cgi[p].original_filename && ! @cgi[p].original_filename.empty? then
-                        @variables.set(p, File.basename(@cgi[p].original_filename))
-                    else
-                        case @cgi.params[p].class
-                            when StringIO.class then @variables.set(p, @cgi[p].read)
-                            when Array.class    then @variables.set(p, @cgi[p].first.to_s)
-                            when String.class   then @variables.set(p, @cgi[p])
-                            when Tempfile.class then @variables.set(p, '[data blob]')
-                        end
-                    end
-                else
-                    @interface.set('log:method','get') unless @interface.get('log:method') == 'post'
-                    @variables.set(p, [@cgi.params[p]].flatten.first.to_s)
-                end
-            end
-        end
-        @interface.set('path:root', File.dirname(@interface.get('path:docroot')))
-    end
-
-    # name in calling script takes precedence
-    # one can set template:whatever as well
-    # todo: in config
-
-    def check_template_file(tag='',filename='exalogin-template.htm')
-        @interface.set('file:template', filename) if @interface.get('file:template').empty?
-        @interface.set('tag:template', tag)
-        @interface.set('file:template', @interface.get('tag:template')) unless @interface.get('tag:template').empty?
-    end
-
-    def update_interface()
-        root = @interface.get('path:docroot')
-        @interface.set('path:docroot', File.expand_path("#{root}"))
-        @@paths.each do |path|
-            @interface.set("path:#{path}", File.expand_path("#{root}/../#{path}"))
-        end
-        @interface.set('file:template', @interface.get('tag:template')) unless @interface.get('tag:template').empty?
-    end
-
-    def indirect?(result)
-        size = FileTest.size?(result) || 0
-        @interface.true?('trace:errors') || @interface.true?('trace:run') || @interface.true?('process:indirect') ||
-            ((! @interface.empty?('process:threshold')) && (size > @interface.get('process:threshold').to_i)) ||
-            ((! @session.empty?('threshold'))           && (size > @session.get('threshold').to_i))
-    end
-
-end
-
-# files
-
-class WWW
-
-    def sesname
-        File.basename(@session_file)
-    end
-    def dirname
-        File.basename(@session_file.sub(/ses$/,'dir'))
-    end
-    def lckname
-        File.basename(@session_file.sub(/ses$/,'lck'))
-    end
-
-    def work_root(expand=true)
-        p = if expand then File.expand_path(@interface.get('path:work')) else @interface.get('path:work') end
-        if @interface.true?('process:background') then
-            File.join(@interface.get('path:work'),'watch')
-        else
-            File.join(@interface.get('path:work'),'direct')
-        end
-    end
-
-    def work_roots(expand=true)
-        p = if expand then File.expand_path(@interface.get('path:work')) else @interface.get('path:work') end
-        [File.join(@interface.get('path:work'),'watch'),File.join(@interface.get('path:work'),'direct')]
-    end
-
-    def cache_root(expand=true)
-        p = if expand then File.expand_path(@interface.get('path:work')) else @interface.get('path:work') end
-        File.join(@interface.get('path:work'),'cache')
-    end
-
-    def cleanup_path(dir)
-        FileUtils::rm_r(pth) rescue false
-    end
-
-    def tmp_path(dir)
-        @interface.set('path:templates', File.expand_path(@interface.get('path:templates'))) # to be sure; here ? ? ?
-        pth = File.join(work_root,dir)
-        File.makedirs(pth) rescue false
-        pth
-    end
-
-    def locked?(lck)
-        FileTest.file?(lck)
-    end
-
-end
-
-# sessions
-
-class WWW
-
-    @@session_tags  = ['id','domain','project','username','password','gui','path','process','command','filename','action','status', 'starttime','endtime','runtime','task','option','threshold','url'].sort
-    @@session_keep  = ['id','domain','project','username','password','process'].sort
-    @@session_reset = @@session_tags - @@session_keep
-
-    def new_session()
-        if @variables.empty?('exa:session') then
-            @session_id = new_session_id
-        else
-            @session_id = @variables.get('exa:session')
-        end
-        if @session_id == 'default' then # ???
-            @session_id = new_session_id
-        end
-        @session_file = File.join(work_root,"#{@@session_prefix}#{@session_id}.ses")
-        register_session
-        return @session_id
-    end
-
-    def reset_session(all=false)
-        (if all then @@session_tags else @@session_reset end).each do |k|
-            @session.set(k)
-        end
-    end
-
-    def valid_session
-        @session_id = request_variable('id')
-        if @session_id.empty? then
-            begin
-                if @cgi then
-                    if @session_id = @cgi.cookies['session_id'] then
-                        @session_id = @session_id.first || ''
-                    else
-                        @session_id = ''
-                    end
-                elsif @webrick_request then
-                    @webrick_request.cookies.flatten.each do |cookie|
-                        if cookie.name == 'session_id' then
-                            unless cookie.value.empty? then
-                                @session_id = cookie.value
-                                # break
-                            end
-                        end
-                    end
-                else
-                    @session_id = ''
-                end
-            rescue
-                @interface.set('log:session',"[error in request #{$!}]")
-                return false
-            end
-        end
-        if @session_id.empty? then
-            @interface.set('log:session','[no id, check work dir permissions]')
-            return false
-        else
-            @interface.set('log:session',@session_id)
-            load_session
-            if ! @session.empty?('domain') && ! @session.empty?('project') && ! @session.empty?('username') then
-                register_session
-                return @session_id
-            else
-                return false
-            end
-        end
-    end
-
-    def touch_session(id=nil)
-        begin
-            t = Time.now
-            File.utime(t,t,File.join(work_root,"#{@@session_prefix}#{id || @session_id}.ses")) rescue false
-        rescue
-            false
-        end
-    end
-
-    def forced_session
-        @session_id = new_session
-        if @session_id.empty? then
-            @interface.set('log:session','[no id, check work dir permissions]')
-            return false
-        else
-            return check_session
-        end
-    end
-
-    def client_session
-        request, done = @variables.get('exa:request'), false
-        request.sub!(/(^.*]*>.*?)\s*\s*(.*)\s*<\/exa:client>\s*(.*?<\/exa:request>.*$)/mio) do
-            pre, client, post = $1, $2, $3
-            client.scan(/(.*?)<\/exa:\1>/mio) do
-                @variables.set($1, $2)
-            end
-            done = true
-            pre + post
-        end
-        if done then
-            return forced_session
-        else
-            return nil
-        end
-    end
-
-    def register_session
-        if @cgi then
-            @cgi_cookie = CGI::Cookie::new(
-                'name'    => 'session_id',
-                'value'   => @session_id,
-                'expires' => Time.now + @interface.get('process:timeout').to_i
-            )
-            # @cgi_cookie = CGI::Cookie::new('session_id',@session_id)
-        elsif @webrick_response then
-            if cookie = WEBrick::Cookie.new('session_id', @session_id) then
-                cookie.expires = Time.now + @interface.get('process:timeout').to_i
-                cookie.max_age = @interface.get('process:timeout').to_i
-                cookie.comment = 'exa identifier'
-                @webrick_response.cookies.clear
-                @webrick_response.cookies << cookie
-            end
-        end
-    end
-
-    def new_session_id # taken from cgi
-        md5 = Digest::MD5::new
-        now = Time::now
-        md5.update(now.to_s)
-        md5.update(String(now.usec))
-        md5.update(String(rand(0)))
-        md5.update(String($$))
-        md5.update('foobar')
-        @new_session = true
-        md5.hexdigest[0,32] # was 16
-    end
-
-    @@hide_passwords = true
-    HIDDEN = 'hidden'
-
-    def same_passwords(password) # password in cfg file
-        if @@hide_passwords && (@session.get('password') == HIDDEN) && (@session_id == @session.get('id')) then
-            # this condition is only true when a same session id is found and
-            # the password is checked once and set to HIDDEN
-            same = true
-        elsif password =~ /^MD5:/ then
-            # so, one cannot send a known encrypted password since it will be
-            # encrypted twice then
-            same = (password == "MD5:" + MD5.new(@session.get('password')).hexdigest.upcase)
-        else
-            if (@session.default?('domain') && @session.default?('project') && @session.default?('username')) then
-                @session.default('password') # is this safe enough?
-            end
-            same = (password == @session.get('password'))
-        end
-        if @@hide_passwords && same then
-            @session.set('password', HIDDEN)
-            save_session # next time this session is ok anyway
-        end
-        return same
-    end
-
-    @@session_line  = /^\s*(?![\#\%])(.*?)\s*\=\s*(.*?)\s*$/o
-    @@session_begin = 'begin exa session'
-    @@session_end   = 'end exa session'
-
-    def loaded_session_data(filename)
-        begin
-            if data = IO.readlines(filename) then
-                return data if (data.first =~ /^[\#\%]\s*#{@@session_begin}/o) && (data.last =~ /^[\#\%]\s*#{@@session_end}/o)
-            end
-        rescue
-        end
-        return nil
-    end
-
-    def load_session()
-        begin
-            @session_file = File.join(work_root,"#{@@session_prefix}#{@session_id}.ses")
-            if data = loaded_session_data(@session_file) then
-                data.each do |line|
-                    if line =~ @@session_line then
-                        @session.set($1, $2 || '')
-                    end
-                end
-            else
-                return false
-            end
-        rescue
-            return false
-        else
-            return true
-        end
-    end
-
-    def load_session_file(filename)
-        begin
-            if data = loaded_session_data(filename) then
-                session = Hash.new
-                data.each do |line|
-                    if line =~ @@session_line then
-                        session[$1] = $2 || ''
-                    end
-                end
-            else
-                Hash.new
-            end
-        rescue
-            Hash.new
-        else
-            session
-        end
-    end
-
-    def save_session
-        begin
-            unless @session_id.empty? then
-                @session_file = File.join(work_root,"#{@@session_prefix}#{@session_id}.ses")
-                @session_file = File.join(work_root,"#{@@session_prefix}#{@session_id}.ses")
-                File.open(@session_file,'w') do |f|
-                    f << "\# #{@@session_begin}\n"
-                    @@session_tags.each do |tag|
-                        if @session && @session.key?(tag) then
-                            if ! @session.get(tag).empty? then # no one liner, fails
-                                f << "#{tag}=#{@session.get(tag)}\n"
-                            end
-                        elsif @variables.key?(tag) && ! @variables.empty?(key) then
-                            f << "#{tag}=#{@variables.get(tag)}\n"
-                        end
-                    end
-                    @session.subset("ENV").keys.each do |tag|
-                        f << "#{tag}=#{@session.get(tag)}\n"
-                    end
-                    f << "\# #{@@session_end}\n"
-                end
-            end
-        rescue
-            return false
-        else
-            return true
-        end
-    end
-
-    def logged_in_session(force_default=false)
-        if force_default || (@variables.default?('domain') && @variables.default?('project') && @variables.default?('username')) then
-            id = default_session
-        else
-            id = check_session
-        end
-    end
-
-    def default_session
-        if @interface.true?('process:autologin') then
-            @variables.default('domain')
-            @variables.default('project')
-            @variables.default('username')
-            @variables.default('password')
-            check_session
-        else
-            @session_id = nil
-        end
-    end
-
-    def check_session
-        @session.set('domain',   @variables.get('domain').downcase)
-        @session.set('project',  @variables.get('project').downcase)
-        @session.set('username', @variables.get('username').downcase)
-        @session.set('password', @variables.get('password').downcase)
-        new_session
-        @session.set('id', @session_id)
-        save_session
-        return @session_id
-    end
-
-    def delete_session(id=nil)
-        File.delete(work_root,"#{@@session_prefix}#{id || @session_id}.ses") rescue false
-    end
-
-    def cleanup_sessions(max_age=nil)
-        begin
-            now, age = Time.now, (max_age||@interface.get('process:timeout')).to_i
-            Dir.glob("{#{work_root},#{cache_root}/#{@@session_prefix}*").each do |s|
-                begin
-                    if (now - File.mtime(s)) > age then
-                        if FileTest.directory?(s) then
-                            FileUtils::rm_r(s)
-                        else
-                            File.delete(s)
-                        end
-                    end
-                rescue
-                    # maybe purged in the meantime
-                end
-            end
-        rescue
-            # maybe another process is busy
-        end
-    end
-
-end
-
-# templates
-
-class WWW
-
-    def filled_template(title,text,showtime=false,refresh=0,refreshurl=nil)
-        template = @interface.get("template:#{@interface.get('tag:template')}")
-        template = @interface.get("template:status") if template.empty?
-        fullname = File.join(@interface.get('path:templates'),template)
-        @interface.set('log:templatename',template)
-        @interface.set('log:templatefile',fullname)
-        append_status(text)
-        htmreply = ''
-        if FileTest.file?(fullname) then
-            begin
-                htmreply = IO.read(fullname)
-            rescue
-                htmreply = ''
-            end
-        end
-        if refresh>0 then
-            if refreshurl then
-                metadata = ""
-            else
-                metadata = ""
-            end
-        else
-            metadata = ''
-        end
-        if ! htmreply || htmreply.empty? then
-            # in head: 
-            htmreply = <<-EOD
-                
-                    #{metadata}
-                    
-                        #{title}
-                    
-                    
-                        
#{title}

-                        
#{Time.now}

-                        #{text}
-                    
-                
-            EOD
-        else
-            if showtime then
-                exa_template = "
#{title}
\n
#{Time.now}
\n#{text}\n"
-            else
-                exa_template = "
#{title}
#{text}\n"
-            end
-            htmreply = replace_template_placeholder(htmreply,exa_template,metadata)
-        end
-        htmreply
-    end
-
-    def message(title,str='',showtime=false,refresh=0,refreshurl=nil)
-        if @cgi then
-            @cgi.out("cookie"=>[@cgi_cookie]) do
-                filled_template(title,str,showtime,refresh,refreshurl)
-            end
-        elsif @webrick_response then
-            @webrick_response['content-type'] = 'text/html'
-            @webrick_response.body = filled_template(title,str,showtime,refresh,refreshurl)
-        else
-            filled_template(title,str,showtime,refresh,refreshurl)
-        end
-    end
-
-    def plaintext(str)
-        if @cgi then
-            @cgi.out('cookie'=>[@cgi_cookie],'content-type'=>'text/plain') do
-                str
-            end
-        elsif @webrick_response then
-            @webrick_response['content-type'] = 'text/plain'
-            @webrick_response.body = str
-        else
-            str
-        end
-    end
-
-    def exareply(status='',url='',size='',comment='')
-        exaurl = @interface.get('process:exaurl')
-        str  = "\n\n"
-        str << "\n"
-        str << "  #{@session_id}\n" unless @session_id.empty?
-        str << "  #{status}\n"        unless (status || '').empty?
-        str << "  #{exaurl}/#{url}\n"       unless (url    || '').empty?
-        str << "  #{size}\n"              unless (size   || '').empty?
-        str << "  #{comment}\n"     unless (comment|| '').empty?
-        str << "\n"
-        return str
-    end
-
-    def append_status(str='')
-        if @interface.true?('trace:errors') then
-            if $! && $@ then
-                str << "


Error:
#{$!}
"
-                str << $@.join("\n")
-            end
-            str << '


'
-            str << status_data
-            str << 'Paths
'
-            str << '
'
-            @interface.subset('path:').each do |k,v|
-                if FileTest.directory?(v) then
-                    if FileTest.writable?(v) then
-                        str << "#{v} exists and is writable\n"
-                    else
-                        str << "#{v} is not writable\n"
-                    end
-                else
-                    str << "#{v} does not exist\n"
-                end
-            end
-            str << '
'
-        end
-        str
-    end
-
-    def simpleurl(url)
-        if url then url.sub(/(:80|:443)$/,'') else '' end
-    end
-
-    def replace_exa_placeholders(data)
-        data.gsub(/([\"\'])\@exa\_([a-zA-Z0-9\-\_]+)\1/) do
-            quot, key, value = $1, $2, ''
-            begin
-                value = @variables.get(key)
-            rescue
-                value = ''
-            end
-            quot + value + quot
-        end
-    end
-
-    def replace_url_placeholder(data)
-        data.gsub!(/(http:\/\/|\/+)*\@exa\_main\_url/, @interface.get('process:exaurl'))
-        replace_exa_placeholders(data)
-    end
-
-    def replace_template_placeholder(data,template='',metadata='')
-        data.gsub!(/(http:\/\/|\/+)*\@exa\_main\_url/, @interface.get('process:exaurl'))
-        data.gsub!(/\@exa\_template/, template)
-        data.gsub!(/\@exa\_metadata/, metadata)
-        replace_exa_placeholders(data)
-    end
-
-    def escaped(str)
-        str
-    end
-
-end
-
-# send files
-
-class WWW
-
-    def send_file(filename,parse=false) # this can take a lot of memory, look for alternative (fastcgi ?)
-        begin
-            if filename =~ /\.pdf$/ then
-                mimetype, parse = 'application/pdf', false
-            elsif filename =~ /\.(html|htm)$/ then
-                mimetype, parse = 'text/html', true
-            else
-                mimetype, parse = 'text/plain', false
-            end
-            if FileTest.file?(filename) then
-                if @webrick_response then
-                    begin
-                        @webrick_response['content-type'] = mimetype
-                        @webrick_response['content-length'] = FileTest.size?(filename)
-                        if parse then
-                            File.open(filename, 'rb') do |f|
-                                @webrick_response.body = replace_url_placeholder(f.read)
-                            end
-                        else
-                            @webrick_response.body = File.open(filename, 'rb')
-                        end
-                    rescue
-                    else
-                        return
-                    end
-                elsif @cgi then
-                    begin
-                        # the following works ok, but stores the whole file in memory (see @cgi.out)
-                        #
-                        # File.open(filename, 'rb') do |f|
-                            # @cgi.out('cookie'=>[@cgi_cookie],'connection'=>'close', 'length'=>File.size(filename), 'type'=>mimetype) do
-                                # if parse then replace_url_placeholder(f.read) else f.read end
-                            # end
-                        # end
-                        if parse then
-                            File.open(filename, 'rb') do |f|
-                                @cgi.out('cookie'=>[@cgi_cookie],'connection'=>'close', 'length'=>File.size(filename), 'type'=>mimetype) do
-                                    replace_url_placeholder(f.read)
-                                end
-                            end
-                        else
-                            @cgi.print(@cgi.header('cookie'=>[@cgi_cookie],'connection'=>'close', 'length'=>File.size(filename), 'type'=>mimetype))
-                            File.open(filename, 'rb') do |f|
-                                while str = f.gets do
-                                    @cgi.print(str)
-                                end
-                            end
-                        end
-                    rescue
-                    else
-                        return
-                    end
-                end
-            end
-        rescue
-        end
-        message('Error', "There is a problem with sending file #{File.basename(filename)}.")
-    end
-
-    def send_htmlfile(filename,parse=false)
-        send_file(filename,parse)
-    end
-    def send_pdffile(filename) # this can take a lot of memory, look for alternative (fastcgi ?)
-        send_file(filename,false)
-    end
-
-end
-
-# tracing
-
-class WWW
-
-    def show_vars(a=@variables,title='')
-        if a && a.length > 0 then
-            if title.empty? then
-                str = ''
-            else
-                str = "#{title}"
-            end
-            str << "
\n"
-            a.keys.sort.each do |k|
-                if k && a[k] && ! a[k].empty? then
-                    if k == 'password' then
-                        val = if a[k] == 'default' then 'default' else '******' end
-                    else
-                        # str << "#{k} => #{a[k].sub(/^\s+/moi,'').sub(/\s+$/moi,'')}\n"
-                        val = a[k].to_s.strip
-                        val.gsub!("&","&")
-                        val.gsub!("<","<")
-                        val.gsub!(">",">")
-                        val.gsub!("\n","\n    ")
-                    end
-                    str << "#{k} => #{val}\n"
-                end
-            end
-            str << "

\n"
-            return str
-        else
-            return ''
-        end
-    end
-
-    def status_data
-        show_vars(@session  , 'Session'    ) +
-        show_vars(@variables, 'Variables'  ) +
-        show_vars(@interface, 'Interface'  ) +
-        show_vars(ENV       , 'Environment')
-    end
-
-    def report_status
-        check_template_file('status')
-        message('Status',status_data)
-    end
-
-end
-
-# attachments
-
-class WWW
-
-    def extract_sent_files(dir)
-        files = Array.new
-        if @cgi then
-            @cgi.params.keys.each do |tag|
-                begin
-                    if filename = @cgi[tag].original_filename then
-                        files << extract_file_content(dir,filename,@cgi[tag]) unless filename.empty?
-                    end
-                rescue
-                end
-            end
-        elsif @webrick_request then
-            @webrick_request.query.keys.each do |tag|
-                begin
-                    if filename = @webrick_request.query[tag].filename then
-                        files << extract_file_content(dir,filename,@webrick_request.query[tag]) unless filename.empty?
-                    end
-                rescue
-                end
-            end
-        end
-        @interface.set('log:attachments', files.compact.uniq.join('|'))
-    end
-
-    def extract_file_content(dir,filename,data)
-        filename = File.join(dir,File.basename(filename))
-        begin
-            @interface.set('log:attachclass', data.class.inspect)
-            if data.class == Tempfile then
-                begin
-                    File.copy(data.path,filename)
-                rescue
-                    begin
-                        File.open(filename,'wb') do |f|
-                            File.open(data.path,'rb') do |g|
-                                while str = g.gets do
-                                    f.write(str)
-                                end
-                            end
-                        end
-                    rescue
-                        @interface.set('log:attachstate', "saving tempfile #{filename} failed (#{$!})")
-                    else
-                        @interface.set('log:attachstate', "tempfile #{filename} has been saved")
-                    end
-                else
-                    @interface.set('log:attachstate', "#{data.path} copied to #{filename}")
-                end
-            elsif data.class == String then
-                begin
-                    File.open(filename,'wb') do |f|
-                        f.write(data)
-                    end
-                rescue
-                    @interface.set('log:attachstate', "saving string #{filename} failed (#{$!})")
-                else
-                    @interface.set('log:attachstate', "string #{filename} has been saved")
-                end
-            elsif data.class == StringIO then
-                begin
-                    File.open(filename,'wb') do |f|
-                        f.write(data.read)
-                    end
-                rescue
-                    @interface.set('log:attachstate', "saving stringio #{filename} failed (#{$!})")
-                else
-                    @interface.set('log:attachstate', "stringio #{filename} has been saved")
-                end
-            else
-                @interface.set('log:attachstate', "unknown attachment class #{data.class.to_s}")
-            end
-        rescue
-            begin File.delete(filename) ; rescue ; end
-        else
-            begin File.delete(filename) if FileTest.size(filename) == 0 ; rescue ; end
-        end
-        return File.basename(filename)
-    end
-
-end
-
-# configuration
-
-class WWW
-
-    def interface_base_name(str)
-        str.sub(/\.(pdf|htm|html)$/, '')
-    end
-
-    def located_interface_file(filename)
-        ['configurations', 'runners', 'scripts'].each do |tag|
-            datafile = File.join(@interface.get("path:#{tag}"),filename)
-            if FileTest.file?(datafile+'.encrypted') then
-                return datafile + '.encrypted'
-            elsif FileTest.file?(datafile) then
-                return datafile
-            end
-        end
-        return nil
-    end
-
-    def load_interface_file(filename=@@data_file)
-        reset_session() # no save yet
-        if datafile = located_interface_file(filename) then
-            nestedfiles = Array.new
-            begin
-                data = IO.read(datafile) || ''
-                unless data.empty? then
-                    loop do # we need to load them recursively
-                        done = false
-                        data.gsub!(/^include\s*:\s*(.*?)\s*$/) do
-                            includedname, done = $1, true
-                            if nestedname = located_interface_file(includedname) then
-                                begin
-                                    str = ("\n" + IO.read(nestedname) + "\n") || ''
-                                rescue
-                                    nestedfiles << File.basename('-'+includedname)
-                                    ''
-                                else
-                                    nestedfiles << File.basename('+'+includedname)
-                                    str
-                                end
-                            else
-                                nestedfiles << File.basename('-'+includedname)
-                                ''
-                            end
-                        end
-                        break unless done
-                    end
-                end
-                @interface.set('log:configurationfile', datafile + ' [' + nestedfiles.join(' ') + ']')
-                return data
-            rescue
-            end
-        end
-        @interface.set('log:configurationfile', filename + ' [not loaded]')
-        return nil
-    end
-
-    def fetch_session_interface_variables(data)
-        data.scan(/^variable\s*:\s*(.*?)\s*\=\s*(.*?)\s*$/) do
-            @interface.set($1, $2)
-        end
-        return true
-    end
-
-    def fetch_session_project_list(data)
-        projectlist, permitted = Array.new, false
-        data.scan(/^user\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*\,\s*(.*?)\s*$/) do
-            domain, username, password, projects = $1, $2, $3, $4
-            if @session.match?('domain',domain) && @session.match?('username',username) then
-                if same_passwords(password) then
-                    projectlist, permitted = @interface.resolved(projects).split(@@re_bar), true
-                    break
-                end
-            end
-        end
-        if permitted then
-            @interface.set('log:projectlist', '['+projectlist.join(' ')+']')
-            if projectlist.length == 0 then
-                return nil
-            else
-                return projectlist
-            end
-        else
-            @interface.set('log:projectlist', '[no projects]')
-            return nil
-        end
-    end
-
-    def fetch_session_command(data)
-        data.scan(/^process\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*$/) do
-            domain, process, command = $1, $2, $3
-            if @session.match?('domain',domain) && @session.match?('process',process) then
-                @session.set('command', @interface.resolved(command))
-            end
-        end
-        return @session.get('command')
-    end
-
-    def fetch_session_settings(data)
-        data.scan(/^setting\s*:\s*(.*?)\s*\,\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*$/) do
-            domain, process, variable, value = $1, $2, $3, $4
-            if @session.match?('domain',domain) && @session.match?('process',process) then
-                @interface.set(variable,value)
-            end
-        end
-    end
-
-    def get_command(action)
-        # @session.set('action', action)
-        # if @session.get('process') == 'none' then
-            # @interface.set('log:child','yes')
-            # @session.set('process', action)
-        # end
-        if data = load_interface_file() then
-            fetch_session_interface_variables(data)
-            if projectlist = fetch_session_project_list(data) then
-                data.scan(/^project\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*,\s*(.*?)\s*,\s*(.*?)\s*$/) do
-                    domain, project, gui, path, process = $1, $2, $3, $4, $5
-                    if @session.match?('domain',domain) then
-                        if @session.match?('project',project) then
-                            if projectlist.include?(project) then
-                                @session.set('process', @interface.resolved(process))
-                                # break # no, else we end up in the parent (e.g. examplap instead of impose)
-                            end
-                        elsif ! action.empty? && project == action then
-                            if projectlist.include?(action) then
-                                @session.set('process', @interface.resolved(process))
-                                # break # no, else we end up in the parent (e.g. examplap instead of impose)
-                            end
-                        end
-                    end
-                end
-                fetch_session_command(data)
-                fetch_session_settings(data)
-            end
-        end
-        return ! @session.nothing?('command')
-    end
-
-    def get_file(filename)
-        @session.set('filename', filename)
-        if data = load_interface_file() then
-            fetch_session_interface_variables(data)
-            if projectlist = fetch_session_project_list(data) then
-                data.scan(/^project\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*,\s*(.*?)\s*,\s*(.*?)\s*$/) do
-                    domain, project, gui, path, process = $1, $2, $3, $4, $5
-                    if @session.match?('domain',domain) then
-                        guilist = @interface.resolved(gui).split(@@re_bar)
-                        guilist.each do |g|
-                            if /#{filename}$/ =~ g then
-                                @session.set('gui',     File.expand_path(@interface.resolved(g)))
-                                @session.set('path',    File.expand_path(@interface.resolved(path)))
-                                @session.set('process', process)
-                                break # take first matching interface
-                            end
-                        end
-                    end
-                end
-            end
-        end
-        return ! (@session.nothing?('gui') && @session.nothing?('path') && @session.nothing?('process'))
-    end
-
-    def get_path(url='')
-        if data = load_interface_file() then
-            fetch_session_interface_variables(data)
-            if projectlist = fetch_session_project_list(data) then
-                data.scan(/^project\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*,\s*(.*?)\s*,\s*(.*?)\s*$/) do
-                    domain, project, gui, path, process = $1, $2, $3, $4, $5
-                    if @session.match?('domain',domain) && @session.match?('project',project) then
-                        @session.set('url',     url)
-                        @session.set('gui',     '')
-                        @session.set('path',    File.expand_path(@interface.resolved(path)))
-                        @session.set('process', '')
-                    end
-                end
-            end
-        end
-        return ! @session.nothing?('path')
-    end
-
-    def get_gui()
-        if data = load_interface_file() then
-            fetch_session_interface_variables(data)
-            if projectlist = fetch_session_project_list(data) then
-                data.scan(/^project\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*,\s*(.*?)\s*,\s*(.*?)\s*$/) do
-                    domain, project, gui, path, process = $1, $2, $3, $4, $5
-                    if @session.match?('domain',domain) && @session.match?('project',project) && projectlist.include?(project) then
-                        @session.set('gui',     File.expand_path(@interface.resolved(gui)))
-                        @session.set('path',    File.expand_path(@interface.resolved(path)))
-                        @session.set('process', process) unless process == 'none'
-                        break # take first matching interface
-                    end
-                end
-                data.scan(/^admin\s*:\s*(.*?)\s*\,\s*(.*?)\s*\=\s*(.*?)\s*\,\s*(.*?)\s*$/) do
-                    domain, project, task, option = $1, $2, $3, $4
-                    if @session.match?('domain',domain) && @session.match?('project',project) && projectlist.include?(project) then
-                        @session.set('task', task)
-                        @session.set('option', option)
-                        break # take first matching task
-                    end
-                end
-            end
-        end
-        return ! (@session.nothing?('gui') && @session.nothing?('path') && @session.nothing?('process'))
-    end
-
-    def get_cfg()
-        if data = load_interface_file() then
-            fetch_session_interface_variables(data)
-        end
-    end
-
-end
-
-class WWW
-
-    def send_reply(logdata='')
-        if @interface.true?('trace:run') then
-            send_result(logdata)
-        else
-            dir, tmp = dirname, tmp_path(dirname)
-            case @session.get('status')
-                when 'running: finished' then
-                    resultname, replyname = 'result.pdf', 'reply.exa'
-                    replyfile = File.join(tmp,replyname)
-                    if FileTest.file?(replyfile) then
-                        begin
-                            data = IO.read(replyfile)
-                            resultname = if data =~ /(.*?)<\/exa:output>/ then $1 else resultname end
-                        rescue
-                            plaintext(exareply('error in reply'))
-                            return
-                        end
-                    end
-                    resultfile = File.join(tmp,resultname)
-                    if FileTest.file?(resultfile) then
-                        if indirect?(resultfile) then
-                            begin
-                                File.makedirs(File.join(cache_root,dir))
-                                FileUtils::mv(resultfile,File.join(cache_root,dir,resultname))
-                            rescue
-                                plaintext(exareply('unable to access cache'))
-                            else
-                                plaintext(exareply('big file', "cache/#{dir}/#{resultname}", "#{File.size?(resultfile)}"))
-                            end
-                        else
-                            send_file(resultfile)
-                        end
-                    else
-                        plaintext(exareply('no result'))
-                    end
-                else # background, running, aborted
-                    plaintext(exareply(@session.get('status')))
-            end
-        end
-    end
-
-    def send_url(fullname)
-        dir, tmp = dirname, tmp_path(dirname)
-        resultname, replyname = 'result.pdf', 'reply.exa'
-        replyfile = File.join(tmp,replyname)
-        resultfile = File.join(tmp,resultname)
-        targetname = File.join(cache_root,dir,resultname)
-        # make sure that there is no target left in case of an
-        # error; needed in case of given session name
-        if FileTest.directory?(File.join(cache_root,dir)) then
-            File.delete(targetname) rescue false
-        end
-        # now try to locate the file
-        if FileTest.file?(fullname) then
-            if indirect?(fullname) then
-                begin
-                    # check if directory exists and (if so) delete left overs
-                    File.makedirs(File.join(cache_root,dir))
-                    File.delete(targetname) rescue false
-                    File.symlink(fullname,targetname) rescue message('Status',$!)
-                    unless FileTest.file?(targetname) then
-                        FileUtils::cp(fullname,targetname) rescue false
-                    end
-                rescue
-                    plaintext(exareply('unable to access cache'))
-                else
-                    plaintext(exareply('big file', "cache/#{dir}/#{resultname}", "#{File.size?(fullname)}"))
-                end
-            else
-                send_file(fullname)
-            end
-        else
-            message('Status', 'The file is not found')
-        end
-    end
-
-    def send_result(logdata='')
-        check_template_file('exalogin','exalogin-template.htm')
-        dir, tmp = dirname, tmp_path(dirname)
-        resultname, replyname, logname = 'result.pdf', 'reply.exa', 'log.htm'
-        case @session.get('status')
-            when 'running: background' then
-                if st = @session.get('starttime') then # fuzzy
-                    st = Time.now.to_i if st.empty?
-                    if (Time.now.to_i - st.to_i) > @interface.get('process:timeout').to_i then
-                        message('Status', 'Your request has been aborted (timeout)',true)
-                    else
-                        message('Status', 'Your request is queued',true,5,'exastatus')
-                    end
-                end
-            when 'running: busy' then
-                if st = @session.get('starttime') then # fuzzy
-                    st = Time.now.to_i if st.empty?
-                    if (Time.now.to_i - st.to_i) > @interface.get('process:timeout').to_i then
-                        message('Status', 'Your request has been aborted (timeout)',true)
-                    else
-                        message('Status', 'Your request is being processed',true,5,'exastatus')
-                    end
-                end
-            when 'running: aborted' then
-                message('Status', 'Your request has been aborted (timeout)',true)
-            when 'running: finished' then
-                if @interface.true?('trace:run') then
-                    logfile = File.join(tmp,logname)
-                    begin
-                        if f = File.open(logname,'w') then
-                            if logdata.empty? then
-                                begin
-                                    logdata = IO.read('www-watch.out')
-                                rescue
-                                    logdata = 'no log data'
-                                end
-                            end
-                            f << filled_template('Log',"
#{CGI::escapeHTML(logdata)}
")
-                            f.close
-                        end
-                    rescue
-                        message('Error', '')
-                    end
-                    if FileTest.file?(logfile) then
-                        begin
-                            File.makedirs(File.join(cache_root,dir))
-                            FileUtils::mv(logfile,File.join(cache_root,dir,logname))
-                        rescue
-                            logdata = "

unable to access cache"
-                        else
-                            logdata = "

#{logname}"
-                        end
-                    else
-                        logdata = ''
-                    end
-                else
-                    logdata = ''
-                end
-                # todo: generate reply.exa if no reply
-                replyfile = File.join(tmp,replyname)
-                if FileTest.file?(replyfile) then
-                    begin
-                        data = IO.read(replyfile)
-                        resultname = if data =~ /(.*?)<\/exa:output>/ then $1 else resultname end
-                    rescue
-                        message('Error','There is a problem in handling this request (invalid reply).')
-                        return
-                    end
-                end
-                resultfile = File.join(tmp,resultname)
-                if FileTest.file?(resultfile) then
-                    if indirect?(resultfile) then
-                        begin
-                            File.makedirs(File.join(cache_root,dir))
-                            FileUtils::mv(resultfile,File.join(cache_root,dir,resultname))
-                        rescue
-                            str = "

unable to access cache"
-                        else
-                            str = "

#{resultname}  (#{File.size?(resultname)} bytes)"
-                        end
-                        message('Result', 'You can pick up the result here:' + str + logdata)
-                    else
-                        send_file(resultfile)
-                    end
-                else
-                    message('Error', 'There is a problem in handling this request (no result file).' + logdata)
-                end
-        end
-    end
-
-end
diff --git a/scripts/context/ruby/www/login.rb b/scripts/context/ruby/www/login.rb
deleted file mode 100644
index 1c88a97e6..000000000
--- a/scripts/context/ruby/www/login.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-require 'www/lib'
-
-# basic login
-
-class WWW
-
-    def handle_login()
-        check_template_file('login','exalogin.htm')
-        set('password', '')
-        message('Login','')
-    end
-
-end
diff --git a/scripts/context/ruby/wwwclient.rb b/scripts/context/ruby/wwwclient.rb
deleted file mode 100644
index d41541a09..000000000
--- a/scripts/context/ruby/wwwclient.rb
+++ /dev/null
@@ -1,677 +0,0 @@
-#!/usr/bin/env ruby
-
-# a direct request is just passed on
-#
-#   exaclient --direct --request=somerequest.exa --result=somefile.pdf
-#
-# in an extended request the filename in the template file is replaced by the filename
-# given on the command line; templates are located on the current path and at parent
-# directories (two levels); the filename is expanded to a full path
-#
-#   exaclient --extend --template=tmicare-l-h.exa --file=somefile.xml --result=somefile.pdf
-#
-# a constructed request is build out of the provided filename and action; the filename is
-# expanded to a full path
-#
-#   exaclient --construct --action=tmicare-s-h.exa --file=somefile.xml --result=somefile.pdf
-#
-# in all cases, the result is either determined by a switch or taken from a reply file
-
-banner = ['WWWClient', 'version 1.0.0', '2003-2006', 'PRAGMA ADE/POD']
-
-$: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq!
-
-require 'base/switch'
-require 'base/logger'
-
-require 'timeout'
-require 'thread'
-require 'rexml/document'
-require 'net/http'
-
-class File
-
-    def File.backtracked(filename,level=3)
-        if level > 0 && filename && ! filename.empty? then
-            if FileTest.file?(filename) then
-                filename
-            else
-                File.backtracked('../'+filename,level-1)
-            end
-        else
-            filename
-        end
-    end
-
-    def File.expanded(filename)
-          File.expand_path(filename)
-    end
-
-end
-
-class Commands
-
-    include CommandBase
-
-end
-
-class Commands
-
-    @@namespace = "xmlns:exa='http://www.pragma-ade.com/schemas/example.rng'"
-    @@randchars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789" + "abcdefghijklmnopqrstuvwxyz"
-
-    def traceback
-        "(error: #{$!})" + "\n  -- " + $@.join("\n  >>")
-    end
-
-    def pdf(action,filename,enabled)
-        if enabled && FileTest.file?(filename) then
-            begin
-                report("pdf action #{action} on #{filename}")
-                case action
-                    when 'close' then system("pdfclose --all")
-                    when 'open'  then system("pdfopen --file #{filename}")
-                end
-            rescue
-                # forget about it
-            end
-        end
-    end
-
-    def status(replyfile,str) # when block, then ok
-        begin
-            # def status(*whatever)
-            # end
-            File.open(replyfile,'w') do |f|
-                report("saving reply info in '#{replyfile}'")
-                f.puts("\n\n")
-                f.puts("\n")
-                if block_given? then
-                    f.puts("  ok\n")
-                    f.puts("  #{yield}\n")
-                else
-                    f.puts("  error\n")
-                end
-                f.puts("  " + str + "\n")
-                f.puts("\n")
-                f.close
-                report("saving status: #{str}")
-            end
-        rescue
-            report("saving reply info in '#{replyfile}' fails")
-        ensure
-            exit
-        end
-        exit # to be real sure
-    end
-
-
-    def boundary_string (length) # copied from webrick/utils
-        rand_max = @@randchars.size
-        ret = ""
-        length.times do
-            ret << @@randchars[rand(rand_max)]
-        end
-        ret.upcase
-    end
-
-end
-
-class Commands
-
-    @@connecttimeout = 10*60 # ten minutes
-    @@processtimeout = 60*60 # an hour
-    @@polldelay      =  5    # 5 seconds
-
-    def main
-
-        datatemplate = @commandline.option('template')
-        datafile     = @commandline.option('file')
-        dataaction   = @commandline.option('action')
-
-        if ! datatemplate.empty? then
-            report("template '#{datatemplate}' specified without --construct")
-            report("aborting")
-        elsif ! dataaction.empty? then
-            report("action data '#{dataaction}' specified without --construct or --extend")
-            report("aborting")
-        elsif ! datafile.empty? then
-            report("action file '#{datafile}' specified without --construct or --extend")
-            report("aborting")
-        else
-            report("assuming --direct")
-            direct()
-        end
-
-    end
-
-    def construct
-
-        requestfile  = @commandline.option('request')
-        replyfile    = @commandline.option('reply')
-
-        datatemplate = @commandline.option('template')
-        datafile     = @commandline.option('file')
-        dataaction   = @commandline.option('action')
-
-        domain       = @commandline.option('domain')
-        project      = @commandline.option('project')
-        username     = @commandline.option('username')
-        password     = @commandline.option('password')
-
-        threshold    = @commandline.option('threshold')
-
-        datablob     = ''
-
-        begin
-            datablob = IO.read(datatemplate)
-        rescue
-            datablob = ''
-        else
-            begin
-                request = REXML::Document.new(datablob)
-                if e = REXML::XPath.match(request.root,"/exa:request/exa:data") then
-                    datablob = e.to_s.chomp
-                end
-            rescue
-                datablob = ''
-            end
-        end
-
-        begin
-            File.open(requestfile,'w') do |f|
-                f.puts "\n"
-                f.puts "\n"
-                f.puts "  \n"
-                f.puts "    #{dataaction}\n"      unless dataaction.empty?
-                f.puts "    #{datafile}\n"    unless datafile.empty?
-                f.puts "    #{threshold}\n" unless threshold.empty?
-                f.puts "  \n"
-                f.puts "  \n"
-                f.puts "    #{domain}\n"
-                f.puts "    #{project}\n"
-                f.puts "    #{username}\n"
-                f.puts "    #{password}\n"
-                f.puts "  \n"
-                if datablob.empty? then
-                    f.puts "  \n"
-                else
-                    f.puts "  #{datablob.chomp}\n"
-                end
-                f.puts ""
-            end
-        rescue
-            status(replyfile,"unable to create '#{requestfile}'")
-        end
-
-        direct()
-
-    end
-
-    def extend
-
-        requestfile  = @commandline.option('request')
-        replyfile    = @commandline.option('reply')
-
-        datatemplate = @commandline.option('template')
-        datafile     = @commandline.option('file')
-        dataaction   = @commandline.option('action')
-
-        threshold    = @commandline.option('threshold')
-
-        if datatemplate.empty? then
-            status(replyfile,"invalid data template '#{datatemplate}'")
-        else
-            begin
-                if FileTest.file?(datatemplate) && oldrequest = IO.read(datatemplate) then
-                    request, done = REXML::Document.new(oldrequest), false
-                    if ! threshold.empty? && e = REXML::XPath.match(request.root,"/exa:request/exa:application/exa:threshold") then
-                        e.text, done = threshold, true
-                    end
-                    if ! dataaction.empty? && e = REXML::XPath.match(request.root,"/exa:request/exa:application/exa:action") then
-                        e.text, done = dataaction, true
-                    end
-                    if ! datafile.empty? && e = REXML::XPath.match(request.root,"/exa:request/exa:application/exa:filename") then
-                        e.text, done = datafile, true
-                    end
-                    #
-                    if ! threshold.empty? && e = REXML::XPath.match(request.root,"/exa:request/exa:application") then
-                        e = e.add_element('exa:threshold')
-                        e.add_text(threshold.to_s)
-                        done = true
-                    end
-                    #
-                    report("nothing replaced in template file") unless done
-                    begin
-                        File.open(requestfile,'w') do |f|
-                            f.puts(newrequest.to_s)
-                        end
-                    rescue
-                        status(replyfile,"unable to create '#{requestfile}'")
-                    end
-                else
-                    status(replyfile,"unable to read data template '#{datatemplate}'")
-                end
-            rescue
-                status(replyfile,"unable to handle data template '#{datatemplate}'")
-            end
-        end
-
-        direct()
-
-    end
-
-    def direct
-
-        requestpath  = @commandline.option('path')
-        requestfile  = @commandline.option('request')
-        replyfile    = @commandline.option('reply')
-        resultfile   = @commandline.option('result')
-        datatemplate = @commandline.option('template')
-        datafile     = @commandline.option('file')
-        threshold    = @commandline.option('threshold')
-        address      = @commandline.option('address')
-        port         = @commandline.option('port')
-        session_id   = @commandline.option('session')
-        exaurl       = @commandline.option('exaurl')
-
-        exaurl = "/#{exaurl}" unless exaurl =~ /^\//
-
-        address.sub!(/^http\:\/\//io) do
-            ''
-        end
-        address.sub!(/\:(\d+)$/io) do
-            port = $1
-            ''
-        end
-
-        autopdf      = @commandline.option('autopdf')
-
-        dialogue     = nil
-
-        resultfile.sub!(/\.[a-z]+?$/, '') # don't overwrite the source
-
-        unless requestpath.empty? then
-            begin
-                if FileTest.directory?(requestpath) then
-                    if Dir.chdir(requestpath) then
-                        report("gone to path '#{requestpath}'")
-                    else
-                        status(replyfile,"unable to go to path '#{requestpath}")
-                    end
-                else
-                    status(replyfile,"unable to locate '#{requestpath}'")
-                end
-            rescue
-                status(replyfile,"unable to handle '#{requestpath}'")
-            end
-        end
-
-        datafile     = File.expand_path(datafile)       unless datafile.empty?
-        datatemplate = File.backtracked(datatemplate,3) unless datatemplate.empty?
-
-        # request must be valid
-
-        status(replyfile,'no request file') if requestfile.empty?
-        status(replyfile,"invalid request file '#{requestfile}'") unless FileTest.file?(requestfile)
-
-        begin
-            request = IO.readlines(requestfile).join('')
-            request = REXML::Document.new(request)
-            status(replyfile,'invalid request (no request)')           unless request.root.fully_expanded_name=='exa:request'
-            status(replyfile,'invalid request (no application block)') unless request.elements['exa:request'].elements['exa.application'] == nil # explicit nil test needed
-        rescue REXML::ParseException
-            status(replyfile,'invalid request (invalid xml file)')
-        rescue
-            status(replyfile,'invalid request (invalid file)')
-        else
-            report("using request file '#{requestfile}'")
-        end
-
-        # request can force session_id
-
-        if session_id && session_id.empty? then
-            begin
-                id = request.elements['exa:request'].elements['exa:application'].elements['exa:session'].text
-            rescue Exception
-                id = ''
-            ensure
-                if id && ! id.empty? then
-                    session_id = id
-                end
-            end
-        end
-
-        # request can overload reply name
-
-        begin
-            rreplyfile = request.elements['exa:request'].elements['exa:application'].elements['exa:output'].text
-        rescue Exception
-            rreplyfile = nil
-        ensure
-            if rreplyfile && ! rreplyfile.empty? then
-                replyfile = rreplyfile
-                report("reply file '#{replyfile} set by request'")
-            else
-                report("using reply file '#{replyfile}'")
-            end
-        end
-
-        # request can overload result name
-
-        begin
-            rresultfile = request.elements['exa:request'].elements['exa:application'].elements['exa:result']
-        rescue Exception
-            rresultfile = nil
-        ensure
-            if rresultfile && ! rresultfile.empty? then
-                resultfile = rresultfile
-                report("result file '#{resultfile}' set by request")
-            else
-                report("using result file '#{resultfile}'")
-            end
-        end
-
-        # try to connect to server
-
-        start_time = Time.now
-
-        processtimeout = begin @commandline.option('timeout').to_i rescue @@processtimeout end
-        processtimeout = @@processtimeout if processtimeout == 0 # 'xx'.to_i => 0
-
-        dialogue = start_dialogue(address, port, processtimeout)
-
-        if dialogue then
-            # continue
-        else
-            status(replyfile,'no connection')
-        end
-
-        # post request
-
-        timeout (@@processtimeout-10) do # -10 so that we run into this one first
-            begin
-                report("posting request of type '#{exaurl}'")
-                report("using session id '#{session_id}'") if session_id && ! session_id.empty?
-                firstline, chunks, total = nil, 0, 0
-                body, boundary, crlf = '', boundary_string(32), "\x0d\x0a"
-                body << '--' + boundary + crlf
-                body << "Content-Disposition: form-data; name=\"exa:request\""
-                body << crlf
-                body << "Content-Type: text/plain"
-                body << crlf + crlf
-                body << request.to_s
-                body << crlf + '--' + boundary + crlf
-if session_id && ! session_id.empty? then
-    body << "Content-Disposition: form-data; name=\"exa:session\""
-    body << "Content-Type: text/plain"
-    body << crlf + crlf
-    body << session_id
-    body << crlf + '--' + boundary + crlf
-end
-                begin
-                    File.open(datafile,'rb') do |df|
-                        body << "Content-Disposition: form-data; name=\"filename\""
-                        body << "Content-Type: text/plain"
-                        body << crlf + crlf
-                        body << datafile
-                        body << crlf + '--' + boundary + crlf
-                        body << "Content-Disposition: form-data; name=\"fakename\" ; filename=\"#{datafile}\""
-                        body << "Content-Type: application/octetstream"
-                        body << "Content-Transfer-Encoding: binary"
-                        body << crlf + crlf
-                        body << df.read
-                        body << crlf + '--' + boundary + '--' + crlf
-                    end
-                rescue
-                    # skip
-                end
-                headers = Hash.new
-                headers['content-type']   = "multipart/form-data; boundary=#{boundary}"
-                headers['content-length'] = body.length.to_s
-                begin
-                    File.open(resultfile,'wb') do |rf|
-                        begin
-                            # firstline is max 1024 but ok for reply
-                            dialogue.post(exaurl,body,headers) do |str|
-                                if ! firstline || firstline.empty? then
-                                    report('receiving result') if total == 0
-                                    firstline = str
-                                end
-                                total += 1
-                                rf.write(str)
-                            end
-                        rescue
-                            report("forced close #{traceback}")
-                        end
-                    end
-                rescue
-                    status(replyfile,'cannot open file')
-                end
-                begin
-                    File.delete(resultfile) if File.zero?(resultfile)
-                rescue
-                end
-                unless FileTest.file?(resultfile) then
-                    report("deleting empty resultfile")
-                    begin
-                        File.delete(resultfile)
-                    rescue
-                        # nice try, an error anyway
-                    end
-                    status(replyfile,'empty file')
-                else
-                    n, id, status = 0, '', ''
-                    loop do
-                        again = false
-                        if ! dialogue then
-                            again = true
-                        elsif firstline =~ /(\ 1
-                            unless dialogue then
-                                report('reestablishing connection')
-                                dialogue = start_dialogue(address, port, processtimeout)
-                            end
-                            if dialogue then
-                                begin
-                                    File.open(resultfile,'wb') do |rf|
-                                        begin
-                                            body = "id=#{id}"
-                                            headers = Hash.new
-                                            headers['content-type']   = "application/x-www-form-urlencoded"
-                                            headers['content-length'] = body.length.to_s
-                                            total, firstline = 0, ''
-                                            dialogue.post("/exastatus",body,headers) do |str|
-                                                if ! firstline || firstline.empty? then
-                                                    firstline = str
-                                                end
-                                                total += 1
-                                                rf.write(str)
-                                            end
-                                        rescue
-                                            report("forced close #{traceback}")
-                                            dialogue = nil
-                                            again = true
-                                        end
-                                    end
-                                    begin
-                                        File.delete(resultfile) if File.zero?(resultfile)
-                                    rescue
-                                    end
-                                rescue
-                                    report("error in opening file #{traceback}")
-                                    status(replyfile,'cannot open file')
-                                end
-                            else
-                                report("unable to make a connection")
-                                status(replyfile,'unable to make a connection') # exit
-                            end
-                        else
-                            break
-                        end
-                    end
-                    case firstline
-                        when /<\?xml\s*version=.*?\?>\s*#{resultfile}"
-                            end
-                        when /html/io then
-                            report("done, file #{resultfile}, type html, #{total} chunks, #{File.size? rescue 0} bytes")
-                            if resultfile =~ /\.(htm|html)$/i then
-                                report("file identified as 'html'")
-                            elsif resultfile =~ /\..*$/o
-                                report("result file suffix should be 'htm'")
-                            else
-                                newresultfile = resultfile + '.htm'
-                                begin
-                                    File.delete(newresultfile) if FileTest.file?(newresultfile)
-                                    resultfile = newresultfile if File.rename(resultfile,newresultfile)
-                                rescue
-                                    report("adding 'htm' suffix to result name failed")
-                                else
-                                    report("'htm' suffix added to result name")
-                                end
-                            end
-                            report("result saved in '#{resultfile}'")
-                            status(replyfile,'ok') do
-                                "#{resultfile}"
-                            end
-                        else
-                            report("no result file, first line #{firstline}")
-                            status(replyfile,'no result file')
-                    end
-                end
-            rescue TimeoutError
-                report("aborted due to time out")
-                status(replyfile,'time out')
-            rescue
-                report("aborted due to some problem #{traceback}")
-                status(replyfile,"no answer #{traceback}")
-            end
-        end
-
-        begin
-            report("run time: #{Time.now-start_time} seconds")
-        rescue
-        end
-
-    end
-
-    def start_dialogue(address, port, processtimeout)
-        timeout(@@connecttimeout) do
-            report("trying to connect to #{address}:#{port}")
-            begin
-                begin
-                    if dialogue = Net::HTTP.new(address, port) then
-                        # dialogue.set_debug_output $stderr
-                        dialogue.read_timeout = processtimeout # set this before start
-                        if dialogue.start then
-                            report("connected to #{address}:#{port}, timeout: #{processtimeout}")
-                        else
-                            retry
-                        end
-                    else
-                        retry
-                    end
-                rescue
-                    sleep(2)
-                    retry
-                else
-                    return dialogue
-                end
-            rescue TimeoutError
-                return nil
-            rescue
-                return nil
-            end
-        end
-    end
-
-end
-
-logger      = Logger.new(banner.shift)
-commandline = CommandLine.new
-
-commandline.registerflag('autopdf')
-
-commandline.registervalue('path'      , '')
-
-commandline.registervalue('request'   , 'request.exa')
-commandline.registervalue('reply'     , 'reply.exa')
-commandline.registervalue('result'    , 'result')
-
-commandline.registervalue('template'  , '')
-commandline.registervalue('file'      , '')
-commandline.registervalue('action'    , '')
-commandline.registervalue('timeout'   , '')
-
-commandline.registervalue('domain'    , 'default')
-commandline.registervalue('project'   , 'default')
-commandline.registervalue('username'  , 'guest')
-commandline.registervalue('password'  , 'anonymous')
-commandline.registervalue('exaurl'    , 'exarequest')
-commandline.registervalue('threshold' , '0')
-commandline.registervalue('session'   , '')
-
-commandline.registervalue('address'   , 'localhost')
-commandline.registervalue('port'      , '80')
-
-commandline.registeraction('direct'   , '[--path --request --reply --result --autopdf]')
-commandline.registeraction('construct', '[--path --request --reply --result --autopdf] --file --action')
-commandline.registeraction('extend'   , '[--path --request --reply --result --autopdf] --file --action --template')
-
-commandline.registeraction('direct')
-commandline.registeraction('construct')
-commandline.registeraction('extend')
-
-commandline.registerflag('verbose')
-commandline.registeraction('help')
-commandline.registeraction('version')
-
-commandline.expand
-
-Commands.new(commandline,logger,banner).send(commandline.action || 'main')
diff --git a/scripts/context/ruby/wwwserver.rb b/scripts/context/ruby/wwwserver.rb
deleted file mode 100644
index 13d5d1312..000000000
--- a/scripts/context/ruby/wwwserver.rb
+++ /dev/null
@@ -1,293 +0,0 @@
-#!/usr/bin/env ruby
-
-banner = ['WWWServer', 'version 1.0.0', '2003-2006', 'PRAGMA ADE/POD']
-
-$: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq!
-
-require 'base/switch'
-require 'base/logger'
-
-require 'monitor'
-
-# class WWW < Monitor
-# end
-# class Server < Monitor
-# end
-
-require 'www/lib'
-require 'www/dir'
-require 'www/login'
-require 'www/exa'
-
-require 'tempfile'
-require 'ftools'
-require 'webrick'
-
-class Server
-
-    attr_accessor :document_root, :work_path, :logs_path, :port_number, :exa_url, :verbose, :trace, :direct
-
-    def initialize(logger)
-        @httpd = nil
-        @document_root = ''
-        @work_path = ''
-        @logs_path = ''
-        @port_number = 8061
-        @exa_url = 'http://localhost:8061'
-        @logger = logger
-        @n_of_clients = 500
-        @request_timeout = 5*60
-        @verbose = false
-        @trace = false
-        @direct = false
-    end
-
-    def report(str)
-        @logger.report(str) if @logger
-    end
-
-    def setup
-        if @document_root.empty? then
-            rootpath = File.expand_path($0)
-            @document_root = File.expand_path(File.join(File.dirname(rootpath),'..','documents'))
-            unless FileTest.directory?(@document_root) then # todo: optional
-                loop do
-                    prevpath = rootpath.dup
-                    rootpath = File.dirname(rootpath)
-                    if prevpath == rootpath then
-                        break
-                    else
-                        checkpath = File.join(rootpath,'documents')
-                        # report("locating: #{checkpath}")
-                        if FileTest.directory?(checkpath) then
-                            @document_root = checkpath
-                            break
-                        else
-                            checkpath = File.join(rootpath,'docroot/documents')
-                            # report("locating: #{checkpath}")
-                            if FileTest.directory?(checkpath) then
-                                @document_root = checkpath
-                                break
-                            end
-                        end
-                    end
-                end
-            end
-        end
-        @document_root = File.join(Dir.pwd, 'documents') unless FileTest.directory?(@document_root)
-        unless FileTest.directory?(@document_root) then
-            report("invalid document root: #{@document_root}")
-            exit
-        else
-            report("using document root: #{@document_root}")
-        end
-        #
-        @work_path = File.expand_path(File.join(@document_root,'..','work')) if @work_path.empty?
-        # begin File.makedirs(@work_path) ; rescue ; end # no, let's auto-temp
-        if ! FileTest.directory?(@work_path) || ! FileTest.writable?(@work_path) then
-            @work_path = File.expand_path(File.join(Dir.tmpdir,'exaserver','work'))
-            begin File.makedirs(@logs_path) ; rescue ; end
-        end
-        report("using work path: #{@work_path}")
-        #
-        @logs_path = File.expand_path(File.join(@document_root,'..','logs')) if @logs_path.empty?
-        # begin File.makedirs(@logs_path) ; rescue ; end # no, let's auto-temp
-        if ! FileTest.directory?(@logs_path) || ! FileTest.writable?(@logs_path) then
-            @logs_path = File.expand_path(File.join(Dir.tmpdir,'exaserver','logs'))
-            begin File.makedirs(@logs_path) ; rescue ; end
-        end
-        report("using log path: #{@logs_path}")
-        #
-        if @logs_path.empty? then
-            @logfile = $stderr
-            @accfile = $stderr
-        else
-            @logfile = File.join(@logs_path,'exa-info.log')
-            @accfile = File.join(@logs_path,'exa-access.log')
-            begin File.delete(@logfile) ; rescue ; end
-            begin File.delete(@accfile) ; rescue ; end
-        end
-        #
-        begin
-            @httpd = WEBrick::HTTPServer.new(
-                :DocumentRoot        => @document_root,
-                :DocumentRootOptions => { :FancyIndexing => false },
-                :DirectoryIndex      => ['index.html','index.htm','showcase.pdf'],
-                :Port                => @port_number.to_i,
-                :Logger              => WEBrick::Log.new(@logfile, WEBrick::Log::INFO), # DEBUG
-                :RequestTimeout      => @request_timeout,
-                :MaxClients          => @n_of_clients,
-                :AccessLog           => [
-                    [ @accfile, WEBrick::AccessLog::COMMON_LOG_FORMAT  ],
-                    [ @accfile, WEBrick::AccessLog::REFERER_LOG_FORMAT ],
-                    [ @accfile, WEBrick::AccessLog::AGENT_LOG_FORMAT   ],
-                # :CGIPathEnv   => ENV["PATH"]   # PATH environment variable for CGI.
-                ]
-            )
-        rescue
-            report("starting server at port: #{@port_number} failed")
-            exit
-        else
-            report("running server at port: #{@port_number}")
-        end
-
-        begin
-            #
-            @httpd.mount_proc("/dir") do |request,reply|
-                report("accepting /dir") if @verbose
-                web_session(request,reply).handle_dir
-            end
-            @httpd.mount_proc("/login") do |request,reply|
-                report("accepting /login") if @verbose
-                web_session(request,reply).handle_login
-            end
-            @httpd.mount("/cache", WEBrick::HTTPServlet::FileHandler, File.join(@work_path,'cache'))
-            # @httpd.mount_proc("/cache") do |request,reply|
-                # WEBrick::HTTPServlet::FileHandler(@httpd,@work_path) # not ok
-            # end
-            @httpd.mount_proc("/exalogin") do |request,reply|
-                report("accepting /exalogin") if @verbose
-                web_session(request,reply).handle_exalogin
-            end
-            @httpd.mount_proc("/exadefault") do |request,reply|
-                report("accepting /exadefault") if @verbose
-                web_session(request,reply).handle_exadefault
-            end
-            @httpd.mount_proc("/exainterface") do |request,reply|
-                report("accepting /exainterface") if @verbose
-                web_session(request,reply).handle_exainterface
-            end
-            @httpd.mount_proc("/exarequest") do |request,reply|
-                report("accepting /exarequest") if @verbose
-                web_session(request,reply).handle_exarequest
-            end
-            @httpd.mount_proc("/exacommand") do |request,reply|
-                report("accepting /exacommand") if @verbose
-                web_session(request,reply).handle_exacommand
-            end
-            @httpd.mount_proc("/exastatus") do |request,reply|
-                report("accepting /exastatus") if @verbose
-                web_session(request,reply).handle_exastatus
-            end
-            @httpd.mount_proc("/exaadmin") do |request,reply|
-                report("accepting /exaadmin") if @verbose
-                web_session(request,reply).handle_exaadmin
-            end
-            #
-        rescue
-            report("problem in starting server: #{$!}")
-        end
-        [:INT, :TERM, :EXIT].each do |signal|
-            trap(signal) do
-                @httpd.shutdown
-            end
-        end
-    end
-
-    def start
-        unless @httpd then
-            setup
-            @httpd.start
-        end
-    end
-
-    def stop
-        @httpd.shutdown if @httpd
-    end
-
-    def restart
-        stop
-        start
-    end
-
-    private
-
-    def web_session(request,reply)
-        www = WWW.new(@httpd,request,reply)
-        www.set('path:work', @work_path)
-        www.set('path:logs', @logs_path)
-        www.set('path:root', File.dirname(@document_root))
-        www.set('process:exaurl', @exa_url)
-        www.set('trace:errors','yes') if @trace
-        www.set('process:background', 'no') if @direct
-        return www
-    end
-
-end
-
-class Commands
-
-    include CommandBase
-
-    def start
-        if server = setup then server.start end
-    end
-
-    def stop
-        if server = setup then server.stop end
-    end
-
-    def restart
-        if server = setup then server.restart end
-    end
-
-    private
-
-    def setup
-        server = Server.new(logger)
-        server.document_root = @commandline.option('root')
-        server.verbose = @commandline.option('verbose')
-        if @commandline.option('forcetemp') then
-            server.work_path = Dir.tmpdir + '/exa/work'
-            server.logs_path = Dir.tmpdir + '/exa/logs'
-            [server.work_path,server.logs_path].each do |d|
-                begin
-                    File.makedirs(d) unless FileTest.directory?(d)
-                rescue
-                    report("unable to create #{d}")
-                    exit
-                end
-                unless FileTest.writable?(d) then
-                    report("unable to access #{d}")
-                    exit
-                end
-            end
-        else
-            server.work_path = @commandline.option('work')
-            server.logs_path = @commandline.option('logs')
-        end
-        server.port_number = @commandline.option('port')
-        server.exa_url = @commandline.option('url')
-        server.trace = @commandline.option('trace')
-        server.direct = @commandline.option('direct')
-        return server
-    end
-
-end
-
-logger      = Logger.new(banner.shift)
-commandline = CommandLine.new
-
-commandline.registervalue('root'   , '')
-commandline.registervalue('work'   , '')
-commandline.registervalue('logs'   , '')
-commandline.registervalue('address', 'localhost')
-commandline.registervalue('port'   , '8061')
-commandline.registervalue('url'    , 'http://localhost:8061')
-
-commandline.registeraction('start'  , 'start the server [--root --forcetemp --work --logs --address --port --url]')
-commandline.registeraction('stop'   , 'stop the server')
-commandline.registeraction('restart', 'restart the server')
-
-commandline.registerflag('forcetemp')
-commandline.registerflag('direct')
-commandline.registerflag('verbose')
-commandline.registerflag('trace')
-
-commandline.registeraction('help')
-commandline.registeraction('version')
-
-commandline.expand
-
-Commands.new(commandline,logger,banner).send(commandline.action || 'start')
-
diff --git a/scripts/context/ruby/wwwwatch.rb b/scripts/context/ruby/wwwwatch.rb
deleted file mode 100644
index 0faa45aec..000000000
--- a/scripts/context/ruby/wwwwatch.rb
+++ /dev/null
@@ -1,497 +0,0 @@
-#!/usr/bin/env ruby
-
-banner = ['WWWWatch', 'version 1.0.0', '2003-2006', 'PRAGMA ADE/POD']
-
-$: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq!
-
-require 'base/switch'
-require 'base/logger'
-
-require 'www/common'
-
-require 'monitor'
-require 'fileutils'
-require 'ftools'
-require 'tempfile'
-require 'timeout'
-require 'thread'
-
-class Watch < Monitor
-
-    include Common
-
-    @@session_prefix  = ''
-    @@check_factor    = 4
-    @@process_timeout = 1*60*60
-    @@fast_wait_loop  = false
-
-    @@session_line    = /^\s*(?![\#\%])(.*?)\s*\=\s*(.*?)\s*$/o
-    @@session_begin   = 'begin exa session'
-    @@session_end     = 'end exa session'
-
-    attr_accessor :root_path, :work_path, :create, :cache_path, :delay, :max_threads, :max_age, :verbose
-
-    def initialize(logger) # we need to register all @vars here becase of the monitor
-        @threads     = Hash.new
-        @files       = Array.new
-        @stats       = Hash.new
-        @skips       = Hash.new
-        @root_path   = ''
-        @work_path   = Dir.tmpdir
-        @cache_path  = @work_path
-        @last_action = Time.now
-        @delay       = 1
-        @max_threads = 5
-        @max_age     = @@process_timeout
-        @logger      = logger
-        @verbose     = false
-        @create      = false
-        @onlyonerun  = false
-        # [:INT, :TERM, :EXIT].each do |signal|
-            # trap(signal) do
-                # kill
-                # exit # rescue false
-            # end
-        # end
-        # at_exit do
-            # kill
-        # end
-    end
-
-    def trace
-        if @verbose && @logger then
-            @logger.report("exception: #{$!})")
-            $@.each do |t|
-                @logger.report(">> #{t}")
-            end
-        end
-    end
-
-    def report(str)
-        @logger.report(str) if @logger
-    end
-
-    def setup
-        @threads = Hash.new
-        @files   = Array.new
-        @stats   = Hash.new
-        @skips   = Hash.new
-        @root_path = File.expand_path(File.join(File.dirname(Dir.pwd),'.')) if @root_path.empty?
-        @work_path = File.expand_path(File.join(@root_path,'work','watch')) if @work_path.empty?
-        # @cache_path = File.expand_path(File.join(@root_path,'work','cache')) if @cache_path.empty?
-        @cache_path = File.expand_path(File.join(File.dirname(@work_path),'cache')) if @cache_path.empty?
-        if @create then
-            begin File.makedirs(@work_path)  ; rescue ; end
-            begin File.makedirs(@cache_path) ; rescue ; end
-        end
-        unless File.writable?(@work_path) then
-            @work_path = File.expand_path(File.join(Dir.tmpdir,'work','watch'))
-            if @create then
-                begin File.makedirs(@work_path) ; rescue ; end
-            end
-        end
-        unless File.writable?(@cache_path) then
-            @cache_path = File.expand_path(File.join(Dir.tmpdir,'work','cache'))
-            if @create then
-                begin File.makedirs(@cache_path) ; rescue ; end
-            end
-        end
-        unless File.writable?(@work_path) then
-            puts "no valid work path: #{@work_path}"
-            exit! rescue false # no checking, no at_exit done
-        end
-        unless File.writable?(@cache_path) then
-            puts "no valid cache path: #{@cache_path}" ; # no reason to exit
-        end
-        @last_action = Time.now
-        report("watching path #{@work_path}") if @verbose
-    end
-
-    def lock(lck)
-        begin
-            report("watchdog: locking #{lck}") if @verbose
-            File.open(lck,'w') do |f|
-                f << Time.now
-            end
-        rescue
-            trace
-        end
-    end
-
-    def unlock(lck)
-        begin
-            report("watchdog: unlocking #{lck}") if @verbose
-            File.delete(lck)
-        rescue
-            trace
-        end
-    end
-
-    def kill
-        @threads.each do |t|
-            t.kill rescue false
-        end
-    end
-
-    def restart
-        @files = Array.new
-        @skips = Hash.new
-        @stats = Hash.new
-        kill # threads
-    end
-
-    def collect
-        begin
-            @files = Array.new
-            Dir.glob("#{@work_path}/#{@@session_prefix}*.ses").each do |sessionfile|
-                sessionfile = File.expand_path(sessionfile)
-                begin
-                    if @threads.key?(sessionfile) then
-                        # leave alone
-                    elsif (Time.now - File.mtime(sessionfile)) > @max_age.to_i then
-                        # delete
-                        FileUtils::rm_r(sessionfile) rescue false
-                        FileUtils::rm_r(sessionfile.sub(/ses$/,'dir')) rescue false
-                        FileUtils::rm_r(sessionfile.sub(/ses$/,'lck')) rescue false
-                        begin
-                            FileUtils::rm_r(File.join(@cache_path, File.basename(sessionfile.sub(/ses$/,'dir'))))
-                        rescue
-                            report("watchdog: problems in cache cleanup #{$!}") # if @verbose
-                        end
-                        @stats.delete(sessionfile) rescue false
-                        @skips.delete(sessionfile) rescue false
-                        report("watchdog: removing session #{sessionfile}") if @verbose
-                    elsif ! @skips.key?(sessionfile) then
-                        @files << sessionfile
-                        report("watchdog: checking session #{sessionfile}") if @verbose
-                    end
-                rescue
-                    # maybe purged in the meantime
-                end
-            end
-        rescue
-            if File.directory?(@work_path) then
-                @files = Array.new
-            else
-                # maybe dir is deleted (manual cleanup)
-                restart
-            end
-        end
-        begin
-            Dir.glob("#{@cache_path}/*.dir").each do |dirname|
-                begin
-                    if (Time.now - File.mtime(dirname)) > @max_age.to_i then
-                        begin
-                            FileUtils::rm_r(dirname)
-                        rescue
-                            report("watchdog: problems in cache cleanup #{$!}") # if @verbose
-                        end
-                    end
-                rescue
-                    # maybe purged in the meantime
-                end
-            end
-        rescue
-        end
-    end
-
-    def purge
-        begin
-            Dir.glob("#{@work_path}/#{@@session_prefix}*").each do |sessionfile|
-                sessionfile = File.expand_path(sessionfile)
-                begin
-                    if (Time.now - File.mtime(sessionfile)) > @max_age.to_i then
-                        begin
-                            if FileTest.directory?(sessionfile) then
-                                FileUtils::rm_r(sessionfile)
-                            else
-                                File.delete(sessionfile)
-                            end
-                        rescue
-                        end
-                        begin
-                            @stats.delete(sessionfile)
-                            @skips.delete(sessionfile)
-                        rescue
-                        end
-                        report("watchdog: purging session #{sessionfile}") if @verbose
-                    end
-                rescue
-                    # maybe purged in the meantime
-                end
-            end
-        rescue
-        end
-    end
-
-    def loaded_session_data(filename)
-        begin
-            if data = IO.readlines(filename) then
-                return data if (data.first =~ /^[\#\%]\s*#{@@session_begin}/o) && (data.last =~ /^[\#\%]\s*#{@@session_end}/o)
-            end
-        rescue
-            trace
-        end
-        return nil
-    end
-
-    def load(sessionfile)
-        # we assume that we get an exception when the file is locked
-        begin
-            if data = loaded_session_data(sessionfile) then
-                report("watchdog: loading session #{sessionfile}") if @verbose
-                vars = Hash.new
-                data.each do |line|
-                    begin
-                        if line.chomp =~ /^(.*?)\s*\=\s*(.*?)\s*$/o then
-                            key, value = $1, $2
-                            vars[key] = value
-                        end
-                    rescue
-                    end
-                end
-                return vars
-            else
-                return nil
-            end
-        rescue
-            trace
-            return nil
-        end
-    end
-
-    def save(sessionfile, vars)
-        begin
-            report("watchdog: saving session #{sessionfile}") if @verbose
-            if @stats.key?(sessionfile) then
-                @stats[sessionfile] = File.mtime(sessionfile)
-            elsif @stats[sessionfile] == File.mtime(sessionfile) then
-            else
-                # construct data first
-                str = "\# #{@@session_begin}\n"
-                for k,v in vars do
-                    str << "#{k}=#{v}\n"
-                end
-                str << "\# #{@@session_end}\n"
-                # save as fast as possible
-                File.open(sessionfile,'w') do |f|
-                    f.puts(str)
-                end
-            end
-        rescue
-            report("watchdog: unable to save session #{sessionfile}") if @verbose
-            trace
-            return false
-        else
-            return true
-        end
-    end
-
-    def launch
-        begin
-            @files.each do |sessionfile|
-                if @threads.length < @max_threads then
-                    begin
-                        if ! @skips.key?(sessionfile) && (vars = load(sessionfile)) then
-                            if (id = vars['id']) && vars['status'] then
-                                if vars['status'] == 'running: background' then
-                                    @last_action = Time.now
-                                    @threads[sessionfile] = Thread.new(vars, sessionfile) do |vars, sessionfile|
-                                        begin
-                                            report("watchdog: starting thread #{sessionfile}") if @verbose
-                                            dir = File.expand_path(sessionfile.sub(/ses$/,'dir'))
-                                            lck = File.expand_path(sessionfile.sub(/ses$/,'lck'))
-                                            start_of_run = Time.now
-                                            start_of_job = start_of_run.dup
-                                            max_time = @max_age
-                                            begin
-                                                start_of_job = vars['starttime'].to_i || start_of_run
-                                                start_of_job = start_of_run if start_of_job == 0
-                                            rescue
-                                                start_of_job = Time.now
-                                            end
-                                            begin
-                                                max_runtime = vars['maxtime'].to_i || @max_age
-                                                max_runtime = @max_age if max_runtime == 0
-                                                max_runtime = max_runtime - (Time.now.to_i - start_of_job.to_i)
-                                            rescue
-                                                max_runtime = @max_age
-                                            end
-                                            lock(lck)
-                                            if max_runtime > 0 then
-                                                command = vars['command'] || ''
-                                                if ! command.empty? then
-                                                    vars['status'] = 'running: busy'
-                                                    vars['timeout'] = max_runtime.to_s
-                                                    save(sessionfile,vars)
-                                                    timeout(max_runtime) do
-                                                        begin
-                                                            command = command_string(dir,command,'process.log')
-                                                            report("watchdog: #{command}") if @verbose
-                                                            system(command)
-                                                        rescue TimeoutError
-                                                            vars['status'] = 'running: timeout'
-                                                        rescue
-                                                            trace
-                                                            vars['status'] = 'running: aborted'
-                                                        else
-                                                            vars['status'] = 'running: finished'
-                                                            vars['runtime'] = sprintf("%.02f",(Time.now - start_of_run))
-                                                            vars['endtime'] = Time.now.to_i.to_s
-                                                        end
-                                                    end
-                                                else
-                                                    vars['status'] = 'running: aborted' # no command
-                                                end
-                                            else
-                                                vars['status'] = 'running: aborted' # not enough time
-                                            end
-                                            save(sessionfile,vars)
-                                            unlock(lck)
-                                            report("watchdog: ending thread #{sessionfile}") if @verbose
-                                            @threads.delete(sessionfile)
-                                        rescue
-                                            trace
-                                        end
-                                    end
-                                else
-                                    report("watchdog: skipping - id (#{vars['id']}) / status (#{vars['status']})") if @verbose
-                                end
-                                if @onlyonerun then
-                                    @skips[sessionfile] = true
-                                else
-                                    @skips.delete(sessionfile)
-                                end
-                            else
-                                # not yet ok
-                            end
-                        else
-                            # maybe a lock
-                        end
-                    rescue
-                        trace
-                    end
-                else
-                    break
-                end
-            end
-        rescue
-            trace
-        end
-    end
-
-    def wait
-        begin
-            # report(Time.now.to_s) if @verbose
-            loop do
-                @threads.delete_if do |k,v|
-                    begin
-                        v == nil || v.stop?
-                    rescue
-                        true
-                    else
-                        false
-                    end
-                end
-                if @threads.length == @max_threads then
-                    if @delay > @max_threads then
-                        sleep(@delay)
-                    else
-                        sleep(@max_threads)
-                    end
-                    break if @@fast_wait_loop
-                else
-                    sleep(@delay)
-                    break
-                end
-            end
-        rescue
-            trace
-        end
-    end
-
-    def check
-        begin
-            time = Time.now
-            if (time - @last_action) > @@check_factor*@max_age then
-                report("watchdog: cleanup") if @verbose
-                @stats = Hash.new
-                @last_action = time
-                kill
-            end
-        rescue
-            trace
-        end
-    end
-
-    def cycle
-        loop do
-            begin
-                collect
-                launch
-                wait
-                check
-            rescue
-                trace
-                report("watchdog: some problem, restarting loop")
-            end
-        end
-    end
-
-end
-
-class Commands
-
-    include CommandBase
-
-    def watch
-        if watch = setup then
-            watch.cycle
-        else
-            report("provide valid work path")
-        end
-    end
-    def main
-        watch
-    end
-
-    private
-
-    def setup
-        if watch = Watch.new(logger) then
-            watch.root_path  = @commandline.option('root')
-            watch.work_path  = @commandline.option('work')
-            watch.cache_path = @commandline.option('cache')
-            watch.create     = @commandline.option('create')
-            watch.verbose    = @commandline.option('verbose')
-            begin
-                watch.max_threads = @commandline.option('threads').to_i
-            rescue
-                watch.max_threads = 5
-            end
-            watch.setup
-        end
-        return watch
-    end
-
-end
-
-logger      = Logger.new(banner.shift)
-commandline = CommandLine.new
-
-commandline.registervalue('root', '')
-commandline.registervalue('work', '')
-commandline.registervalue('cache', '')
-commandline.registervalue('threads', '5')
-
-commandline.registerflag('create')
-
-commandline.registeraction('watch', '[--work=path] [--root=path] [--create]')
-
-commandline.registerflag('verbose')
-commandline.registeraction('help')
-commandline.registeraction('version')
-
-commandline.expand
-
-Commands.new(commandline,logger,banner).send(commandline.action || 'main')
diff --git a/scripts/context/stubs/mswin/ctxtools.bat b/scripts/context/stubs/mswin/ctxtools.bat
index f1f5e019e..8047c9b68 100755
--- a/scripts/context/stubs/mswin/ctxtools.bat
+++ b/scripts/context/stubs/mswin/ctxtools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart ctxtools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute ctxtools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/exatools.bat b/scripts/context/stubs/mswin/exatools.bat
deleted file mode 100755
index 57f798e82..000000000
--- a/scripts/context/stubs/mswin/exatools.bat
+++ /dev/null
@@ -1,2 +0,0 @@
-@echo off
-texmfstart exatools.rb %*
diff --git a/scripts/context/stubs/mswin/luatools.lua b/scripts/context/stubs/mswin/luatools.lua
new file mode 100644
index 000000000..aacdbd16d
--- /dev/null
+++ b/scripts/context/stubs/mswin/luatools.lua
@@ -0,0 +1,6977 @@
+#!/usr/bin/env texlua
+
+if not modules then modules = { } end modules ['luatools'] = {
+    version   = 1.001,
+    comment   = "companion to context.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+-- one can make a stub:
+--
+-- #!/bin/sh
+-- env LUATEXDIR=/....../texmf/scripts/context/lua texlua luatools.lua "$@"
+
+-- Although this script is part of the ConTeXt distribution it is
+-- relatively indepent of ConTeXt.  The same is true for some of
+-- the luat files. We may may make them even less dependent in
+-- the future. As long as Luatex is under development the
+-- interfaces and names of functions may change.
+
+-- For the sake of independence we optionally can merge the library
+-- code here. It's too much code, but that does not harm. Much of the
+-- library code is used elsewhere. We don't want dependencies on
+-- Lua library paths simply because these scripts are located in the
+-- texmf tree and not in some Lua path. Normally this merge is not
+-- needed when texmfstart is used, or when the proper stub is used or
+-- when (windows) suffix binding is active.
+
+texlua = true
+
+-- begin library merge
+
+
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-string'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep
+
+if not string.split then
+
+    -- this will be overloaded by a faster lpeg variant
+
+    function string:split(pattern)
+        if #self > 0 then
+            local t = { }
+            for s in gmatch(self..pattern,"(.-)"..pattern) do
+                t[#t+1] = s
+            end
+            return t
+        else
+            return { }
+        end
+    end
+
+end
+
+local chr_to_esc = {
+    ["%"] = "%%",
+    ["."] = "%.",
+    ["+"] = "%+", ["-"] = "%-", ["*"] = "%*",
+    ["^"] = "%^", ["$"] = "%$",
+    ["["] = "%[", ["]"] = "%]",
+    ["("] = "%(", [")"] = "%)",
+    ["{"] = "%{", ["}"] = "%}"
+}
+
+string.chr_to_esc = chr_to_esc
+
+function string:esc() -- variant 2
+    return (gsub(self,"(.)",chr_to_esc))
+end
+
+function string:unquote()
+    return (gsub(self,"^([\"\'])(.*)%1$","%2"))
+end
+
+function string:quote() -- we could use format("%q")
+    return '"' .. self:unquote() .. '"'
+end
+
+function string:count(pattern) -- variant 3
+    local n = 0
+    for _ in gmatch(self,pattern) do
+        n = n + 1
+    end
+    return n
+end
+
+function string:limit(n,sentinel)
+    if #self > n then
+        sentinel = sentinel or " ..."
+        return sub(self,1,(n-#sentinel)) .. sentinel
+    else
+        return self
+    end
+end
+
+function string:strip()
+    return (gsub(self,"^%s*(.-)%s*$", "%1"))
+end
+
+function string:is_empty()
+    return not find(find,"%S")
+end
+
+function string:enhance(pattern,action)
+    local ok, n = true, 0
+    while ok do
+        ok = false
+        self = gsub(self,pattern, function(...)
+            ok, n = true, n + 1
+            return action(...)
+        end)
+    end
+    return self, n
+end
+
+local chr_to_hex, hex_to_chr = { }, { }
+
+for i=0,255 do
+    local c, h = char(i), format("%02X",i)
+    chr_to_hex[c], hex_to_chr[h] = h, c
+end
+
+function string:to_hex()
+    return (gsub(self or "","(.)",chr_to_hex))
+end
+
+function string:from_hex()
+    return (gsub(self or "","(..)",hex_to_chr))
+end
+
+if not string.characters then
+
+    local function nextchar(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, str:sub(index,index)
+    end
+    function string:characters()
+        return nextchar, self, 0
+    end
+    local function nextbyte(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, byte(str:sub(index,index))
+    end
+    function string:bytes()
+        return nextbyte, self, 0
+    end
+
+end
+
+-- we can use format for this (neg n)
+
+function string:rpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self .. self.rep(chr or " ",m)
+    else
+        return self
+    end
+end
+
+function string:lpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self.rep(chr or " ",m) .. self
+    else
+        return self
+    end
+end
+
+string.padd = string.rpadd
+
+function is_number(str) -- tonumber
+    return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1
+end
+
+--~ print(is_number("1"))
+--~ print(is_number("1.1"))
+--~ print(is_number(".1"))
+--~ print(is_number("-0.1"))
+--~ print(is_number("+0.1"))
+--~ print(is_number("-.1"))
+--~ print(is_number("+.1"))
+
+function string:split_settings() -- no {} handling, see l-aux for lpeg variant
+    if find(self,"=") then
+        local t = { }
+        for k,v in gmatch(self,"(%a+)=([^%,]*)") do
+            t[k] = v
+        end
+        return t
+    else
+        return nil
+    end
+end
+
+local patterns_escapes = {
+    ["-"] = "%-",
+    ["."] = "%.",
+    ["+"] = "%+",
+    ["*"] = "%*",
+    ["%"] = "%%",
+    ["("] = "%)",
+    [")"] = "%)",
+    ["["] = "%[",
+    ["]"] = "%]",
+}
+
+function string:pattesc()
+    return (gsub(self,".",patterns_escapes))
+end
+
+function string:tohash()
+    local t = { }
+    for s in gmatch(self,"([^, ]+)") do -- lpeg
+        t[s] = true
+    end
+    return t
+end
+
+local pattern = lpeg.Ct(lpeg.C(1)^0)
+
+function string:totable()
+    return pattern:match(self)
+end
+
+--~ for _, str in ipairs {
+--~     "1234567123456712345671234567",
+--~     "a\tb\tc",
+--~     "aa\tbb\tcc",
+--~     "aaa\tbbb\tccc",
+--~     "aaaa\tbbbb\tcccc",
+--~     "aaaaa\tbbbbb\tccccc",
+--~     "aaaaaa\tbbbbbb\tcccccc",
+--~ } do print(string.tabtospace(str)) end
+
+function string.tabtospace(str,tab)
+    -- we don't handle embedded newlines
+    while true do
+        local s = find(str,"\t")
+        if s then
+            if not tab then tab = 7 end -- only when found
+            local d = tab-(s-1)%tab
+            if d > 0 then
+                str = gsub(str,"\t",rep(" ",d),1)
+            else
+                str = gsub(str,"\t","",1)
+            end
+        else
+            break
+        end
+    end
+    return str
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-lpeg'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
+
+--~ l-lpeg.lua :
+
+--~ lpeg.digit         = lpeg.R('09')^1
+--~ lpeg.sign          = lpeg.S('+-')^1
+--~ lpeg.cardinal      = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.integer       = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.float         = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1)
+--~ lpeg.number        = lpeg.float + lpeg.integer
+--~ lpeg.oct           = lpeg.P("0") * lpeg.R('07')^1
+--~ lpeg.hex           = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1
+--~ lpeg.uppercase     = lpeg.P("AZ")
+--~ lpeg.lowercase     = lpeg.P("az")
+
+--~ lpeg.eol           = lpeg.S('\r\n\f')^1 -- includes formfeed
+--~ lpeg.space         = lpeg.S(' ')^1
+--~ lpeg.nonspace      = lpeg.P(1-lpeg.space)^1
+--~ lpeg.whitespace    = lpeg.S(' \r\n\f\t')^1
+--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1
+
+local hash = { }
+
+function lpeg.anywhere(pattern) --slightly adapted from website
+    return P { P(pattern) + 1 * lpeg.V(1) }
+end
+
+function lpeg.startswith(pattern) --slightly adapted
+    return P(pattern)
+end
+
+function lpeg.splitter(pattern, action)
+    return (((1-P(pattern))^1)/action+1)^0
+end
+
+-- variant:
+
+--~ local parser = lpeg.Ct(lpeg.splitat(newline))
+
+local crlf     = P("\r\n")
+local cr       = P("\r")
+local lf       = P("\n")
+local space    = S(" \t\f\v")  -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local newline  = crlf + cr + lf
+local spacing  = space^0 * newline
+
+local empty    = spacing * Cc("")
+local nonempty = Cs((1-spacing)^1) * spacing^-1
+local content  = (empty + nonempty)^1
+
+local capture = Ct(content^0)
+
+function string:splitlines()
+    return capture:match(self)
+end
+
+lpeg.linebyline = content -- better make a sublibrary
+
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps->what->more"))  -- oeps what more
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps->what->more"))  -- oeps what->more
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps"))              -- oeps
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps"))              -- oeps
+
+local splitters_s, splitters_m = { }, { }
+
+local function splitat(separator,single)
+    local splitter = (single and splitters_s[separator]) or splitters_m[separator]
+    if not splitter then
+        separator = P(separator)
+        if single then
+            local other, any = C((1 - separator)^0), P(1)
+            splitter = other * (separator * C(any^0) + "")
+            splitters_s[separator] = splitter
+        else
+            local other = C((1 - separator)^0)
+            splitter = other * (separator * other)^0
+            splitters_m[separator] = splitter
+        end
+    end
+    return splitter
+end
+
+lpeg.splitat = splitat
+
+local cache = { }
+
+function string:split(separator)
+    local c = cache[separator]
+    if not c then
+        c = Ct(splitat(separator))
+        cache[separator] = c
+    end
+    return c:match(self)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-table'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+table.join = table.concat
+
+local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove
+local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump
+local getmetatable, setmetatable = getmetatable, setmetatable
+local type, next, tostring, ipairs = type, next, tostring, ipairs
+
+function table.strip(tab)
+    local lst = { }
+    for i=1,#tab do
+        local s = gsub(tab[i],"^%s*(.-)%s*$","%1")
+        if s == "" then
+            -- skip this one
+        else
+            lst[#lst+1] = s
+        end
+    end
+    return lst
+end
+
+local function sortedkeys(tab)
+    local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+        if kind == 3 then
+            -- no further check
+        else
+            local tkey = type(key)
+            if tkey == "string" then
+            --  if kind == 2 then kind = 3 else kind = 1 end
+                kind = (kind == 2 and 3) or 1
+            elseif tkey == "number" then
+            --  if kind == 1 then kind = 3 else kind = 2 end
+                kind = (kind == 1 and 3) or 2
+            else
+                kind = 3
+            end
+        end
+    end
+    if kind == 0 or kind == 3 then
+        sort(srt,function(a,b) return (tostring(a) < tostring(b)) end)
+    else
+        sort(srt)
+    end
+    return srt
+end
+
+local function sortedhashkeys(tab) -- fast one
+    local srt = { }
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+    end
+    sort(srt)
+    return srt
+end
+
+table.sortedkeys     = sortedkeys
+table.sortedhashkeys = sortedhashkeys
+
+function table.sortedpairs(t)
+    local s = sortedhashkeys(t) -- maybe just sortedkeys
+    local n = 0
+    local function kv(s)
+        n = n + 1
+        local k = s[n]
+        return k, t[k]
+    end
+    return kv, s
+end
+
+function table.append(t, list)
+    for _,v in next, list do
+        insert(t,v)
+    end
+end
+
+function table.prepend(t, list)
+    for k,v in next, list do
+        insert(t,k,v)
+    end
+end
+
+function table.merge(t, ...) -- first one is target
+    t = t or {}
+    local lst = {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            t[k] = v
+        end
+    end
+    return t
+end
+
+function table.merged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            tmp[k] = v
+        end
+    end
+    return tmp
+end
+
+function table.imerge(t, ...)
+    local lst = {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            t[#t+1] = nst[j]
+        end
+    end
+    return t
+end
+
+function table.imerged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            tmp[#tmp+1] = nst[j]
+        end
+    end
+    return tmp
+end
+
+local function fastcopy(old) -- fast one
+    if old then
+        local new = { }
+        for k,v in next, old do
+            if type(v) == "table" then
+                new[k] = fastcopy(v) -- was just table.copy
+            else
+                new[k] = v
+            end
+        end
+        -- optional second arg
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
+        end
+        return new
+    else
+        return { }
+    end
+end
+
+local function copy(t, tables) -- taken from lua wiki, slightly adapted
+    tables = tables or { }
+    local tcopy = {}
+    if not tables[t] then
+        tables[t] = tcopy
+    end
+    for i,v in next, t do -- brrr, what happens with sparse indexed
+        if type(i) == "table" then
+            if tables[i] then
+                i = tables[i]
+            else
+                i = copy(i, tables)
+            end
+        end
+        if type(v) ~= "table" then
+            tcopy[i] = v
+        elseif tables[v] then
+            tcopy[i] = tables[v]
+        else
+            tcopy[i] = copy(v, tables)
+        end
+    end
+    local mt = getmetatable(t)
+    if mt then
+        setmetatable(tcopy,mt)
+    end
+    return tcopy
+end
+
+table.fastcopy = fastcopy
+table.copy     = copy
+
+-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack)
+
+function table.sub(t,i,j)
+    return { unpack(t,i,j) }
+end
+
+function table.replace(a,b)
+    for k,v in next, b do
+        a[k] = v
+    end
+end
+
+-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice)
+
+function table.is_empty(t)
+    return not t or not next(t)
+end
+
+function table.one_entry(t)
+    local n = next(t)
+    return n and not next(t,n)
+end
+
+function table.starts_at(t)
+    return ipairs(t,1)(t,0)
+end
+
+function table.tohash(t,value)
+    local h = { }
+    if t then
+        if value == nil then value = true end
+        for _, v in next, t do -- no ipairs here
+            h[v] = value
+        end
+    end
+    return h
+end
+
+function table.fromhash(t)
+    local h = { }
+    for k, v in next, t do -- no ipairs here
+        if v then h[#h+1] = k end
+    end
+    return h
+end
+
+--~ print(table.serialize(t), "\n")
+--~ print(table.serialize(t,"name"), "\n")
+--~ print(table.serialize(t,false), "\n")
+--~ print(table.serialize(t,true), "\n")
+--~ print(table.serialize(t,"name",true), "\n")
+--~ print(table.serialize(t,"name",true,true), "\n")
+
+table.serialize_functions = true
+table.serialize_compact   = true
+table.serialize_inline    = true
+
+local noquotes, hexify, handle, reduce, compact, inline, functions
+
+local reserved = table.tohash { -- intercept a language flaw, no reserved words as key
+    'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
+    'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
+}
+
+local function simple_table(t)
+    if #t > 0 then
+        local n = 0
+        for _,v in next, t do
+            n = n + 1
+        end
+        if n == #t then
+            local tt = { }
+            for i=1,#t do
+                local v = t[i]
+                local tv = type(v)
+                if tv == "number" then
+                    if hexify then
+                        tt[#tt+1] = format("0x%04X",v)
+                    else
+                        tt[#tt+1] = tostring(v) -- tostring not needed
+                    end
+                elseif tv == "boolean" then
+                    tt[#tt+1] = tostring(v)
+                elseif tv == "string" then
+                    tt[#tt+1] = format("%q",v)
+                else
+                    tt = nil
+                    break
+                end
+            end
+            return tt
+        end
+    end
+    return nil
+end
+
+-- Because this is a core function of mkiv I moved some function calls
+-- inline.
+--
+-- twice as fast in a test:
+--
+-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) )
+
+local function do_serialize(root,name,depth,level,indexed)
+    if level > 0 then
+        depth = depth .. " "
+        if indexed then
+            handle(format("%s{",depth))
+        elseif name then
+        --~ handle(format("%s%s={",depth,key(name)))
+            if type(name) == "number" then -- or find(k,"^%d+$") then
+                if hexify then
+                    handle(format("%s[0x%04X]={",depth,name))
+                else
+                    handle(format("%s[%s]={",depth,name))
+                end
+            elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then
+                handle(format("%s%s={",depth,name))
+            else
+                handle(format("%s[%q]={",depth,name))
+            end
+        else
+            handle(format("%s{",depth))
+        end
+    end
+    if root and next(root) then
+        local first, last = nil, 0 -- #root cannot be trusted here
+        if compact then
+            -- NOT: for k=1,#root do (we need to quit at nil)
+            for k,v in ipairs(root) do -- can we use next?
+                if not first then first = k end
+                last = last + 1
+            end
+        end
+        local sk = sortedkeys(root)
+        for i=1,#sk do
+            local k = sk[i]
+            local v = root[k]
+            --~ if v == root then
+                -- circular
+            --~ else
+            local t = type(v)
+            if compact and first and type(k) == "number" and k >= first and k <= last then
+                if t == "number" then
+                    if hexify then
+                        handle(format("%s 0x%04X,",depth,v))
+                    else
+                        handle(format("%s %s,",depth,v))
+                    end
+                elseif t == "string" then
+                    if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                        handle(format("%s %s,",depth,v))
+                    else
+                        handle(format("%s %q,",depth,v))
+                    end
+                elseif t == "table" then
+                    if not next(v) then
+                        handle(format("%s {},",depth))
+                    elseif inline then -- and #t > 0
+                        local st = simple_table(v)
+                        if st then
+                            handle(format("%s { %s },",depth,concat(st,", ")))
+                        else
+                            do_serialize(v,k,depth,level+1,true)
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1,true)
+                    end
+                elseif t == "boolean" then
+                    handle(format("%s %s,",depth,tostring(v)))
+                elseif t == "function" then
+                    if functions then
+                        handle(format('%s loadstring(%q),',depth,dump(v)))
+                    else
+                        handle(format('%s "function",',depth))
+                    end
+                else
+                    handle(format("%s %q,",depth,tostring(v)))
+                end
+            elseif k == "__p__" then -- parent
+                if false then
+                    handle(format("%s __p__=nil,",depth))
+                end
+            elseif t == "number" then
+            --~ if hexify then
+            --~     handle(format("%s %s=0x%04X,",depth,key(k),v))
+            --~ else
+            --~     handle(format("%s %s=%s,",depth,key(k),v))
+            --~ end
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,v))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    if hexify then
+                        handle(format("%s %s=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s %s=%s,",depth,k,v))
+                    end
+                else
+                    if hexify then
+                        handle(format("%s [%q]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                end
+            elseif t == "string" then
+                if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                --~ handle(format("%s %s=%s,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%s,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%s,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%s,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                else
+                --~ handle(format("%s %s=%q,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%q,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%q,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%q,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%q,",depth,k,v))
+                    end
+                end
+            elseif t == "table" then
+                if not next(v) then
+                    --~ handle(format("%s %s={},",depth,key(k)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]={},",depth,k))
+                        else
+                            handle(format("%s [%s]={},",depth,k))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s={},",depth,k))
+                    else
+                        handle(format("%s [%q]={},",depth,k))
+                    end
+                elseif inline then
+                    local st = simple_table(v)
+                    if st then
+                    --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", ")))
+                        if type(k) == "number" then -- or find(k,"^%d+$") then
+                            if hexify then
+                                handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", ")))
+                            else
+                                handle(format("%s [%s]={ %s },",depth,k,concat(st,", ")))
+                            end
+                        elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                            handle(format("%s %s={ %s },",depth,k,concat(st,", ")))
+                        else
+                            handle(format("%s [%q]={ %s },",depth,k,concat(st,", ")))
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1)
+                    end
+                else
+                    do_serialize(v,k,depth,level+1)
+                end
+            elseif t == "boolean" then
+            --~ handle(format("%s %s=%s,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%s,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%s,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%s,",depth,k,tostring(v)))
+                end
+            elseif t == "function" then
+                if functions then
+                    --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v)))
+                        else
+                            handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v)))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=loadstring(%q),",depth,k,dump(v)))
+                    else
+                        handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v)))
+                    end
+                end
+            else
+                --~ handle(format("%s %s=%q,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%q,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%q,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%q,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%q,",depth,k,tostring(v)))
+                end
+            end
+            --~ end
+        end
+    end
+   if level > 0 then
+        handle(format("%s},",depth))
+    end
+end
+
+-- replacing handle by a direct t[#t+1] = ... (plus test) is not much
+-- faster (0.03 on 1.00 for zapfino.tma)
+
+local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
+    noquotes = _noquotes
+    hexify = _hexify
+    handle = _handle or print
+    reduce = _reduce or false
+    compact = table.serialize_compact
+    inline  = compact and table.serialize_inline
+    functions = table.serialize_functions
+    local tname = type(name)
+    if tname == "string" then
+        if name == "return" then
+            handle("return {")
+        else
+            handle(name .. "={")
+        end
+    elseif tname == "number" then
+        if hexify then
+            handle(format("[0x%04X]={",name))
+        else
+            handle("[" .. name .. "]={")
+        end
+    elseif tname == "boolean" then
+        if name then
+            handle("return {")
+        else
+            handle("{")
+        end
+    else
+        handle("t={")
+    end
+    if root and next(root) then
+        do_serialize(root,name,"",0,indexed)
+    end
+    handle("}")
+end
+
+--~ name:
+--~
+--~ true     : return     { }
+--~ false    :            { }
+--~ nil      : t        = { }
+--~ string   : string   = { }
+--~ 'return' : return     { }
+--~ number   : [number] = { }
+
+function table.serialize(root,name,reduce,noquotes,hexify)
+    local t = { }
+    local function flush(s)
+        t[#t+1] = s
+    end
+    serialize(root,name,flush,reduce,noquotes,hexify)
+    return concat(t,"\n")
+end
+
+function table.tohandle(handle,root,name,reduce,noquotes,hexify)
+    serialize(root,name,handle,reduce,noquotes,hexify)
+end
+
+-- sometimes tables are real use (zapfino extra pro is some 85M) in which
+-- case a stepwise serialization is nice; actually, we could consider:
+--
+-- for line in table.serializer(root,name,reduce,noquotes) do
+--    ...(line)
+-- end
+--
+-- so this is on the todo list
+
+table.tofile_maxtab = 2*1024
+
+function table.tofile(filename,root,name,reduce,noquotes,hexify)
+    local f = io.open(filename,'w')
+    if f then
+        local maxtab = table.tofile_maxtab
+        if maxtab > 1 then
+            local t = { }
+            local function flush(s)
+                t[#t+1] = s
+                if #t > maxtab then
+                    f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice
+                    t = { }
+                end
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+            f:write(concat(t,"\n"),"\n")
+        else
+            local function flush(s)
+                f:write(s,"\n")
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+        end
+        f:close()
+    end
+end
+
+local function flatten(t,f,complete)
+    for i=1,#t do
+        local v = t[i]
+        if type(v) == "table" then
+            if complete or type(v[1]) == "table" then
+                flatten(v,f,complete)
+            else
+                f[#f+1] = v
+            end
+        else
+            f[#f+1] = v
+        end
+    end
+end
+
+function table.flatten(t)
+    local f = { }
+    flatten(t,f,true)
+    return f
+end
+
+function table.unnest(t) -- bad name
+    local f = { }
+    flatten(t,f,false)
+    return f
+end
+
+table.flatten_one_level = table.unnest
+
+-- the next three may disappear
+
+function table.remove_value(t,value) -- todo: n
+    if value then
+        for i=1,#t do
+            if t[i] == value then
+                remove(t,i)
+                -- remove all, so no: return
+            end
+        end
+    end
+end
+
+function table.insert_before_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i,str)
+                    return
+                end
+            end
+        end
+        insert(t,1,str)
+    elseif value then
+        insert(t,1,value)
+    end
+end
+
+function table.insert_after_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i+1,str)
+                    return
+                end
+            end
+        end
+        t[#t+1] = str
+    elseif value then
+        t[#t+1] = value
+    end
+end
+
+local function are_equal(a,b,n,m) -- indexed
+    if #a == #b then
+        n = n or 1
+        m = m or #a
+        for i=n,m do
+            local ai, bi = a[i], b[i]
+            if ai==bi then
+                -- same
+            elseif type(ai)=="table" and type(bi)=="table" then
+                if not are_equal(ai,bi) then
+                    return false
+                end
+            else
+                return false
+            end
+        end
+        return true
+    else
+        return false
+    end
+end
+
+local function identical(a,b) -- assumes same structure
+    for ka, va in next, a do
+        local vb = b[k]
+        if va == vb then
+            -- same
+        elseif type(va) == "table" and  type(vb) == "table" then
+            if not identical(va,vb) then
+                return false
+            end
+        else
+            return false
+        end
+    end
+    return true
+end
+
+table.are_equal = are_equal
+table.identical = identical
+
+-- maybe also make a combined one
+
+function table.compact(t)
+    if t then
+        for k,v in next, t do
+            if not next(v) then
+                t[k] = nil
+            end
+        end
+    end
+end
+
+function table.contains(t, v)
+    if t then
+        for i=1, #t do
+            if t[i] == v then
+                return i
+            end
+        end
+    end
+    return false
+end
+
+function table.count(t)
+    local n, e = 0, next(t)
+    while e do
+        n, e = n + 1, next(t,e)
+    end
+    return n
+end
+
+function table.swapped(t)
+    local s = { }
+    for k, v in next, t do
+        s[v] = k
+    end
+    return s
+end
+
+--~ function table.are_equal(a,b)
+--~     return table.serialize(a) == table.serialize(b)
+--~ end
+
+function table.clone(t,p) -- t is optional or nil or table
+    if not p then
+        t, p = { }, t or { }
+    elseif not t then
+        t = { }
+    end
+    setmetatable(t, { __index = function(_,key) return p[key] end })
+    return t
+end
+
+function table.hexed(t,seperator)
+    local tt = { }
+    for i=1,#t do tt[i] = format("0x%04X",t[i]) end
+    return concat(tt,seperator or " ")
+end
+
+function table.reverse_hash(h)
+    local r = { }
+    for k,v in next, h do
+        r[v] = lower(gsub(k," ",""))
+    end
+    return r
+end
+
+function table.reverse(t)
+    local tt = { }
+    if #t > 0 then
+        for i=#t,1,-1 do
+            tt[#tt+1] = t[i]
+        end
+    end
+    return tt
+end
+
+--~ function table.keys(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return k
+--~ end
+
+--~ function table.keys_as_string(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return concat(k,"")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-io'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local byte = string.byte
+
+if string.find(os.getenv("PATH"),";") then
+    io.fileseparator, io.pathseparator = "\\", ";"
+else
+    io.fileseparator, io.pathseparator = "/" , ":"
+end
+
+function io.loaddata(filename,textmode)
+    local f = io.open(filename,(textmode and 'r') or 'rb')
+    if f then
+        local data = f:read('*all')
+    --  garbagecollector.check(data)
+        f:close()
+        return data
+    else
+        return nil
+    end
+end
+
+function io.savedata(filename,data,joiner)
+    local f = io.open(filename,"wb")
+    if f then
+        if type(data) == "table" then
+            f:write(table.join(data,joiner or ""))
+        elseif type(data) == "function" then
+            data(f)
+        else
+            f:write(data)
+        end
+        f:close()
+        return true
+    else
+        return false
+    end
+end
+
+function io.exists(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return false
+    else
+        assert(f:close())
+        return true
+    end
+end
+
+function io.size(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return 0
+    else
+        local s = f:seek("end")
+        assert(f:close())
+        return s
+    end
+end
+
+function io.noflines(f)
+    local n = 0
+    for _ in f:lines() do
+        n = n + 1
+    end
+    f:seek('set',0)
+    return n
+end
+
+local nextchar = {
+    [ 4] = function(f)
+        return f:read(1,1,1,1)
+    end,
+    [ 2] = function(f)
+        return f:read(1,1)
+    end,
+    [ 1] = function(f)
+        return f:read(1)
+    end,
+    [-2] = function(f)
+        local a, b = f:read(1,1)
+        return b, a
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        return d, c, b, a
+    end
+}
+
+function io.characters(f,n)
+    if f then
+        return nextchar[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+local nextbyte = {
+    [4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(a), byte(b), byte(c), byte(d)
+        else
+            return nil, nil, nil, nil
+        end
+    end,
+    [2] = function(f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(a), byte(b)
+        else
+            return nil, nil
+        end
+    end,
+    [1] = function (f)
+        local a = f:read(1)
+        if a then
+            return byte(a)
+        else
+            return nil
+        end
+    end,
+    [-2] = function (f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(b), byte(a)
+        else
+            return nil, nil
+        end
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(d), byte(c), byte(b), byte(a)
+        else
+            return nil, nil, nil, nil
+        end
+    end
+}
+
+function io.bytes(f,n)
+    if f then
+        return nextbyte[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+function io.ask(question,default,options)
+    while true do
+        io.write(question)
+        if options then
+            io.write(string.format(" [%s]",table.concat(options,"|")))
+        end
+        if default then
+            io.write(string.format(" [%s]",default))
+        end
+        io.write(string.format(" "))
+        local answer = io.read()
+        answer = answer:gsub("^%s*(.*)%s*$","%1")
+        if answer == "" and default then
+            return default
+        elseif not options then
+            return answer
+        else
+            for _,v in pairs(options) do
+                if v == answer then
+                    return answer
+                end
+            end
+            local pattern = "^" .. answer
+            for _,v in pairs(options) do
+                if v:find(pattern) then
+                    return v
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-number'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+number = number or { }
+
+-- a,b,c,d,e,f = number.toset(100101)
+
+function number.toset(n)
+    return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)")
+end
+
+function number.toevenhex(n)
+    local s = format("%X",n)
+    if #s % 2 == 0 then
+        return s
+    else
+        return "0" .. s
+    end
+end
+
+-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5%
+-- on
+--
+-- for i=1,1000000 do
+--     local a,b,c,d,e,f,g,h = number.toset(12345678)
+--     local a,b,c,d         = number.toset(1234)
+--     local a,b,c           = number.toset(123)
+-- end
+--
+-- of course dedicated "(.)(.)(.)(.)" matches are even faster
+
+local one = lpeg.C(1-lpeg.S(''))^1
+
+function number.toset(n)
+    return one:match(tostring(n))
+end
+
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-set'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+set = set or { }
+
+local nums   = { }
+local tabs   = { }
+local concat = table.concat
+
+set.create = table.tohash
+
+function set.tonumber(t)
+    if next(t) then
+        local s = ""
+    --  we could save mem by sorting, but it slows down
+        for k, v in pairs(t) do
+            if v then
+            --  why bother about the leading space
+                s = s .. " " .. k
+            end
+        end
+        if not nums[s] then
+            tabs[#tabs+1] = t
+            nums[s] = #tabs
+        end
+        return nums[s]
+    else
+        return 0
+    end
+end
+
+function set.totable(n)
+    if n == 0 then
+        return { }
+    else
+        return tabs[n] or { }
+    end
+end
+
+function set.contains(n,s)
+    if type(n) == "table" then
+        return n[s]
+    elseif n == 0 then
+        return false
+    else
+        local t = tabs[n]
+        return t and t[s]
+    end
+end
+
+--~ local c = set.create{'aap','noot','mies'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ local c = set.create{'zus','wim','jet'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ print(t['jet'])
+--~ print(set.contains(t,'jet'))
+--~ print(set.contains(t,'aap'))
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-os'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lub.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+function os.resultof(command)
+    return io.popen(command,"r"):read("*all")
+end
+
+if not os.exec  then os.exec  = os.execute end
+if not os.spawn then os.spawn = os.execute end
+
+--~ os.type : windows | unix (new, we already guessed os.platform)
+--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new)
+
+if not io.fileseparator then
+    if find(os.getenv("PATH"),";") then
+        io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows"
+    else
+        io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix"
+    end
+end
+
+os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix"
+
+function os.launch(str)
+    if os.platform == "windows" then
+        os.execute("start " .. str) -- os.spawn ?
+    else
+        os.execute(str .. " &")     -- os.spawn ?
+    end
+end
+
+if not os.setenv then
+    function os.setenv() return false end
+end
+
+if not os.times then
+    -- utime  = user time
+    -- stime  = system time
+    -- cutime = children user time
+    -- cstime = children system time
+    function os.times()
+        return {
+            utime  = os.gettimeofday(), -- user
+            stime  = 0,                 -- system
+            cutime = 0,                 -- children user
+            cstime = 0,                 -- children system
+        }
+    end
+end
+
+os.gettimeofday = os.gettimeofday or os.clock
+
+local startuptime = os.gettimeofday()
+
+function os.runtime()
+    return os.gettimeofday() - startuptime
+end
+
+--~ print(os.gettimeofday()-os.time())
+--~ os.sleep(1.234)
+--~ print (">>",os.runtime())
+--~ print(os.date("%H:%M:%S",os.gettimeofday()))
+--~ print(os.date("%H:%M:%S",os.time()))
+
+os.arch = os.arch or function()
+    local a = os.resultof("uname -m") or "linux"
+    os.arch = function()
+        return a
+    end
+    return a
+end
+
+local platform
+
+function os.currentplatform(name,default)
+    if not platform then
+        local name = os.name or os.platform or name -- os.name is built in, os.platform is mine
+        if not name then
+            platform = default or "linux"
+        elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then
+            if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then
+                platform = "mswin-64"
+            else
+                platform = "mswin"
+            end
+        else
+            local architecture = os.arch()
+            if name == "linux" then
+                if find(architecture,"x86_64") then
+                    platform = "linux-64"
+                elseif find(architecture,"ppc") then
+                    platform = "linux-ppc"
+                else
+                    platform = "linux"
+                end
+            elseif name == "macosx" then
+                if find(architecture,"i386") then
+                    platform = "osx-intel"
+                else
+                    platform = "osx-ppc"
+                end
+            elseif name == "sunos" then
+                if find(architecture,"sparc") then
+                    platform = "solaris-sparc"
+                else -- if architecture == 'i86pc'
+                    platform = "solaris-intel"
+                end
+            elseif name == "freebsd" then
+                if find(architecture,"amd64") then
+                    platform = "freebsd-amd64"
+                else
+                    platform = "freebsd"
+                end
+            else
+                platform = default or name
+            end
+        end
+        function os.currentplatform()
+            return platform
+        end
+    end
+    return platform
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-file'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- needs a cleanup
+
+file = file or { }
+
+local concat = table.concat
+local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub
+
+function file.removesuffix(filename)
+    return (gsub(filename,"%.[%a%d]+$",""))
+end
+
+function file.addsuffix(filename, suffix)
+    if not find(filename,"%.[%a%d]+$") then
+        return filename .. "." .. suffix
+    else
+        return filename
+    end
+end
+
+function file.replacesuffix(filename, suffix)
+    return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix
+end
+
+function file.dirname(name)
+    return match(name,"^(.+)[/\\].-$") or ""
+end
+
+function file.basename(name)
+    return match(name,"^.+[/\\](.-)$") or name
+end
+
+function file.nameonly(name)
+    return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$",""))
+end
+
+function file.extname(name)
+    return match(name,"^.+%.([^/\\]-)$") or  ""
+end
+
+file.suffix = file.extname
+
+--~ print(file.join("x/","/y"))
+--~ print(file.join("http://","/y"))
+--~ print(file.join("http://a","/y"))
+--~ print(file.join("http:///a","/y"))
+--~ print(file.join("//nas-1","/y"))
+
+function file.join(...)
+    local pth = concat({...},"/")
+    pth = gsub(pth,"\\","/")
+    local a, b = match(pth,"^(.*://)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    a, b = match(pth,"^(//)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    return (gsub(pth,"//+","/"))
+end
+
+function file.iswritable(name)
+    local a = lfs.attributes(name)
+    if a and a.permissions:sub(2,2) == "w" then
+        return true
+    else
+        name = file.dirname(name) or "."
+        if name == "" then name = "." end
+        a = lfs.attributes(name)
+        return a and a.permissions:sub(2,2) == "w"
+    end
+end
+
+function file.isreadable(name)
+    local a = lfs.attributes(name)
+    return a and a.permissions:sub(1,1) == "r"
+end
+
+file.is_readable = file.isreadable
+file.is_writable = file.iswritable
+
+-- todo: lpeg
+
+function file.split_path(str)
+    local t = { }
+    str = gsub(str,"\\", "/")
+    str = gsub(str,"(%a):([;/])", "%1\001%2")
+    for name in gmatch(str,"([^;:]+)") do
+        if name ~= "" then
+            t[#t+1] = gsub(name,"\001",":")
+        end
+    end
+    return t
+end
+
+function file.join_path(tab)
+    return concat(tab,io.pathseparator) -- can have trailing //
+end
+
+function file.collapse_path(str)
+    str = gsub(str,"/%./","/")
+    local n, m = 1, 1
+    while n > 0 or m > 0 do
+        str, n = gsub(str,"[^/%.]+/%.%.$","")
+        str, m = gsub(str,"[^/%.]+/%.%./","")
+    end
+    str = gsub(str,"([^/])/$","%1")
+    str = gsub(str,"^%./","")
+    str = gsub(str,"/%.$","")
+    if str == "" then str = "." end
+    return str
+end
+
+--~ print(file.collapse_path("a/./b/.."))
+--~ print(file.collapse_path("a/aa/../b/bb"))
+--~ print(file.collapse_path("a/../.."))
+--~ print(file.collapse_path("a/.././././b/.."))
+--~ print(file.collapse_path("a/./././b/.."))
+--~ print(file.collapse_path("a/b/c/../.."))
+
+function file.robustname(str)
+    return (gsub(str,"[^%a%d%/%-%.\\]+","-"))
+end
+
+file.readdata = io.loaddata
+file.savedata = io.savedata
+
+function file.copy(oldname,newname)
+    file.savedata(newname,io.loaddata(oldname))
+end
+
+-- lpeg variants, slightly faster, not always
+
+--~ local period    = lpeg.P(".")
+--~ local slashes   = lpeg.S("\\/")
+--~ local noperiod  = 1-period
+--~ local noslashes = 1-slashes
+--~ local name      = noperiod^1
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1
+
+--~ function file.extname(name)
+--~     return pattern:match(name) or ""
+--~ end
+
+--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1)
+
+--~ function file.removesuffix(name)
+--~     return pattern:match(name)
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1
+
+--~ function file.basename(name)
+--~     return pattern:match(name) or name
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1
+
+--~ function file.dirname(name)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2)
+--~     else
+--~         return ""
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.addsuffix(name, suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.replacesuffix(name,suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2) .. "." .. suffix
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1
+
+--~ function file.nameonly(name)
+--~     local a, b = pattern:match(name)
+--~     if b then
+--~         return name:sub(a,b-2)
+--~     elseif a then
+--~         return name:sub(a)
+--~     else
+--~         return name
+--~     end
+--~ end
+
+--~ local test = file.extname
+--~ local test = file.basename
+--~ local test = file.dirname
+--~ local test = file.addsuffix
+--~ local test = file.replacesuffix
+--~ local test = file.nameonly
+
+--~ print(1,test("./a/b/c/abd.def.xxx","!!!"))
+--~ print(2,test("./../b/c/abd.def.xxx","!!!"))
+--~ print(3,test("a/b/c/abd.def.xxx","!!!"))
+--~ print(4,test("a/b/c/def.xxx","!!!"))
+--~ print(5,test("a/b/c/def","!!!"))
+--~ print(6,test("def","!!!"))
+--~ print(7,test("def.xxx","!!!"))
+
+--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim)
+
+-- also rewrite previous
+
+local letter    = lpeg.R("az","AZ") + lpeg.S("_-+")
+local separator = lpeg.P("://")
+
+local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/")
+local rootbased = lpeg.P("/") + letter*lpeg.P(":")
+
+-- ./name ../name  /name c: :// name/name
+
+function file.is_qualified_path(filename)
+    return qualified:match(filename)
+end
+
+function file.is_rootbased_path(filename)
+    return rootbased:match(filename)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-md5'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This also provides file checksums and checkers.
+
+local gsub, format, byte = string.gsub, string.format, string.byte
+
+local function convert(str,fmt)
+    return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end))
+end
+
+if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end
+if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end
+if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end
+
+--~ if not md5.HEX then
+--~     local function remap(chr) return format("%02X",byte(chr)) end
+--~     function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.hex then
+--~     local function remap(chr) return format("%02x",byte(chr)) end
+--~     function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.dec then
+--~     local function remap(chr) return format("%03i",byte(chr)) end
+--~     function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+
+file.needs_updating_threshold = 1
+
+function file.needs_updating(oldname,newname) -- size modification access change
+    local oldtime = lfs.attributes(oldname, modification)
+    local newtime = lfs.attributes(newname, modification)
+    if newtime >= oldtime then
+        return false
+    elseif oldtime - newtime < file.needs_updating_threshold then
+        return false
+    else
+        return true
+    end
+end
+
+function file.checksum(name)
+    if md5 then
+        local data = io.loaddata(name)
+        if data then
+            return md5.HEXsum(data)
+        end
+    end
+    return nil
+end
+
+function file.loadchecksum(name)
+    if md5 then
+        local data = io.loaddata(name .. ".md5")
+        return data and data:gsub("%s","")
+    end
+    return nil
+end
+
+function file.savechecksum(name, checksum)
+    if not checksum then checksum = file.checksum(name) end
+    if checksum then
+        io.savedata(name .. ".md5",checksum)
+        return checksum
+    end
+    return nil
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-url'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local char, gmatch = string.char, string.gmatch
+local tonumber, type = tonumber, type
+
+-- from the spec (on the web):
+--
+--     foo://example.com:8042/over/there?name=ferret#nose
+--     \_/   \______________/\_________/ \_________/ \__/
+--      |           |            |            |        |
+--   scheme     authority       path        query   fragment
+--      |   _____________________|__
+--     / \ /                        \
+--     urn:example:animal:ferret:nose
+
+url = url or { }
+
+local function tochar(s)
+    return char(tonumber(s,16))
+end
+
+local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1)
+
+local hexdigit  = lpeg.R("09","AF","af")
+local plus      = lpeg.P("+")
+local escaped   = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar)
+
+local scheme    =                 lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("")
+local authority = slash * slash * lpeg.Cs((escaped+(1-      slash-qmark-hash))^0)         + lpeg.Cc("")
+local path      = slash *         lpeg.Cs((escaped+(1-            qmark-hash))^0)         + lpeg.Cc("")
+local query     = qmark         * lpeg.Cs((escaped+(1-                  hash))^0)         + lpeg.Cc("")
+local fragment  = hash          * lpeg.Cs((escaped+(1-           endofstring))^0)         + lpeg.Cc("")
+
+local parser = lpeg.Ct(scheme * authority * path * query * fragment)
+
+function url.split(str)
+    return (type(str) == "string" and parser:match(str)) or str
+end
+
+function url.hashed(str)
+    local s = url.split(str)
+    return {
+        scheme = (s[1] ~= "" and s[1]) or "file",
+        authority = s[2],
+        path = s[3],
+        query = s[4],
+        fragment = s[5],
+        original = str
+    }
+end
+
+function url.filename(filename)
+    local t = url.hashed(filename)
+    return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename
+end
+
+function url.query(str)
+    if type(str) == "string" then
+        local t = { }
+        for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do
+            t[k] = v
+        end
+        return t
+    else
+        return str
+    end
+end
+
+--~ print(url.filename("file:///c:/oeps.txt"))
+--~ print(url.filename("c:/oeps.txt"))
+--~ print(url.filename("file:///oeps.txt"))
+--~ print(url.filename("file:///etc/test.txt"))
+--~ print(url.filename("/oeps.txt"))
+
+--~ from the spec on the web (sort of):
+--~
+--~ function test(str)
+--~     print(table.serialize(url.hashed(str)))
+--~ end
+--~
+--~ test("%56pass%20words")
+--~ test("file:///c:/oeps.txt")
+--~ test("file:///c|/oeps.txt")
+--~ test("file:///etc/oeps.txt")
+--~ test("file://./etc/oeps.txt")
+--~ test("file:////etc/oeps.txt")
+--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt")
+--~ test("http://www.ietf.org/rfc/rfc2396.txt")
+--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what")
+--~ test("mailto:John.Doe@example.com")
+--~ test("news:comp.infosystems.www.servers.unix")
+--~ test("tel:+1-816-555-1212")
+--~ test("telnet://192.0.2.16:80/")
+--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2")
+--~ test("/etc/passwords")
+--~ test("http://www.pragma-ade.com/spaced%20name")
+
+--~ test("zip:///oeps/oeps.zip#bla/bla.tex")
+--~ test("zip:///oeps/oeps.zip?bla/bla.tex")
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-dir'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type = type
+local find, gmatch = string.find, string.gmatch
+
+dir = dir or { }
+
+-- optimizing for no string.find (*) does not save time
+
+local attributes = lfs.attributes
+local walkdir    = lfs.dir
+
+local function glob_pattern(path,patt,recurse,action)
+    local ok, scanner
+    if path == "/" then
+        ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
+    else
+        ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
+    end
+    if ok and type(scanner) == "function" then
+        if not find(path,"/$") then path = path .. '/' end
+        for name in scanner do
+            local full = path .. name
+            local mode = attributes(full,'mode')
+            if mode == 'file' then
+                if find(full,patt) then
+                    action(full)
+                end
+            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
+                glob_pattern(full,patt,recurse,action)
+            end
+        end
+    end
+end
+
+dir.glob_pattern = glob_pattern
+
+local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V
+
+local pattern = Ct {
+    [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3),
+    [2] = C(((1-S("*?/"))^0 * P("/"))^0),
+    [3] = C(P(1)^0)
+}
+
+local filter = Cs ( (
+    P("**") / ".*" +
+    P("*")  / "[^/]*" +
+    P("?")  / "[^/]" +
+    P(".")  / "%%." +
+    P("+")  / "%%+" +
+    P("-")  / "%%-" +
+    P(1)
+)^0 )
+
+local function glob(str,t)
+    if type(str) == "table" then
+        local t = t or { }
+        for s=1,#str do
+            glob(str[s],t)
+        end
+        return t
+    elseif lfs.isfile(str) then
+        local t = t or { }
+        t[#t+1] = str
+        return t
+    else
+        local split = pattern:match(str)
+        if split then
+            local t = t or { }
+            local action = action or function(name) t[#t+1] = name end
+            local root, path, base = split[1], split[2], split[3]
+            local recurse = find(base,"%*%*")
+            local start = root .. path
+            local result = filter:match(start .. base)
+            glob_pattern(start,result,recurse,action)
+            return t
+        else
+            return { }
+        end
+    end
+end
+
+dir.glob = glob
+
+--~ list = dir.glob("**/*.tif")
+--~ list = dir.glob("/**/*.tif")
+--~ list = dir.glob("./**/*.tif")
+--~ list = dir.glob("oeps/**/*.tif")
+--~ list = dir.glob("/oeps/**/*.tif")
+
+local function globfiles(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func -- alas, we need this indirect way
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if recurse then
+                    globfiles(path .. "/" .. name,recurse,func,files)
+                end
+            elseif mode == "file" then
+                if func then
+                    if func(name) then
+                        files[#files+1] = path .. "/" .. name
+                    end
+                else
+                    files[#files+1] = path .. "/" .. name
+                end
+            end
+        end
+    end
+    return files
+end
+
+dir.globfiles = globfiles
+
+-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
+-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
+-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
+-- t = dir.glob("f:/minimal/tex/**/*")
+-- print(dir.ls("f:/minimal/tex/**/*"))
+-- print(dir.ls("*.tex"))
+
+function dir.ls(pattern)
+    return table.concat(glob(pattern),"\n")
+end
+
+--~ mkdirs("temp")
+--~ mkdirs("a/b/c")
+--~ mkdirs(".","/a/b/c")
+--~ mkdirs("a","b","c")
+
+local make_indeed = true -- false
+
+if string.find(os.getenv("PATH"),";") then
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        local first, middle, last
+        local drive = false
+        first, middle, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            -- empty network path == local path
+        else
+            first, last = str:match("^(//)/*(.-)$")
+            if first then
+                middle, last = str:match("([^/]+)/+(.-)$")
+                if middle then
+                    pth = "//" .. middle
+                else
+                    pth = "//" .. last
+                    last = ""
+                end
+            else
+                first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$")
+                if first then
+                    pth, drive = first .. middle, true
+                else
+                    middle, last = str:match("^(/*)(.-)$")
+                    if not middle then
+                        last = str
+                    end
+                end
+            end
+        end
+        for s in gmatch(last,"[^/]+") do
+            if pth == "" then
+                pth = s
+            elseif drive then
+                pth, drive = pth .. s, false
+            else
+                pth = pth .. "/" .. s
+            end
+            if make_indeed and not lfs.isdir(pth) then
+                lfs.mkdir(pth)
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("a:"))
+--~         print(dir.mkdirs("a:/b/c"))
+--~         print(dir.mkdirs("a:b/c"))
+--~         print(dir.mkdirs("a:/bbb/c"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        local first, nothing, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            first = lfs.currentdir() .. "/"
+            first = first:gsub("\\","/")
+        end
+        if not first then
+            first, last = str:match("^(//)/*(.*)$")
+        end
+        if not first then
+            first, last = str:match("^([a-zA-Z]:)(.*)$")
+            if first and not find(last,"^/") then
+                local d = lfs.currentdir()
+                if lfs.chdir(first) then
+                    first = lfs.currentdir()
+                    first = first:gsub("\\","/")
+                end
+                lfs.chdir(d)
+            end
+        end
+        if not first then
+            first, last = lfs.currentdir(), str
+            first = first:gsub("\\","/")
+        end
+        last = last:gsub("//","/")
+        last = last:gsub("/%./","/")
+        last = last:gsub("^/*","")
+        first = first:gsub("/*$","")
+        if last == "" then
+            return first
+        else
+            return first .. "/" .. last
+        end
+    end
+
+else
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        str = str:gsub("/+","/")
+        if find(str,"^/") then
+            pth = "/"
+            for s in gmatch(str,"[^/]+") do
+                local first = (pth == "/")
+                if first then
+                    pth = pth .. s
+                else
+                    pth = pth .. "/" .. s
+                end
+                if make_indeed and not first and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        else
+            pth = "."
+            for s in gmatch(str,"[^/]+") do
+                pth = pth .. "/" .. s
+                if make_indeed and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        if not find(str,"^/") then
+            str = lfs.currentdir() .. "/" .. str
+        end
+        str = str:gsub("//","/")
+        str = str:gsub("/%./","/")
+        return str
+    end
+
+end
+
+dir.makedirs = dir.mkdirs
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-boolean'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+boolean = boolean or { }
+
+local type, tonumber = type, tonumber
+
+function boolean.tonumber(b)
+    if b then return 1 else return 0 end
+end
+
+function toboolean(str,tolerant)
+    if tolerant then
+        local tstr = type(str)
+        if tstr == "string" then
+            return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t"
+        elseif tstr == "number" then
+            return tonumber(str) ~= 0
+        elseif tstr == "nil" then
+            return false
+        else
+            return str
+        end
+    elseif str == "true" then
+        return true
+    elseif str == "false" then
+        return false
+    else
+        return str
+    end
+end
+
+function string.is_boolean(str)
+    if type(str) == "string" then
+        if str == "true" or str == "yes" or str == "on" or str == "t" then
+            return true
+        elseif str == "false" or str == "no" or str == "off" or str == "f" then
+            return false
+        end
+    end
+    return nil
+end
+
+function boolean.alwaystrue()
+    return true
+end
+
+function boolean.falsetrue()
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-unicode'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+utf = utf or unicode.utf8
+
+local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs
+
+unicode = unicode or { }
+
+-- 0  EF BB BF      UTF-8
+-- 1  FF FE         UTF-16-little-endian
+-- 2  FE FF         UTF-16-big-endian
+-- 3  FF FE 00 00   UTF-32-little-endian
+-- 4  00 00 FE FF   UTF-32-big-endian
+
+unicode.utfname = {
+    [0] = 'utf-8',
+    [1] = 'utf-16-le',
+    [2] = 'utf-16-be',
+    [3] = 'utf-32-le',
+    [4] = 'utf-32-be'
+}
+
+function unicode.utftype(f) -- \000 fails !
+    local str = f:read(4)
+    if not str then
+        f:seek('set')
+        return 0
+    elseif find(str,"^%z%z\254\255") then
+        return 4
+    elseif find(str,"^\255\254%z%z") then
+        return 3
+    elseif find(str,"^\254\255") then
+        f:seek('set',2)
+        return 2
+    elseif find(str,"^\255\254") then
+        f:seek('set',2)
+        return 1
+    elseif find(str,"^\239\187\191") then
+        f:seek('set',3)
+        return 0
+    else
+        f:seek('set')
+        return 0
+    end
+end
+
+function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg
+    local result, tmp, n, m, p = { }, { }, 0, 0, 0
+    -- lf | cr | crlf / (cr:13, lf:10)
+    local function doit()
+        if n == 10 then
+            if p ~= 13 then
+                result[#result+1] = concat(tmp)
+                tmp = { }
+                p = 0
+            end
+        elseif n == 13 then
+            result[#result+1] = concat(tmp)
+            tmp = { }
+            p = n
+        else
+            tmp[#tmp+1] = utfchar(n)
+            p = 0
+        end
+    end
+    for l,r in bytepairs(str) do
+        if r then
+            if endian then
+                n = l*256 + r
+            else
+                n = r*256 + l
+            end
+            if m > 0 then
+                n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000
+                m = 0
+                doit()
+            elseif n >= 0xD800 and n <= 0xDBFF then
+                m = n
+            else
+                doit()
+            end
+        end
+    end
+    if #tmp > 0 then
+        result[#result+1] = concat(tmp)
+    end
+    return result
+end
+
+function unicode.utf32_to_utf8(str, endian)
+    local result = { }
+    local tmp, n, m, p = { }, 0, -1, 0
+    -- lf | cr | crlf / (cr:13, lf:10)
+    local function doit()
+        if n == 10 then
+            if p ~= 13 then
+                result[#result+1] = concat(tmp)
+                tmp = { }
+                p = 0
+            end
+        elseif n == 13 then
+            result[#result+1] = concat(tmp)
+            tmp = { }
+            p = n
+        else
+            tmp[#tmp+1] = utfchar(n)
+            p = 0
+        end
+    end
+    for a,b in bytepairs(str) do
+        if a and b then
+            if m < 0 then
+                if endian then
+                    m = a*256*256*256 + b*256*256
+                else
+                    m = b*256 + a
+                end
+            else
+                if endian then
+                    n = m + a*256 + b
+                else
+                    n = m + b*256*256*256 + a*256*256
+                end
+                m = -1
+                doit()
+            end
+        else
+            break
+        end
+    end
+    if #tmp > 0 then
+        result[#result+1] = concat(tmp)
+    end
+    return result
+end
+
+local function little(c)
+    local b = byte(c) -- b = c:byte()
+    if b < 0x10000 then
+        return char(b%256,b/256)
+    else
+        b = b - 0x10000
+        local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00
+        return char(b1%256,b1/256,b2%256,b2/256)
+    end
+end
+
+local function big(c)
+    local b = byte(c)
+    if b < 0x10000 then
+        return char(b/256,b%256)
+    else
+        b = b - 0x10000
+        local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00
+        return char(b1/256,b1%256,b2/256,b2%256)
+    end
+end
+
+function unicode.utf8_to_utf16(str,littleendian)
+    if littleendian then
+        return char(255,254) .. utfgsub(str,".",little)
+    else
+        return char(254,255) .. utfgsub(str,".",big)
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-math'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan
+
+if not math.round then
+    function math.round(x)
+        return floor(x + 0.5)
+    end
+end
+
+if not math.div then
+    function math.div(n,m)
+        return floor(n/m)
+    end
+end
+
+if not math.mod then
+    function math.mod(n,m)
+        return n % m
+    end
+end
+
+local pipi = 2*math.pi/360
+
+function math.sind(d)
+    return sin(d*pipi)
+end
+
+function math.cosd(d)
+    return cos(d*pipi)
+end
+
+function math.tand(d)
+    return tan(d*pipi)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-utils'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- hm, quite unreadable
+
+if not utils        then utils        = { } end
+if not utils.merger then utils.merger = { } end
+if not utils.lua    then utils.lua    = { } end
+
+utils.merger.m_begin = "begin library merge"
+utils.merger.m_end   = "end library merge"
+utils.merger.pattern =
+    "%c+" ..
+    "%-%-%s+" .. utils.merger.m_begin ..
+    "%c+(.-)%c+" ..
+    "%-%-%s+" .. utils.merger.m_end ..
+    "%c+"
+
+function utils.merger._self_fake_()
+    return
+        "-- " .. "created merged file" .. "\n\n" ..
+        "-- " .. utils.merger.m_begin  .. "\n\n" ..
+        "-- " .. utils.merger.m_end    .. "\n\n"
+end
+
+function utils.report(...)
+    print(...)
+end
+
+utils.merger.strip_comment = true
+
+function utils.merger._self_load_(name)
+    local f, data = io.open(name), ""
+    if f then
+        utils.report("reading merge from %s",name)
+        data = f:read("*all")
+        f:close()
+    else
+        utils.report("unknown file to merge %s",name)
+    end
+    if data and utils.merger.strip_comment then
+        -- saves some 20K
+        data = data:gsub("%-%-~[^\n\r]*[\r\n]", "")
+    end
+    return data or ""
+end
+
+function utils.merger._self_save_(name, data)
+    if data ~= "" then
+        local f = io.open(name,'w')
+        if f then
+            utils.report("saving merge from %s",name)
+            f:write(data)
+            f:close()
+        end
+    end
+end
+
+function utils.merger._self_swap_(data,code)
+    if data ~= "" then
+        return (data:gsub(utils.merger.pattern, function(s)
+            return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n"
+        end, 1))
+    else
+        return ""
+    end
+end
+
+--~ stripper:
+--~
+--~ data = string.gsub(data,"%-%-~[^\n]*\n","")
+--~ data = string.gsub(data,"\n\n+","\n")
+
+function utils.merger._self_libs_(libs,list)
+    local result, f, frozen = { }, nil, false
+    result[#result+1] = "\n"
+    if type(libs) == 'string' then libs = { libs } end
+    if type(list) == 'string' then list = { list } end
+    local foundpath = nil
+    for _, lib in ipairs(libs) do
+        for _, pth in ipairs(list) do
+            pth = string.gsub(pth,"\\","/") -- file.clean_path
+            utils.report("checking library path %s",pth)
+            local name = pth .. "/" .. lib
+            if lfs.isfile(name) then
+                foundpath = pth
+            end
+        end
+        if foundpath then break end
+    end
+    if foundpath then
+        utils.report("using library path %s",foundpath)
+        local right, wrong = { }, { }
+        for _, lib in ipairs(libs) do
+            local fullname = foundpath .. "/" .. lib
+            if lfs.isfile(fullname) then
+            --  right[#right+1] = lib
+                utils.report("merging library %s",fullname)
+                result[#result+1] = "do -- create closure to overcome 200 locals limit"
+                result[#result+1] = io.loaddata(fullname,true)
+                result[#result+1] = "end -- of closure"
+            else
+            --  wrong[#wrong+1] = lib
+                utils.report("no library %s",fullname)
+            end
+        end
+        if #right > 0 then
+            utils.report("merged libraries: %s",table.concat(right," "))
+        end
+        if #wrong > 0 then
+            utils.report("skipped libraries: %s",table.concat(wrong," "))
+        end
+    else
+        utils.report("no valid library path found")
+    end
+    return table.concat(result, "\n\n")
+end
+
+function utils.merger.selfcreate(libs,list,target)
+    if target then
+        utils.merger._self_save_(
+            target,
+            utils.merger._self_swap_(
+                utils.merger._self_fake_(),
+                utils.merger._self_libs_(libs,list)
+            )
+        )
+    end
+end
+
+function utils.merger.selfmerge(name,libs,list,target)
+    utils.merger._self_save_(
+        target or name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            utils.merger._self_libs_(libs,list)
+        )
+    )
+end
+
+function utils.merger.selfclean(name)
+    utils.merger._self_save_(
+        name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            ""
+        )
+    )
+end
+
+function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true
+ -- utils.report("compiling",luafile,"into",lucfile)
+    os.remove(lucfile)
+    local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile)
+    if strip ~= false then
+        command = "-s " .. command
+    end
+    local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0)
+    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+     -- utils.report("removing",luafile)
+        os.remove(luafile)
+    end
+    return done
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-tra'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- the  tag is kind of generic and used for functions that are not
+-- bound to a variable, like node.new, node.copy etc (contrary to for instance
+-- node.has_attribute which is bound to a has_attribute local variable in mkiv)
+
+debugger = debugger or { }
+
+local counters = { }
+local names = { }
+local getinfo = debug.getinfo
+local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch
+
+-- one
+
+local function hook()
+    local f = getinfo(2,"f").func
+    local n = getinfo(2,"Sn")
+--  if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end
+    if f then
+        local cf = counters[f]
+        if cf == nil then
+            counters[f] = 1
+            names[f] = n
+        else
+            counters[f] = cf + 1
+        end
+    end
+end
+local function getname(func)
+    local n = names[func]
+    if n then
+        if n.what == "C" then
+            return n.name or ''
+        else
+            -- source short_src linedefined what name namewhat nups func
+            local name = n.name or n.namewhat or n.what
+            if not name or name == "" then name = "?" end
+            return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name)
+        end
+    else
+        return "unknown"
+    end
+end
+function debugger.showstats(printer,threshold)
+    printer   = printer or texio.write or print
+    threshold = threshold or 0
+    local total, grandtotal, functions = 0, 0, 0
+    printer("\n") -- ugly but ok
+ -- table.sort(counters)
+    for func, count in pairs(counters) do
+        if count > threshold then
+            local name = getname(func)
+            if not name:find("for generator") then
+                printer(format("%8i  %s", count, name))
+                total = total + count
+            end
+        end
+        grandtotal = grandtotal + count
+        functions = functions + 1
+    end
+    printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+end
+
+-- two
+
+--~ local function hook()
+--~     local n = getinfo(2)
+--~     if n.what=="C" and not n.name then
+--~         local f = tostring(debug.traceback())
+--~         local cf = counters[f]
+--~         if cf == nil then
+--~             counters[f] = 1
+--~             names[f] = n
+--~         else
+--~             counters[f] = cf + 1
+--~         end
+--~     end
+--~ end
+--~ function debugger.showstats(printer,threshold)
+--~     printer   = printer or texio.write or print
+--~     threshold = threshold or 0
+--~     local total, grandtotal, functions = 0, 0, 0
+--~     printer("\n") -- ugly but ok
+--~  -- table.sort(counters)
+--~     for func, count in pairs(counters) do
+--~         if count > threshold then
+--~             printer(format("%8i  %s", count, func))
+--~             total = total + count
+--~         end
+--~         grandtotal = grandtotal + count
+--~         functions = functions + 1
+--~     end
+--~     printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+--~ end
+
+-- rest
+
+function debugger.savestats(filename,threshold)
+    local f = io.open(filename,'w')
+    if f then
+        debugger.showstats(function(str) f:write(str) end,threshold)
+        f:close()
+    end
+end
+
+function debugger.enable()
+    debug.sethook(hook,"c")
+end
+
+function debugger.disable()
+    debug.sethook()
+--~ counters[debug.getinfo(2,"f").func] = nil
+end
+
+function debugger.tracing()
+    local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0
+    if n > 0 then
+        function debugger.tracing() return true  end ; return true
+    else
+        function debugger.tracing() return false end ; return false
+    end
+end
+
+--~ debugger.enable()
+
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+
+--~ debugger.disable()
+
+--~ print("")
+--~ debugger.showstats()
+--~ print("")
+--~ debugger.showstats(print,3)
+
+trackers = trackers or { }
+
+local data, done = { }, { }
+
+local function set(what,value)
+    for w in gmatch(lower(what),"[^, ]+") do
+        for d, f in next, data do
+            if done[d] then
+                -- prevent recursion due to wildcards
+            elseif find(d,w) then
+                done[d] = true
+                for i=1,#f do
+                    f[i](value)
+                end
+            end
+        end
+    end
+end
+
+local function reset()
+    for d, f in next, data do
+        for i=1,#f do
+            f[i](false)
+        end
+    end
+end
+
+function trackers.register(what,...)
+    what = lower(what)
+    local w = data[what]
+    if not w then
+        w = { }
+        data[what] = w
+    end
+    for _, fnc in next, { ... } do
+        local typ = type(fnc)
+        if typ == "function" then
+            w[#w+1] = fnc
+        elseif typ == "string" then
+            w[#w+1] = function(value) set(fnc,value,nesting) end
+        end
+    end
+end
+
+function trackers.enable(what)
+    done = { }
+    set(what,true)
+end
+
+function trackers.disable(what)
+    done = { }
+    if not what or what == "" then
+        trackers.reset(what)
+    else
+        set(what,false)
+    end
+end
+
+function trackers.reset(what)
+    done = { }
+    reset()
+end
+
+function trackers.list() -- pattern
+    local list = table.sortedkeys(data)
+    local user, system = { }, { }
+    for l=1,#list do
+        local what = list[l]
+        if find(what,"^%*") then
+            system[#system+1] = what
+        else
+            user[#user+1] = what
+        end
+    end
+    return user, system
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-env'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- A former version provided functionality for non embeded core
+-- scripts i.e. runtime library loading. Given the amount of
+-- Lua code we use now, this no longer makes sense. Much of this
+-- evolved before bytecode arrays were available and so a lot of
+-- code has disappeared already.
+
+local trace_verbose  = false  trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+
+local format = string.format
+
+-- precautions
+
+os.setlocale(nil,nil) -- useless feature and even dangerous in luatex
+
+function os.setlocale()
+    -- no way you can mess with it
+end
+
+-- dirty tricks
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then
+    profiler.start("luatex-profile.log")
+end
+
+-- environment
+
+environment             = environment or { }
+environment.arguments   = { }
+environment.files       = { }
+environment.sortedflags = nil
+
+if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end
+if not environment.version or environment.version == "" then             environment.version = "unknown"   end
+if not environment.jobname                              then             environment.jobname = "unknown"   end
+
+function environment.initialize_arguments(arg)
+    local arguments, files = { }, { }
+    environment.arguments, environment.files, environment.sortedflags = arguments, files, nil
+    for index, argument in pairs(arg) do
+        if index > 0 then
+            local flag, value = argument:match("^%-+(.+)=(.-)$")
+            if flag then
+                arguments[flag] = string.unquote(value or "")
+            else
+                flag = argument:match("^%-+(.+)")
+                if flag then
+                    arguments[flag] = true
+                else
+                    files[#files+1] = argument
+                end
+            end
+        end
+    end
+    environment.ownname = environment.ownname or arg[0] or 'unknown.lua'
+end
+
+function environment.setargument(name,value)
+    environment.arguments[name] = value
+end
+
+-- todo: defaults, better checks e.g on type (boolean versus string)
+--
+-- tricky: too many hits when we support partials unless we add
+-- a registration of arguments so from now on we have 'partial'
+
+function environment.argument(name,partial)
+    local arguments, sortedflags = environment.arguments, environment.sortedflags
+    if arguments[name] then
+        return arguments[name]
+    elseif partial then
+        if not sortedflags then
+            sortedflags = { }
+            for _,v in pairs(table.sortedkeys(arguments)) do
+                sortedflags[#sortedflags+1] = "^" .. v
+            end
+            environment.sortedflags = sortedflags
+        end
+        -- example of potential clash: ^mode ^modefile
+        for _,v in ipairs(sortedflags) do
+            if name:find(v) then
+                return arguments[v:sub(2,#v)]
+            end
+        end
+    end
+    return nil
+end
+
+function environment.split_arguments(separator) -- rather special, cut-off before separator
+    local done, before, after = false, { }, { }
+    for _,v in ipairs(environment.original_arguments) do
+        if not done and v == separator then
+            done = true
+        elseif done then
+            after[#after+1] = v
+        else
+            before[#before+1] = v
+        end
+    end
+    return before, after
+end
+
+function environment.reconstruct_commandline(arg,noquote)
+    arg = arg or environment.original_arguments
+    if noquote and #arg == 1 then
+        local a = arg[1]
+        a = resolvers.resolve(a)
+        a = a:unquote()
+        return a
+    elseif next(arg) then
+        local result = { }
+        for _,a in ipairs(arg) do -- ipairs 1 .. #n
+            a = resolvers.resolve(a)
+            a = a:unquote()
+            a = a:gsub('"','\\"') -- tricky
+            if a:find(" ") then
+                result[#result+1] = a:quote()
+            else
+                result[#result+1] = a
+            end
+        end
+        return table.join(result," ")
+    else
+        return ""
+    end
+end
+
+if arg then
+
+    -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later)
+    local newarg, instring = { }, false
+
+    for index, argument in ipairs(arg) do
+        if argument:find("^\"") then
+            newarg[#newarg+1] = argument:gsub("^\"","")
+            if not argument:find("\"$") then
+                instring = true
+            end
+        elseif argument:find("\"$") then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","")
+            instring = false
+        elseif instring then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument
+        else
+            newarg[#newarg+1] = argument
+        end
+    end
+    for i=1,-5,-1 do
+        newarg[i] = arg[i]
+    end
+
+    environment.initialize_arguments(newarg)
+    environment.original_arguments = newarg
+    environment.raw_arguments = arg
+
+    arg = { } -- prevent duplicate handling
+
+end
+
+-- weird place ... depends on a not yet loaded module
+
+function environment.texfile(filename)
+    return resolvers.find_file(filename,'tex')
+end
+
+function environment.luafile(filename)
+    local resolved = resolvers.find_file(filename,'tex') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    resolved = resolvers.find_file(filename,'texmfscripts') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    return resolvers.find_file(filename,'luatexlibs') or ""
+end
+
+environment.loadedluacode = loadfile -- can be overloaded
+
+--~ function environment.loadedluacode(name)
+--~     if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then
+--~         local chunk = loadstring(io.loaddata("texluac.luc"))
+--~         os.remove("texluac.luc")
+--~         return chunk
+--~     else
+--~         environment.loadedluacode = loadfile -- can be overloaded
+--~         return loadfile(name)
+--~     end
+--~ end
+
+function environment.luafilechunk(filename) -- used for loading lua bytecode in the format
+    filename = file.replacesuffix(filename, "lua")
+    local fullname = environment.luafile(filename)
+    if fullname and fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading file %s", fullname)
+        end
+        return environment.loadedluacode(fullname)
+    else
+        if trace_verbose then
+            logs.report("fileio","unknown file %s", filename)
+        end
+        return nil
+    end
+end
+
+-- the next ones can use the previous ones / combine
+
+function environment.loadluafile(filename, version)
+    local lucname, luaname, chunk
+    local basename = file.removesuffix(filename)
+    if basename == filename then
+        lucname, luaname = basename .. ".luc",  basename .. ".lua"
+    else
+        lucname, luaname = nil, basename -- forced suffix
+    end
+    -- when not overloaded by explicit suffix we look for a luc file first
+    local fullname = (lucname and environment.luafile(lucname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+    end
+    if chunk then
+        assert(chunk)()
+        if version then
+            -- we check of the version number of this chunk matches
+            local v = version -- can be nil
+            if modules and modules[filename] then
+                v = modules[filename].version -- new method
+            elseif versions and versions[filename] then
+                v = versions[filename]        -- old method
+            end
+            if v == version then
+                return true
+            else
+                if trace_verbose then
+                    logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version)
+                end
+                environment.loadluafile(filename)
+            end
+        else
+            return true
+        end
+    end
+    fullname = (luaname and environment.luafile(luaname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+        if not chunk then
+            if verbose then
+                logs.report("fileio","unknown file %s", filename)
+            end
+        else
+            assert(chunk)()
+            return true
+        end
+    end
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-inf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+local statusinfo, n, registered = { }, 0, { }
+
+statistics = statistics or { }
+
+statistics.enable    = true
+statistics.threshold = 0.05
+
+-- timing functions
+
+local clock = os.gettimeofday or os.clock
+
+function statistics.hastimer(instance)
+    return instance and instance.starttime
+end
+
+function statistics.starttiming(instance)
+    if instance then
+        local it = instance.timing
+        if not it then
+            it = 0
+        end
+        if it == 0 then
+            instance.starttime = clock()
+            if not instance.loadtime then
+                instance.loadtime = 0
+            end
+        end
+        instance.timing = it + 1
+    end
+end
+
+function statistics.stoptiming(instance, report)
+    if instance then
+        local it = instance.timing
+        if it > 1 then
+            instance.timing = it - 1
+        else
+            local starttime = instance.starttime
+            if starttime then
+                local stoptime = clock()
+                local loadtime = stoptime - starttime
+                instance.stoptime = stoptime
+                instance.loadtime = instance.loadtime + loadtime
+                if report then
+                    statistics.report("load time %0.3f",loadtime)
+                end
+                instance.timing = 0
+                return loadtime
+            end
+        end
+    end
+    return 0
+end
+
+function statistics.elapsedtime(instance)
+    return format("%0.3f",(instance and instance.loadtime) or 0)
+end
+
+function statistics.elapsedindeed(instance)
+    local t = (instance and instance.loadtime) or 0
+    return t > statistics.threshold
+end
+
+-- general function
+
+function statistics.register(tag,fnc)
+    if statistics.enable and type(fnc) == "function" then
+        local rt = registered[tag] or (#statusinfo + 1)
+        statusinfo[rt] = { tag, fnc }
+        registered[tag] = rt
+        if #tag > n then n = #tag end
+    end
+end
+
+function statistics.show(reporter)
+    if statistics.enable then
+        if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end
+        -- this code will move
+        local register = statistics.register
+        register("luatex banner", function()
+            return string.lower(status.banner)
+        end)
+        register("control sequences", function()
+            return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra)
+        end)
+        register("callbacks", function()
+            local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
+            return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
+        end)
+        register("current memory usage", statistics.memused)
+        register("runtime",statistics.runtime)
+--         --
+        for i=1,#statusinfo do
+            local s = statusinfo[i]
+            local r = s[2]()
+            if r then
+                reporter(s[1],r,n)
+            end
+        end
+        statistics.enable = false
+    end
+end
+
+function statistics.show_job_stat(tag,data,n)
+    texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data))
+end
+
+function statistics.memused() -- no math.round yet -)
+    local round = math.round or math.floor
+    return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000))
+end
+
+if statistics.runtime then
+    -- already loaded and set
+elseif luatex and luatex.starttime then
+    statistics.starttime = luatex.starttime
+    statistics.loadtime = 0
+    statistics.timing = 0
+else
+    statistics.starttiming(statistics)
+end
+
+function statistics.runtime()
+    statistics.stoptiming(statistics)
+    return statistics.formatruntime(statistics.elapsedtime(statistics))
+end
+
+function statistics.formatruntime(runtime)
+    return format("%s seconds", statistics.elapsedtime(statistics))
+end
+
+function statistics.timed(action,report)
+    local timer = { }
+    report = report or logs.simple
+    statistics.starttiming(timer)
+    action()
+    statistics.stoptiming(timer)
+    report("total runtime: %s",statistics.elapsedtime(timer))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-log'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this is old code that needs an overhaul
+
+local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+
+if texlua then
+    write_nl = print
+    write    = io.write
+end
+
+--[[ldx--
+
This is a prelude to a more extensive logging module. For the sake
+of parsing log files, in addition to the standard logging we will
+provide an  structured file. Actually, any logging that
+is hooked into callbacks will be \XML\ by default.

+--ldx]]--
+
+logs     = logs     or { }
+logs.xml = logs.xml or { }
+logs.tex = logs.tex or { }
+
+--[[ldx--
+
This looks pretty ugly but we need to speed things up a bit.

+--ldx]]--
+
+logs.moreinfo = [[
+more information about ConTeXt and the tools that come with it can be found at:
+
+maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
+webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
+wiki     : http://contextgarden.net
+]]
+
+logs.levels = {
+    ['error']   = 1,
+    ['warning'] = 2,
+    ['info']    = 3,
+    ['debug']   = 4,
+}
+
+logs.functions = {
+    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct',
+    'start_run', 'stop_run',
+    'start_page_number', 'stop_page_number',
+    'report_output_pages', 'report_output_log',
+    'report_tex_stat', 'report_job_stat',
+    'show_open', 'show_close', 'show_load',
+}
+
+logs.tracers = {
+}
+
+logs.level = 0
+logs.mode  = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"))
+
+function logs.set_level(level)
+    logs.level = logs.levels[level] or level
+end
+
+function logs.set_method(method)
+    for _, v in next, logs.functions do
+        logs[v] = logs[method][v] or function() end
+    end
+end
+
+-- tex logging
+
+function logs.tex.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(category .. " | " .. format(fmt,...))
+    else
+        write_nl(category .. " |")
+    end
+end
+
+function logs.tex.line(fmt,...) -- new
+    if fmt then
+        write_nl(format(fmt,...))
+    else
+        write_nl("")
+    end
+end
+
+local texcount = tex and tex.count
+
+function logs.tex.start_page_number()
+    local real, user, sub = texcount[0], texcount[1], texcount[2]
+    if real > 0 then
+        if user > 0 then
+            if sub > 0 then
+                write(format("[%s.%s.%s",real,user,sub))
+            else
+                write(format("[%s.%s",real,user))
+            end
+        else
+            write(format("[%s",real))
+        end
+    else
+        write("[-")
+    end
+end
+
+function logs.tex.stop_page_number()
+    write("]")
+end
+
+logs.tex.report_job_stat = statistics.show_job_stat
+
+-- xml logging
+
+function logs.xml.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",category,format(fmt,...)))
+    else
+        write_nl(format("",category))
+    end
+end
+function logs.xml.line(fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",format(fmt,...)))
+    else
+        write_nl("")
+    end
+end
+
+function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
+function logs.xml.stop () if logs.level > 0 then tw("") end end
+function logs.xml.push () if logs.level > 0 then tw("" ) end end
+
+function logs.xml.start_run()
+    write_nl("")
+    write_nl("") --  xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng'
+    write_nl("")
+end
+
+function logs.xml.stop_run()
+    write_nl("")
+end
+
+function logs.xml.start_page_number()
+    write_nl(format("
")
+    write_nl("")
+end
+
+function logs.xml.report_output_pages(p,b)
+    write_nl(format("", p))
+    write_nl(format("", b))
+    write_nl("")
+end
+
+function logs.xml.report_output_log()
+end
+
+function logs.xml.report_tex_stat(k,v)
+    texiowrite_nl("log",""..tostring(v).."")
+end
+
+local level = 0
+
+function logs.xml.show_open(name)
+    level = level + 1
+    texiowrite_nl(format("",level,name))
+end
+
+function logs.xml.show_close(name)
+    texiowrite(" ")
+    level = level - 1
+end
+
+function logs.xml.show_load(name)
+    texiowrite_nl(format("",level+1,name))
+end
+
+--
+
+local name, banner = 'report', 'context'
+
+local function report(category,fmt,...)
+    if fmt then
+        write_nl(format("%s | %s: %s",name,category,format(fmt,...)))
+    elseif category then
+        write_nl(format("%s | %s",name,category))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+local function simple(fmt,...)
+    if fmt then
+        write_nl(format("%s | %s",name,format(fmt,...)))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+function logs.setprogram(_name_,_banner_,_verbose_)
+    name, banner = _name_, _banner_
+    if _verbose_ then
+        trackers.enable("resolvers.verbose")
+    end
+    logs.set_method("tex")
+    logs.report = report -- also used in libraries
+    logs.simple = simple -- only used in scripts !
+    if utils then
+        utils.report = simple
+    end
+    logs.verbose = _verbose_
+end
+
+function logs.setverbose(what)
+    if what then
+        trackers.enable("resolvers.verbose")
+    else
+        trackers.disable("resolvers.verbose")
+    end
+    logs.verbose = what or false
+end
+
+function logs.extendbanner(_banner_,_verbose_)
+    banner = banner .. " | ".. _banner_
+    if _verbose_ ~= nil then
+        logs.setverbose(what)
+    end
+end
+
+logs.verbose = false
+logs.report  = logs.tex.report
+logs.simple  = logs.tex.report
+
+function logs.reportlines(str) -- todo: 
+    for line in str:gmatch("(.-)[\n\r]") do
+        logs.report(line)
+    end
+end
+
+function logs.reportline() -- for scripts too
+    logs.report()
+end
+
+logs.simpleline = logs.reportline
+
+function logs.help(message,option)
+    logs.report(banner)
+    logs.reportline()
+    logs.reportlines(message)
+    local moreinfo = logs.moreinfo or ""
+    if moreinfo ~= "" and option ~= "nomoreinfo" then
+        logs.reportline()
+        logs.reportlines(moreinfo)
+    end
+end
+
+logs.set_level('error')
+logs.set_method('tex')
+
+function logs.system(whereto,process,jobname,category,...)
+    for i=1,10 do
+        local f = io.open(whereto,"a")
+        if f then
+            f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)))
+            f:close()
+            break
+        else
+            sleep(0.1)
+        end
+    end
+end
+
+--~ local syslogname = "oeps.xxx"
+--~
+--~ for i=1,10 do
+--~     logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+    comment   = "companion to luat-lib.tex",
+}
+
+-- After a few years using the code the large luat-inp.lua file
+-- has been split up a bit. In the process some functionality was
+-- dropped:
+--
+-- * support for reading lsr files
+-- * selective scanning (subtrees)
+-- * some public auxiliary functions were made private
+--
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
+-- This lib is multi-purpose and can be loaded again later on so that
+-- additional functionality becomes available. We will split thislogs.report("fileio",
+-- module in components once we're done with prototyping. This is the
+-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing
+-- something in this module one can best check with Taco or Hans first; there
+-- is some nasty trickery going on that relates to traditional kpse support.
+
+-- To be considered: hash key lowercase, first entry in table filename
+-- (any case), rest paths (so no need for optimization). Or maybe a
+-- separate table that matches lowercase names to mixed case when
+-- present. In that case the lower() cases can go away. I will do that
+-- only when we run into problems with names ... well ... Iwona-Regular.
+
+-- Beware, loading and saving is overloaded in luat-tmp!
+
+local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
+local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
+local next, type = next, type
+
+local trace_locating, trace_detail, trace_verbose  = false, false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+trackers.register("resolvers.detail",   function(v) trace_detail   = v trackers.enable("resolvers.verbose,resolvers.detail") end)
+
+if not resolvers then
+    resolvers = {
+        suffixes     = { },
+        formats      = { },
+        dangerous    = { },
+        suffixmap    = { },
+        alternatives = { },
+        locators     = { },  -- locate databases
+        hashers      = { },  -- load databases
+        generators   = { },  -- generate databases
+    }
+end
+
+local resolvers = resolvers
+
+resolvers.locators  .notfound = { nil }
+resolvers.hashers   .notfound = { nil }
+resolvers.generators.notfound = { nil }
+
+resolvers.cacheversion = '1.0.1'
+resolvers.cnfname      = 'texmf.cnf'
+resolvers.luaname      = 'texmfcnf.lua'
+resolvers.homedir      = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~'
+resolvers.cnfdefault   = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
+
+local dummy_path_expr = "^!*unset/*$"
+
+local formats      = resolvers.formats
+local suffixes     = resolvers.suffixes
+local dangerous    = resolvers.dangerous
+local suffixmap    = resolvers.suffixmap
+local alternatives = resolvers.alternatives
+
+formats['afm'] = 'AFMFONTS'       suffixes['afm'] = { 'afm' }
+formats['enc'] = 'ENCFONTS'       suffixes['enc'] = { 'enc' }
+formats['fmt'] = 'TEXFORMATS'     suffixes['fmt'] = { 'fmt' }
+formats['map'] = 'TEXFONTMAPS'    suffixes['map'] = { 'map' }
+formats['mp']  = 'MPINPUTS'       suffixes['mp']  = { 'mp' }
+formats['ocp'] = 'OCPINPUTS'      suffixes['ocp'] = { 'ocp' }
+formats['ofm'] = 'OFMFONTS'       suffixes['ofm'] = { 'ofm', 'tfm' }
+formats['otf'] = 'OPENTYPEFONTS'  suffixes['otf'] = { 'otf' } -- 'ttf'
+formats['opl'] = 'OPLFONTS'       suffixes['opl'] = { 'opl' }
+formats['otp'] = 'OTPINPUTS'      suffixes['otp'] = { 'otp' }
+formats['ovf'] = 'OVFFONTS'       suffixes['ovf'] = { 'ovf', 'vf' }
+formats['ovp'] = 'OVPFONTS'       suffixes['ovp'] = { 'ovp' }
+formats['tex'] = 'TEXINPUTS'      suffixes['tex'] = { 'tex' }
+formats['tfm'] = 'TFMFONTS'       suffixes['tfm'] = { 'tfm' }
+formats['ttf'] = 'TTFONTS'        suffixes['ttf'] = { 'ttf', 'ttc' }
+formats['pfb'] = 'T1FONTS'        suffixes['pfb'] = { 'pfb', 'pfa' }
+formats['vf']  = 'VFFONTS'        suffixes['vf']  = { 'vf' }
+
+formats['fea'] = 'FONTFEATURES'   suffixes['fea'] = { 'fea' }
+formats['cid'] = 'FONTCIDMAPS'    suffixes['cid'] = { 'cid', 'cidmap' }
+
+formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new
+suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua'
+
+formats ['lua'] = 'LUAINPUTS' -- new
+suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' }
+
+-- backward compatible ones
+
+alternatives['map files']            = 'map'
+alternatives['enc files']            = 'enc'
+alternatives['cid files']            = 'cid'
+alternatives['fea files']            = 'fea'
+alternatives['opentype fonts']       = 'otf'
+alternatives['truetype fonts']       = 'ttf'
+alternatives['truetype collections'] = 'ttc'
+alternatives['type1 fonts']          = 'pfb'
+
+-- obscure ones
+
+formats ['misc fonts'] = ''
+suffixes['misc fonts'] = { }
+
+formats     ['sfd']                      = 'SFDFONTS'
+suffixes    ['sfd']                      = { 'sfd' }
+alternatives['subfont definition files'] = 'sfd'
+
+-- In practice we will work within one tds tree, but i want to keep
+-- the option open to build tools that look at multiple trees, which is
+-- why we keep the tree specific data in a table. We used to pass the
+-- instance but for practical pusposes we now avoid this and use a
+-- instance variable.
+
+-- here we catch a few new thingies (todo: add these paths to context.tmf)
+--
+-- FONTFEATURES  = .;$TEXMF/fonts/fea//
+-- FONTCIDMAPS   = .;$TEXMF/fonts/cid//
+
+-- we always have one instance active
+
+resolvers.instance = resolvers.instance or nil -- the current one (slow access)
+local instance = resolvers.instance or nil -- the current one (fast access)
+
+function resolvers.newinstance()
+
+    -- store once, freeze and faster (once reset we can best use
+    -- instance.environment) maybe better have a register suffix
+    -- function
+
+    for k, v in next, suffixes do
+        for i=1,#v do
+            local vi = v[i]
+            if vi then
+                suffixmap[vi] = k
+            end
+        end
+    end
+
+    -- because vf searching is somewhat dangerous, we want to prevent
+    -- too liberal searching esp because we do a lookup on the current
+    -- path anyway; only tex (or any) is safe
+
+    for k, v in next, formats do
+        dangerous[k] = true
+    end
+    dangerous.tex = nil
+
+    -- the instance
+
+    local newinstance = {
+        rootpath        = '',
+        treepath        = '',
+        progname        = 'context',
+        engine          = 'luatex',
+        format          = '',
+        environment     = { },
+        variables       = { },
+        expansions      = { },
+        files           = { },
+        remap           = { },
+        configuration   = { },
+        setup           = { },
+        order           = { },
+        found           = { },
+        foundintrees    = { },
+        kpsevars        = { },
+        hashes          = { },
+        cnffiles        = { },
+        luafiles        = { },
+        lists           = { },
+        remember        = true,
+        diskcache       = true,
+        renewcache      = false,
+        scandisk        = true,
+        cachepath       = nil,
+        loaderror       = false,
+        sortdata        = false,
+        savelists       = true,
+        cleanuppaths    = true,
+        allresults      = false,
+        pattern         = nil, -- lists
+        data            = { }, -- only for loading
+        force_suffixes  = true,
+        fakepaths       = { },
+    }
+
+    local ne = newinstance.environment
+
+    for k,v in next, os.env do
+        ne[k] = resolvers.bare_variable(v)
+    end
+
+    return newinstance
+
+end
+
+function resolvers.setinstance(someinstance)
+    instance = someinstance
+    resolvers.instance = someinstance
+    return someinstance
+end
+
+function resolvers.reset()
+    return resolvers.setinstance(resolvers.newinstance())
+end
+
+local function reset_hashes()
+    instance.lists = { }
+    instance.found = { }
+end
+
+local function check_configuration() -- not yet ok, no time for debugging now
+    local ie = instance.environment
+    local function fix(varname,default)
+        local proname = varname .. "." .. instance.progname or "crap"
+        local p, v = ie[proname], ie[varname]
+        if not ((p and p ~= "") or (v and v ~= "")) then
+            instance.variables[varname] = default -- or environment?
+        end
+    end
+    local name = os.name
+    if name == "windows" then
+        fix("OSFONTDIR", "c:/windows/fonts//")
+    elseif name == "macosx" then
+        fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//")
+    else
+        -- bad luck
+    end
+    fix("LUAINPUTS"   , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm
+    fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("LUATEXLIBS"  , ".;$TEXMF/luatex/lua//")
+end
+
+function resolvers.bare_variable(str) -- assumes str is a string
+    return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2"))
+end
+
+function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail'
+    if n then
+        trackers.disable("resolvers.*")
+        trackers.enable("resolvers."..n)
+    end
+end
+
+resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE"))
+
+function resolvers.osenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
+        if e == nil then
+         -- value = "" -- false
+        else
+            value = resolvers.bare_variable(e)
+        end
+        ie[key] = value
+    end
+    return value or ""
+end
+
+function resolvers.env(key)
+    return instance.environment[key] or resolvers.osenv(key)
+end
+
+--
+
+local function expand_vars(lst) -- simple vars
+    local variables, env = instance.variables, resolvers.env
+    local function resolve(a)
+        return variables[a] or env(a)
+    end
+    for k=1,#lst do
+        lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve)
+    end
+end
+
+local function expanded_var(var) -- simple vars
+    local function resolve(a)
+        return instance.variables[a] or resolvers.env(a)
+    end
+    return (gsub(var,"%$([%a%d%_%-]+)",resolve))
+end
+
+local function entry(entries,name)
+    if name and (name ~= "") then
+        name = gsub(name,'%$','')
+        local result = entries[name..'.'..instance.progname] or entries[name]
+        if result then
+            return result
+        else
+            result = resolvers.env(name)
+            if result then
+                instance.variables[name] = result
+                resolvers.expand_variables()
+                return instance.expansions[name] or ""
+            end
+        end
+    end
+    return ""
+end
+
+local function is_entry(entries,name)
+    if name and name ~= "" then
+        name = gsub(name,'%$','')
+        return (entries[name..'.'..instance.progname] or entries[name]) ~= nil
+    else
+        return false
+    end
+end
+
+-- {a,b,c,d}
+-- a,b,c/{p,q,r},d
+-- a,b,c/{p,q,r}/d/{x,y,z}//
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a{b,c}{d,e}f
+-- {a,b,c,d}
+-- {a,b,c/{p,q,r},d}
+-- {a,b,c/{p,q,r}/d/{x,y,z}//}
+-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
+-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
+-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
+
+-- this one is better and faster, but it took me a while to realize
+-- that this kind of replacement is cleaner than messy parsing and
+-- fuzzy concatenating we can probably gain a bit with selectively
+-- applying lpeg, but experiments with lpeg parsing this proved not to
+-- work that well; the parsing is ok, but dealing with the resulting
+-- table is a pain because we need to work inside-out recursively
+
+local function splitpathexpr(str, t, validate)
+    -- no need for further optimization as it is only called a
+    -- few times, we can use lpeg for the sub; we could move
+    -- the local functions outside the body
+    t = t or { }
+    str = gsub(str,",}",",@}")
+    str = gsub(str,"{,","{@,")
+ -- str = "@" .. str .. "@"
+    local ok, done
+    local function do_first(a,b)
+        local t = { }
+        for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_second(a,b)
+        local t = { }
+        for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_both(a,b)
+        local t = { }
+        for sa in gmatch(a,"[^,]+") do
+            for sb in gmatch(b,"[^,]+") do
+                t[#t+1] = sa .. sb
+            end
+        end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_three(a,b,c)
+        return a .. b.. c
+    end
+    while true do
+        done = false
+        while true do
+            str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both)
+            if ok > 0 then done = true else break end
+        end
+        str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three)
+        if ok > 0 then done = true end
+        if not done then break end
+    end
+    str = gsub(str,"[{}]", "")
+    str = gsub(str,"@","")
+    if validate then
+        for s in gmatch(str,"[^,]+") do
+            s = validate(s)
+            if s then t[#t+1] = s end
+        end
+    else
+        for s in gmatch(str,"[^,]+") do
+            t[#t+1] = s
+        end
+    end
+    return t
+end
+
+local function expanded_path_from_list(pathlist) -- maybe not a list, just a path
+    -- a previous version fed back into pathlist
+    local newlist, ok = { }, false
+    for k=1,#pathlist do
+        if find(pathlist[k],"[{}]") then
+            ok = true
+            break
+        end
+    end
+    if ok then
+        local function validate(s)
+            s = file.collapse_path(s)
+            return s ~= "" and not find(s,dummy_path_expr) and s
+        end
+        for k=1,#pathlist do
+            splitpathexpr(pathlist[k],newlist,validate)
+        end
+    else
+        for k=1,#pathlist do
+            for p in gmatch(pathlist[k],"([^,]+)") do
+                p = file.collapse_path(p)
+                if p ~= "" then newlist[#newlist+1] = p end
+            end
+        end
+    end
+    return newlist
+end
+
+-- we follow a rather traditional approach:
+--
+-- (1) texmf.cnf given in TEXMFCNF
+-- (2) texmf.cnf searched in default variable
+--
+-- also we now follow the stupid route: if not set then just assume *one*
+-- cnf file under texmf (i.e. distribution)
+
+resolvers.ownpath     = resolvers.ownpath or nil
+resolvers.ownbin      = resolvers.ownbin  or arg[-2] or arg[-1] or arg[0] or "luatex"
+resolvers.autoselfdir = true -- false may be handy for debugging
+
+function resolvers.getownpath()
+    if not resolvers.ownpath then
+        if resolvers.autoselfdir and os.selfdir then
+            resolvers.ownpath = os.selfdir
+        else
+            local binary = resolvers.ownbin
+            if os.platform == "windows" then
+                binary = file.replacesuffix(binary,"exe")
+            end
+            for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+                local b = file.join(p,binary)
+                if lfs.isfile(b) then
+                    -- we assume that after changing to the path the currentdir function
+                    -- resolves to the real location and use this side effect here; this
+                    -- trick is needed because on the mac installations use symlinks in the
+                    -- path instead of real locations
+                    local olddir = lfs.currentdir()
+                    if lfs.chdir(p) then
+                        local pp = lfs.currentdir()
+                        if trace_verbose and p ~= pp then
+                            logs.report("fileio","following symlink %s to %s",p,pp)
+                        end
+                        resolvers.ownpath = pp
+                        lfs.chdir(olddir)
+                    else
+                        if trace_verbose then
+                            logs.report("fileio","unable to check path %s",p)
+                        end
+                        resolvers.ownpath =  p
+                    end
+                    break
+                end
+            end
+        end
+        if not resolvers.ownpath then resolvers.ownpath = '.' end
+    end
+    return resolvers.ownpath
+end
+
+local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" }
+
+local function identify_own()
+    local ownpath = resolvers.getownpath() or lfs.currentdir()
+    local ie = instance.environment
+    if ownpath then
+        if resolvers.env('SELFAUTOLOC')    == "" then os.env['SELFAUTOLOC']    = file.collapse_path(ownpath) end
+        if resolvers.env('SELFAUTODIR')    == "" then os.env['SELFAUTODIR']    = file.collapse_path(ownpath .. "/..") end
+        if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end
+    else
+        logs.report("fileio","error: unable to locate ownpath")
+        os.exit()
+    end
+    if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end
+    if resolvers.env('TEXOS')    == "" then os.env['TEXOS']    = resolvers.env('SELFAUTODIR') end
+    if resolvers.env('TEXROOT')  == "" then os.env['TEXROOT']  = resolvers.env('SELFAUTOPARENT') end
+    if trace_verbose then
+        for i=1,#own_places do
+            local v = own_places[i]
+            logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown")
+        end
+    end
+    identify_own = function() end
+end
+
+function resolvers.identify_cnf()
+    if #instance.cnffiles == 0 then
+        -- fallback
+        identify_own()
+        -- the real search
+        resolvers.expand_variables()
+        local t = resolvers.split_path(resolvers.env('TEXMFCNF'))
+        t = expanded_path_from_list(t)
+        expand_vars(t) -- redundant
+        local function locate(filename,list)
+            for i=1,#t do
+                local ti = t[i]
+                local texmfcnf = file.collapse_path(file.join(ti,filename))
+                if lfs.isfile(texmfcnf) then
+                    list[#list+1] = texmfcnf
+                end
+            end
+        end
+        locate(resolvers.luaname,instance.luafiles)
+        locate(resolvers.cnfname,instance.cnffiles)
+    end
+end
+
+local function load_cnf_file(fname)
+    fname = resolvers.clean_path(fname)
+    local lname = file.replacesuffix(fname,'lua')
+    local f = io.open(lname)
+    if f then -- this will go
+        f:close()
+        local dname = file.dirname(fname)
+        if not instance.configuration[dname] then
+            resolvers.load_data(dname,'configuration',lname and file.basename(lname))
+            instance.order[#instance.order+1] = instance.configuration[dname]
+        end
+    else
+        f = io.open(fname)
+        if f then
+            if trace_verbose then
+                logs.report("fileio","loading %s", fname)
+            end
+            local line, data, n, k, v
+            local dname = file.dirname(fname)
+            if not instance.configuration[dname] then
+                instance.configuration[dname] = { }
+                instance.order[#instance.order+1] = instance.configuration[dname]
+            end
+            local data = instance.configuration[dname]
+            while true do
+                local line, n = f:read(), 0
+                if line then
+                    while true do -- join lines
+                        line, n = gsub(line,"\\%s*$", "")
+                        if n > 0 then
+                            line = line .. f:read()
+                        else
+                            break
+                        end
+                    end
+                    if not find(line,"^[%%#]") then
+                        local l = gsub(line,"%s*%%.*$","")
+                        local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$")
+                        if k and v and not data[k] then
+                            v = gsub(v,"[%%#].*",'')
+                            data[k] = gsub(v,"~","$HOME")
+                            instance.kpsevars[k] = true
+                        end
+                    end
+                else
+                    break
+                end
+            end
+            f:close()
+        elseif trace_verbose then
+            logs.report("fileio","skipping %s", fname)
+        end
+    end
+end
+
+local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in next, c do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = resolvers.bare_variable(v)
+                end
+            end
+        end
+    end
+end
+
+function resolvers.load_cnf()
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            load_cnf_file(fname)
+        end
+    end
+    -- instance.cnffiles contain complete names now !
+    if #instance.cnffiles == 0 then
+        if trace_verbose then
+            logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)")
+        end
+    else
+        instance.rootpath = instance.cnffiles[1]
+        for k,fname in ipairs(instance.cnffiles) do
+            instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        if instance.diskcache and not instance.renewcache then
+            resolvers.loadoldconfig(instance.cnffiles)
+            if instance.loaderror then
+                loadoldconfigdata()
+                resolvers.saveoldconfig()
+            end
+        else
+            loadoldconfigdata()
+            if instance.renewcache then
+                resolvers.saveoldconfig()
+            end
+        end
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+function resolvers.load_lua()
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        resolvers.loadnewconfig()
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+-- database loading
+
+function resolvers.load_hash()
+    resolvers.locatelists()
+    if instance.diskcache and not instance.renewcache then
+        resolvers.loadfiles()
+        if instance.loaderror then
+            resolvers.loadlists()
+            resolvers.savefiles()
+        end
+    else
+        resolvers.loadlists()
+        if instance.renewcache then
+            resolvers.savefiles()
+        end
+    end
+end
+
+function resolvers.append_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash append: %s",tag)
+    end
+    insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.prepend_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash prepend: %s",tag)
+    end
+    insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash
+--  local t = resolvers.expanded_path_list('TEXMF') -- full expansion
+    local t = resolvers.split_path(resolvers.env('TEXMF'))
+    insert(t,1,specification)
+    local newspec = concat(t,";")
+    if instance.environment["TEXMF"] then
+        instance.environment["TEXMF"] = newspec
+    elseif instance.variables["TEXMF"] then
+        instance.variables["TEXMF"] = newspec
+    else
+        -- weird
+    end
+    resolvers.expand_variables()
+    reset_hashes()
+end
+
+-- locators
+
+function resolvers.locatelists()
+    for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do
+        if trace_verbose then
+            logs.report("fileio","locating list of %s",path)
+        end
+        resolvers.locatedatabase(file.collapse_path(path))
+    end
+end
+
+function resolvers.locatedatabase(specification)
+    return resolvers.methodhandler('locators', specification)
+end
+
+function resolvers.locators.tex(specification)
+    if specification and specification ~= '' and lfs.isdir(specification) then
+        if trace_locating then
+            logs.report("fileio",'! tex locator found: %s',specification)
+        end
+        resolvers.append_hash('file',specification,filename)
+    elseif trace_locating then
+        logs.report("fileio",'? tex locator not found: %s',specification)
+    end
+end
+
+-- hashers
+
+function resolvers.hashdatabase(tag,name)
+    return resolvers.methodhandler('hashers',tag,name)
+end
+
+function resolvers.loadfiles()
+    instance.loaderror = false
+    instance.files = { }
+    if not instance.renewcache then
+        for _, hash in ipairs(instance.hashes) do
+            resolvers.hashdatabase(hash.tag,hash.name)
+            if instance.loaderror then break end
+        end
+    end
+end
+
+function resolvers.hashers.tex(tag,name)
+    resolvers.load_data(tag,'files')
+end
+
+-- generators:
+
+function resolvers.loadlists()
+    for _, hash in ipairs(instance.hashes) do
+        resolvers.generatedatabase(hash.tag)
+    end
+end
+
+function resolvers.generatedatabase(specification)
+    return resolvers.methodhandler('generators', specification)
+end
+
+-- starting with . or .. etc or funny char
+
+local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t"))
+
+function resolvers.generators.tex(specification)
+    local tag = specification
+    if trace_verbose then
+        logs.report("fileio","scanning path %s",specification)
+    end
+    instance.files[tag] = { }
+    local files = instance.files[tag]
+    local n, m, r = 0, 0, 0
+    local spec = specification .. '/'
+    local attributes = lfs.attributes
+    local directory = lfs.dir
+    local function action(path)
+        local full
+        if path then
+            full = spec .. path .. '/'
+        else
+            full = spec
+        end
+        for name in directory(full) do
+            if not weird:match(name) then
+                local mode = attributes(full..name,'mode')
+                if mode == 'file' then
+                    if path then
+                        n = n + 1
+                        local f = files[name]
+                        if f then
+                            if type(f) == 'string' then
+                                files[name] = { f, path }
+                            else
+                                f[#f+1] = path
+                            end
+                        else -- probably unique anyway
+                            files[name] = path
+                            local lower = lower(name)
+                            if name ~= lower then
+                                files["remap:"..lower] = name
+                                r = r + 1
+                            end
+                        end
+                    end
+                elseif mode == 'directory' then
+                    m = m + 1
+                    if path then
+                        action(path..'/'..name)
+                    else
+                        action(name)
+                    end
+                end
+            end
+        end
+    end
+    action()
+    if trace_verbose then
+        logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r)
+    end
+end
+
+-- savers, todo
+
+function resolvers.savefiles()
+    resolvers.save_data('files')
+end
+
+-- A config (optionally) has the paths split in tables. Internally
+-- we join them and split them after the expansion has taken place. This
+-- is more convenient.
+
+function resolvers.splitconfig()
+    for i,c in ipairs(instance) do
+        for k,v in pairs(c) do
+            if type(v) == 'string' then
+                local t = file.split_path(v)
+                if #t > 1 then
+                    c[k] = t
+                end
+            end
+        end
+    end
+end
+
+function resolvers.joinconfig()
+    for i,c in ipairs(instance.order) do
+        for k,v in pairs(c) do -- ipairs?
+            if type(v) == 'table' then
+                c[k] = file.join_path(v)
+            end
+        end
+    end
+end
+function resolvers.split_path(str)
+    if type(str) == 'table' then
+        return str
+    else
+        return file.split_path(str)
+    end
+end
+function resolvers.join_path(str)
+    if type(str) == 'table' then
+        return file.join_path(str)
+    else
+        return str
+    end
+end
+
+function resolvers.splitexpansions()
+    local ie = instance.expansions
+    for k,v in next, ie do
+        local t, h = { }, { }
+        for _,vv in ipairs(file.split_path(v)) do
+            if vv ~= "" and not h[vv] then
+                t[#t+1] = vv
+                h[vv] = true
+            end
+        end
+        if #t > 1 then
+            ie[k] = t
+        else
+            ie[k] = t[1]
+        end
+    end
+end
+
+-- end of split/join code
+
+function resolvers.saveoldconfig()
+    resolvers.splitconfig()
+    resolvers.save_data('configuration')
+    resolvers.joinconfig()
+end
+
+resolvers.configbanner = [[
+-- This is a Luatex configuration file created by 'luatools.lua' or
+-- 'luatex.exe' directly. For comment, suggestions and questions you can
+-- contact the ConTeXt Development Team. This configuration file is
+-- not copyrighted. [HH & TH]
+]]
+
+function resolvers.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local function dump(k,v,m) -- could be moved inline
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+        end
+    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sortedkeys(files)) do -- ipairs
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sortedkeys(fk)) do -- ipairs
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
+            end
+        end
+    else
+        for k, v in next, files do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in next, v do
+                    t[#t+1] = dump(kk,vv,"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,v,"\t")
+            end
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+function resolvers.save_data(dataname, makename) -- untested without cache overload
+    for cachename, files in next, instance[dataname] do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. ".lua", name .. ".luc"
+        if trace_verbose then
+            logs.report("fileio","preparing %s for %s",dataname,cachename)
+        end
+        for k, v in next, files do
+            if type(v) == "table" and #v == 1 then
+                files[k] = v[1]
+            end
+        end
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = resolvers.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local ok = io.savedata(luaname,resolvers.serialize(data))
+        if ok then
+            if trace_verbose then
+                logs.report("fileio","%s saved in %s",dataname,luaname)
+            end
+            if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip
+                if trace_verbose then
+                    logs.report("fileio","%s compiled to %s",dataname,lucname)
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname)
+                end
+                os.remove(lucname)
+            end
+        elseif trace_verbose then
+            logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname)
+        end
+    end
+end
+
+function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua")
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then
+            if trace_verbose then
+                logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = data.content
+        else
+            if trace_verbose then
+                logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
+        end
+    elseif trace_verbose then
+        logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+    end
+end
+
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
+
+function resolvers.resetconfig()
+    identify_own()
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function resolvers.loadnewconfig()
+    for _, cnf in ipairs(instance.luafiles) do
+        local pathname = file.dirname(cnf)
+        local filename = file.join(pathname,resolvers.luaname)
+        local blob = loadfile(filename)
+        if blob then
+            local data = blob()
+            if data then
+                if trace_verbose then
+                    logs.report("fileio","loading configuration file %s",filename)
+                end
+                if true then
+                    -- flatten to variable.progname
+                    local t = { }
+                    for k, v in next, data do -- v = progname
+                        if type(v) == "string" then
+                            t[k] = v
+                        else
+                            for kk, vv in next, v do -- vv = variable
+                                if type(vv) == "string" then
+                                    t[vv.."."..v] = kk
+                                end
+                            end
+                        end
+                    end
+                    instance['setup'][pathname] = t
+                else
+                    instance['setup'][pathname] = data
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","skipping configuration file %s",filename)
+                end
+                instance['setup'][pathname] = { }
+                instance.loaderror = true
+            end
+        elseif trace_verbose then
+            logs.report("fileio","skipping configuration file %s",filename)
+        end
+        instance.order[#instance.order+1] = instance.setup[pathname]
+        if instance.loaderror then break end
+    end
+end
+
+function resolvers.loadoldconfig()
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            resolvers.load_data(dname,'configuration')
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
+        end
+    end
+    resolvers.joinconfig()
+end
+
+function resolvers.expand_variables()
+    local expansions, environment, variables = { }, instance.environment, instance.variables
+    local env = resolvers.env
+    instance.expansions = expansions
+    if instance.engine   ~= "" then environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then environment['progname'] = instance.progname end
+    for k,v in next, environment do
+        local a, b = match(k,"^(%a+)%_(.*)%s*$")
+        if a and b then
+            expansions[a..'.'..b] = v
+        else
+            expansions[k] = v
+        end
+    end
+    for k,v in next, environment do -- move environment to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    for k,v in next, variables do -- move variables to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    local busy = false
+    local function resolve(a)
+        busy = true
+        return expansions[a] or env(a)
+    end
+    while true do
+        busy = false
+        for k,v in next, expansions do
+            local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve)
+            local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve)
+            if n > 0 or m > 0 then
+                expansions[k]= s
+            end
+        end
+        if not busy then break end
+    end
+    for k,v in next, expansions do
+        expansions[k] = gsub(v,"\\", '/')
+    end
+end
+
+function resolvers.variable(name)
+    return entry(instance.variables,name)
+end
+
+function resolvers.expansion(name)
+    return entry(instance.expansions,name)
+end
+
+function resolvers.is_variable(name)
+    return is_entry(instance.variables,name)
+end
+
+function resolvers.is_expansion(name)
+    return is_entry(instance.expansions,name)
+end
+
+function resolvers.unexpanded_path_list(str)
+    local pth = resolvers.variable(str)
+    local lst = resolvers.split_path(pth)
+    return expanded_path_from_list(lst)
+end
+
+function resolvers.unexpanded_path(str)
+    return file.join_path(resolvers.unexpanded_path_list(str))
+end
+
+do -- no longer needed
+
+    local done = { }
+
+    function resolvers.reset_extra_path()
+        local ep = instance.extra_paths
+        if not ep then
+            ep, done = { }, { }
+            instance.extra_paths = ep
+        elseif #ep > 0 then
+            instance.lists, done = { }, { }
+        end
+    end
+
+    function resolvers.register_extra_path(paths,subpaths)
+        local ep = instance.extra_paths or { }
+        local n = #ep
+        if paths and paths ~= "" then
+            if subpaths and subpaths ~= "" then
+                for p in gmatch(paths,"[^,]+") do
+                    -- we gmatch each step again, not that fast, but used seldom
+                    for s in gmatch(subpaths,"[^,]+") do
+                        local ps = p .. "/" .. s
+                        if not done[ps] then
+                            ep[#ep+1] = resolvers.clean_path(ps)
+                            done[ps] = true
+                        end
+                    end
+                end
+            else
+                for p in gmatch(paths,"[^,]+") do
+                    if not done[p] then
+                        ep[#ep+1] = resolvers.clean_path(p)
+                        done[p] = true
+                    end
+                end
+            end
+        elseif subpaths and subpaths ~= "" then
+            for i=1,n do
+                -- we gmatch each step again, not that fast, but used seldom
+                for s in gmatch(subpaths,"[^,]+") do
+                    local ps = ep[i] .. "/" .. s
+                    if not done[ps] then
+                        ep[#ep+1] = resolvers.clean_path(ps)
+                        done[ps] = true
+                    end
+                end
+            end
+        end
+        if #ep > 0 then
+            instance.extra_paths = ep -- register paths
+        end
+        if #ep > n then
+            instance.lists = { } -- erase the cache
+        end
+    end
+
+end
+
+local function made_list(instance,list)
+    local ep = instance.extra_paths
+    if not ep or #ep == 0 then
+        return list
+    else
+        local done, new = { }, { }
+        -- honour . .. ../.. but only when at the start
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                if find(v,"^[%.%/]$") then
+                    done[v] = true
+                    new[#new+1] = v
+                else
+                    break
+                end
+            end
+        end
+        -- first the extra paths
+        for k=1,#ep do
+            local v = ep[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        -- next the formal paths
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        return new
+    end
+end
+
+function resolvers.clean_path_list(str)
+    local t = resolvers.expanded_path_list(str)
+    if t then
+        for i=1,#t do
+            t[i] = file.collapse_path(resolvers.clean_path(t[i]))
+        end
+    end
+    return t
+end
+
+function resolvers.expand_path(str)
+    return file.join_path(resolvers.expanded_path_list(str))
+end
+
+function resolvers.expanded_path_list(str)
+    if not str then
+        return ep or { }
+    elseif instance.savelists then
+        -- engine+progname hash
+        str = gsub(str,"%$","")
+        if not instance.lists[str] then -- cached
+            local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str)))
+            instance.lists[str] = expanded_path_from_list(lst)
+        end
+        return instance.lists[str]
+    else
+        local lst = resolvers.split_path(resolvers.expansion(str))
+        return made_list(instance,expanded_path_from_list(lst))
+    end
+end
+
+function resolvers.expanded_path_list_from_var(str) -- brrr
+    local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$",""))
+    if tmp ~= "" then
+        return resolvers.expanded_path_list(str)
+    else
+        return resolvers.expanded_path_list(tmp)
+    end
+end
+
+function resolvers.expand_path_from_var(str)
+    return file.join_path(resolvers.expanded_path_list_from_var(str))
+end
+
+function resolvers.format_of_var(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+function resolvers.format_of_suffix(str)
+    return suffixmap[file.extname(str)] or 'tex'
+end
+
+function resolvers.variable_of_format(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+
+function resolvers.var_of_format_or_suffix(str)
+    local v = formats[str]
+    if v then
+        return v
+    end
+    v = formats[alternatives[str]]
+    if v then
+        return v
+    end
+    v = suffixmap[file.extname(str)]
+    if v then
+        return formats[isf]
+    end
+    return ''
+end
+
+function resolvers.expand_braces(str) -- output variable and brace expansion of STRING
+    local ori = resolvers.variable(str)
+    local pth = expanded_path_from_list(resolvers.split_path(ori))
+    return file.join_path(pth)
+end
+
+resolvers.isreadable = { }
+
+function resolvers.isreadable.file(name)
+    local readable = lfs.isfile(name) -- brrr
+    if trace_detail then
+        if readable then
+            logs.report("fileio","+ readable: %s",name)
+        else
+            logs.report("fileio","- readable: %s", name)
+        end
+    end
+    return readable
+end
+
+resolvers.isreadable.tex = resolvers.isreadable.file
+
+-- name
+-- name/name
+
+local function collect_files(names)
+    local filelist = { }
+    for k=1,#names do
+        local fname = names[k]
+        if trace_detail then
+            logs.report("fileio","? blobpath asked: %s",fname)
+        end
+        local bname = file.basename(fname)
+        local dname = file.dirname(fname)
+        if dname == "" or find(dname,"^%.") then
+            dname = false
+        else
+            dname = "/" .. dname .. "$"
+        end
+        local hashes = instance.hashes
+        for h=1,#hashes do
+            local hash = hashes[h]
+            local blobpath = hash.tag
+            local files = blobpath and instance.files[blobpath]
+            if files then
+                if trace_detail then
+                    logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname)
+                end
+                local blobfile = files[bname]
+                if not blobfile then
+                    local rname = "remap:"..bname
+                    blobfile = files[rname]
+                    if blobfile then
+                        bname = files[rname]
+                        blobfile = files[bname]
+                    end
+                end
+                if blobfile then
+                    if type(blobfile) == 'string' then
+                        if not dname or find(blobfile,dname) then
+                            filelist[#filelist+1] = {
+                                hash.type,
+                                file.join(blobpath,blobfile,bname), -- search
+                                resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result
+                            }
+                        end
+                    else
+                        for kk=1,#blobfile do
+                            local vv = blobfile[kk]
+                            if not dname or find(vv,dname) then
+                                filelist[#filelist+1] = {
+                                    hash.type,
+                                    file.join(blobpath,vv,bname), -- search
+                                    resolvers.concatinators[hash.type](blobpath,vv,bname) -- result
+                                }
+                            end
+                        end
+                    end
+                end
+            elseif trace_locating then
+                logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname)
+            end
+        end
+    end
+    if #filelist > 0 then
+        return filelist
+    else
+        return nil
+    end
+end
+
+function resolvers.suffix_of_format(str)
+    if suffixes[str] then
+        return suffixes[str][1]
+    else
+        return ""
+    end
+end
+
+function resolvers.suffixes_of_format(str)
+    if suffixes[str] then
+        return suffixes[str]
+    else
+        return {}
+    end
+end
+
+function resolvers.register_in_trees(name)
+    if not find(name,"^%.") then
+        instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one
+    end
+end
+
+-- split the next one up for readability (bu this module needs a cleanup anyway)
+
+local function can_be_dir(name) -- can become local
+    local fakepaths = instance.fakepaths
+    if not fakepaths[name] then
+        if lfs.isdir(name) then
+            fakepaths[name] = 1 -- directory
+        else
+            fakepaths[name] = 2 -- no directory
+        end
+    end
+    return (fakepaths[name] == 1)
+end
+
+local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc)
+    local result = collected or { }
+    local stamp  = nil
+    filename = file.collapse_path(filename)  -- elsewhere
+    filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere
+    -- speed up / beware: format problem
+    if instance.remember then
+        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format
+        if instance.found[stamp] then
+            if trace_locating then
+                logs.report("fileio",'! remembered: %s',filename)
+            end
+            return instance.found[stamp]
+        end
+    end
+    if not dangerous[instance.format or "?"] then
+        if resolvers.isreadable.file(filename) then
+            if trace_detail then
+                logs.report("fileio",'= found directly: %s',filename)
+            end
+            instance.found[stamp] = { filename }
+            return { filename }
+        end
+    end
+    if find(filename,'%*') then
+        if trace_locating then
+            logs.report("fileio",'! wildcard: %s', filename)
+        end
+        result = resolvers.find_wildcard_files(filename)
+    elseif file.is_qualified_path(filename) then
+        if resolvers.isreadable.file(filename) then
+            if trace_locating then
+                logs.report("fileio",'! qualified: %s', filename)
+            end
+            result = { filename }
+        else
+            local forcedname, ok, suffix = "", false, file.extname(filename)
+            if suffix == "" then -- why
+                if instance.format == "" then
+                    forcedname = filename .. ".tex"
+                    if resolvers.isreadable.file(forcedname) then
+                        if trace_locating then
+                            logs.report("fileio",'! no suffix, forcing standard filetype: tex')
+                        end
+                        result, ok = { forcedname }, true
+                    end
+                else
+                    local suffixes = resolvers.suffixes_of_format(instance.format)
+                    for _, s in next, suffixes do
+                        forcedname = filename .. "." .. s
+                        if resolvers.isreadable.file(forcedname) then
+                            if trace_locating then
+                                logs.report("fileio",'! no suffix, forcing format filetype: %s', s)
+                            end
+                            result, ok = { forcedname }, true
+                            break
+                        end
+                    end
+                end
+            end
+            if not ok and suffix ~= "" then
+                -- try to find in tree (no suffix manipulation), here we search for the
+                -- matching last part of the name
+                local basename = file.basename(filename)
+                local pattern = (filename .. "$"):gsub("([%.%-])","%%%1")
+                local savedformat = instance.format
+                local format = savedformat or ""
+                if format == "" then
+                    instance.format = resolvers.format_of_suffix(suffix)
+                end
+                if not format then
+                    instance.format = "othertextfiles" -- kind of everything, maybe texinput is better
+                end
+                --
+                local resolved = collect_instance_files(basename)
+                if #result == 0 then
+                    local lowered = lower(basename)
+                    if filename ~= lowered then
+                        resolved = collect_instance_files(lowered)
+                    end
+                end
+                resolvers.format = savedformat
+                --
+                for r=1,#resolved do
+                    local rr = resolved[r]
+                    if rr:find(pattern) then
+                        result[#result+1], ok = rr, true
+                    end
+                end
+                -- a real wildcard:
+                --
+                -- if not ok then
+                --     local filelist = collect_files({basename})
+                --     for f=1,#filelist do
+                --         local ff = filelist[f][3] or ""
+                --         if ff:find(pattern) then
+                --             result[#result+1], ok = ff, true
+                --         end
+                --     end
+                -- end
+            end
+            if not ok and trace_locating then
+                logs.report("fileio",'? qualified: %s', filename)
+            end
+        end
+    else
+        -- search spec
+        local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename)
+        if ext == "" then
+            if not instance.force_suffixes then
+                wantedfiles[#wantedfiles+1] = filename
+            end
+        else
+            wantedfiles[#wantedfiles+1] = filename
+        end
+        if instance.format == "" then
+            if ext == "" then
+                local forcedname = filename .. '.tex'
+                wantedfiles[#wantedfiles+1] = forcedname
+                filetype = resolvers.format_of_suffix(forcedname)
+                if trace_locating then
+                    logs.report("fileio",'! forcing filetype: %s',filetype)
+                end
+            else
+                filetype = resolvers.format_of_suffix(filename)
+                if trace_locating then
+                    logs.report("fileio",'! using suffix based filetype: %s',filetype)
+                end
+            end
+        else
+            if ext == "" then
+                local suffixes = resolvers.suffixes_of_format(instance.format)
+                for _, s in next, suffixes do
+                    wantedfiles[#wantedfiles+1] = filename .. "." .. s
+                end
+            end
+            filetype = instance.format
+            if trace_locating then
+                logs.report("fileio",'! using given filetype: %s',filetype)
+            end
+        end
+        local typespec = resolvers.variable_of_format(filetype)
+        local pathlist = resolvers.expanded_path_list(typespec)
+        if not pathlist or #pathlist == 0 then
+            -- no pathlist, access check only / todo == wildcard
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+                logs.report("fileio",'? filetype: %s',filetype or '?')
+                logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | "))
+            end
+            for k=1,#wantedfiles do
+                local fname = wantedfiles[k]
+                if fname and resolvers.isreadable.file(fname) then
+                    filename, done = fname, true
+                    result[#result+1] = file.join('.',fname)
+                    break
+                end
+            end
+            -- this is actually 'other text files' or 'any' or 'whatever'
+            local filelist = collect_files(wantedfiles)
+            local fl = filelist and filelist[1]
+            if fl then
+                filename = fl[3]
+                result[#result+1] = filename
+                done = true
+            end
+        else
+            -- list search
+            local filelist = collect_files(wantedfiles)
+            local doscan, recurse
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+            end
+            -- a bit messy ... esp the doscan setting here
+            for k=1,#pathlist do
+                local path = pathlist[k]
+                if find(path,"^!!") then doscan  = false else doscan  = true  end
+                if find(path,"//$") then recurse = true  else recurse = false end
+                local pathname = gsub(path,"^!+", '')
+                done = false
+                -- using file list
+                if filelist and not (done and not instance.allresults) and recurse then
+                    -- compare list entries with permitted pattern
+                    pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences
+                    pathname = gsub(pathname,"/+$", '/.*')      -- later usage of pathname
+                    pathname = gsub(pathname,"//", '/.-/')      -- not ok for /// but harmless
+                    local expr = "^" .. pathname
+                    for k=1,#filelist do
+                        local fl = filelist[k]
+                        local f = fl[2]
+                        if find(f,expr) then
+                            if trace_detail then
+                                logs.report("fileio",'= found in hash: %s',f)
+                            end
+                            --- todo, test for readable
+                            result[#result+1] = fl[3]
+                            resolvers.register_in_trees(f) -- for tracing used files
+                            done = true
+                            if not instance.allresults then break end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- check if on disk / unchecked / does not work at all / also zips
+                    if resolvers.splitmethod(pathname).scheme == 'file' then -- ?
+                        local pname = gsub(pathname,"%.%*$",'')
+                        if not find(pname,"%*") then
+                            local ppname = gsub(pname,"/+$","")
+                            if can_be_dir(ppname) then
+                                for k=1,#wantedfiles do
+                                    local w = wantedfiles[k]
+                                    local fname = file.join(ppname,w)
+                                    if resolvers.isreadable.file(fname) then
+                                        if trace_detail then
+                                            logs.report("fileio",'= found by scanning: %s',fname)
+                                        end
+                                        result[#result+1] = fname
+                                        done = true
+                                        if not instance.allresults then break end
+                                    end
+                                end
+                            else
+                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
+                            end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- todo: slow path scanning
+                end
+                if done and not instance.allresults then break end
+            end
+        end
+    end
+    for k=1,#result do
+        result[k] = file.collapse_path(result[k])
+    end
+    if instance.remember then
+        instance.found[stamp] = result
+    end
+    return result
+end
+
+if not resolvers.concatinators  then resolvers.concatinators = { } end
+
+resolvers.concatinators.tex  = file.join
+resolvers.concatinators.file = resolvers.concatinators.tex
+
+function resolvers.find_files(filename,filetype,mustexist)
+    if type(mustexist) == boolean then
+        -- all set
+    elseif type(filetype) == 'boolean' then
+        filetype, mustexist = nil, false
+    elseif type(filetype) ~= 'string' then
+        filetype, mustexist = nil, false
+    end
+    instance.format = filetype or ''
+    local result = collect_instance_files(filename)
+    if #result == 0 then
+        local lowered = lower(filename)
+        if filename ~= lowered then
+            return collect_instance_files(lowered)
+        end
+    end
+    instance.format = ''
+    return result
+end
+
+function resolvers.find_file(filename,filetype,mustexist)
+    return (resolvers.find_files(filename,filetype,mustexist)[1] or "")
+end
+
+function resolvers.find_given_files(filename)
+    local bname, result = file.basename(filename), { }
+    local hashes = instance.hashes
+    for k=1,#hashes do
+        local hash = hashes[k]
+        local files = instance.files[hash.tag]
+        local blist = files[bname]
+        if not blist then
+            local rname = "remap:"..bname
+            blist = files[rname]
+            if blist then
+                bname = files[rname]
+                blist = files[bname]
+            end
+        end
+        if blist then
+            if type(blist) == 'string' then
+                result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or ""
+                if not instance.allresults then break end
+            else
+                for kk=1,#blist do
+                    local vv = blist[kk]
+                    result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or ""
+                    if not instance.allresults then break end
+                end
+            end
+        end
+    end
+    return result
+end
+
+function resolvers.find_given_file(filename)
+    return (resolvers.find_given_files(filename)[1] or "")
+end
+
+local function doit(path,blist,bname,tag,kind,result,allresults)
+    local done = false
+    if blist and kind then
+        if type(blist) == 'string' then
+            -- make function and share code
+            if find(lower(blist),path) then
+                result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or ""
+                done = true
+            end
+        else
+            for kk=1,#blist do
+                local vv = blist[kk]
+                if find(lower(vv),path) then
+                    result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or ""
+                    done = true
+                    if not allresults then break end
+                end
+            end
+        end
+    end
+    return done
+end
+
+function resolvers.find_wildcard_files(filename) -- todo: remap:
+    local result = { }
+    local bname, dname = file.basename(filename), file.dirname(filename)
+    local path = gsub(dname,"^*/","")
+    path = gsub(path,"*",".*")
+    path = gsub(path,"-","%%-")
+    if dname == "" then
+        path = ".*"
+    end
+    local name = bname
+    name = gsub(name,"*",".*")
+    name = gsub(name,"-","%%-")
+    path = lower(path)
+    name = lower(name)
+    local files, allresults, done = instance.files, instance.allresults, false
+    if find(name,"%*") then
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            for kk, hh in next, files[hash.tag] do
+                if not find(kk,"^remap:") then
+                    if find(lower(kk),name) then
+                        if doit(path,hh,kk,tag,kind,result,allresults) then done = true end
+                        if done and not allresults then break end
+                    end
+                end
+            end
+        end
+    else
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end
+            if done and not allresults then break end
+        end
+    end
+    -- we can consider also searching the paths not in the database, but then
+    -- we end up with a messy search (all // in all path specs)
+    return result
+end
+
+function resolvers.find_wildcard_file(filename)
+    return (resolvers.find_wildcard_files(filename)[1] or "")
+end
+
+-- main user functions
+
+function resolvers.automount()
+    -- implemented later
+end
+
+function resolvers.load(option)
+    statistics.starttiming(instance)
+    resolvers.resetconfig()
+    resolvers.identify_cnf()
+    resolvers.load_lua()
+    resolvers.expand_variables()
+    resolvers.load_cnf()
+    resolvers.expand_variables()
+    if option ~= "nofiles" then
+        resolvers.load_hash()
+        resolvers.automount()
+    end
+    statistics.stoptiming(instance)
+end
+
+function resolvers.for_files(command, files, filetype, mustexist)
+    if files and #files > 0 then
+        local function report(str)
+            if trace_verbose then
+                logs.report("fileio",str) -- has already verbose
+            else
+                print(str)
+            end
+        end
+        if trace_verbose then
+            report('')
+        end
+        for _, file in ipairs(files) do
+            local result = command(file,filetype,mustexist)
+            if type(result) == 'string' then
+                report(result)
+            else
+                for _,v in ipairs(result) do
+                    report(v)
+                end
+            end
+        end
+    end
+end
+
+-- strtab
+
+resolvers.var_value  = resolvers.variable   -- output the value of variable $STRING.
+resolvers.expand_var = resolvers.expansion  -- output variable expansion of STRING.
+
+function resolvers.show_path(str)     -- output search path for file type NAME
+    return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str)))
+end
+
+-- resolvers.find_file(filename)
+-- resolvers.find_file(filename, filetype, mustexist)
+-- resolvers.find_file(filename, mustexist)
+-- resolvers.find_file(filename, filetype)
+
+function resolvers.register_file(files, name, path)
+    if files[name] then
+        if type(files[name]) == 'string' then
+            files[name] = { files[name], path }
+        else
+            files[name] = path
+        end
+    else
+        files[name] = path
+    end
+end
+
+function resolvers.splitmethod(filename)
+    if not filename then
+        return { } -- safeguard
+    elseif type(filename) == "table" then
+        return filename -- already split
+    elseif not find(filename,"://") then
+        return { scheme="file", path = filename, original=filename } -- quick hack
+    else
+        return url.hashed(filename)
+    end
+end
+
+function table.sequenced(t,sep) -- temp here
+    local s = { }
+    for k, v in pairs(t) do -- pairs?
+        s[#s+1] = k .. "=" .. v
+    end
+    return concat(s, sep or " | ")
+end
+
+function resolvers.methodhandler(what, filename, filetype) -- ...
+    local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb
+    local scheme = specification.scheme
+    if resolvers[what][scheme] then
+        if trace_locating then
+            logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification))
+        end
+        return resolvers[what][scheme](filename,filetype) -- todo: specification
+    else
+        return resolvers[what].tex(filename,filetype) -- todo: specification
+    end
+end
+
+function resolvers.clean_path(str)
+    if str then
+        str = gsub(str,"\\","/")
+        str = gsub(str,"^!+","")
+        str = gsub(str,"^~",resolvers.homedir)
+        return str
+    else
+        return nil
+    end
+end
+
+function resolvers.do_with_path(name,func)
+    for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs?
+        func("^"..resolvers.clean_path(v))
+    end
+end
+
+function resolvers.do_with_var(name,func)
+    func(expanded_var(name))
+end
+
+function resolvers.with_files(pattern,handle)
+    for _, hash in ipairs(instance.hashes) do
+        local blobpath = hash.tag
+        local blobtype = hash.type
+        if blobpath then
+            local files = instance.files[blobpath]
+            if files then
+                for k,v in next, files do
+                    if find(k,"^remap:") then
+                        k = files[k]
+                        v = files[k] -- chained
+                    end
+                    if find(k,pattern) then
+                        if type(v) == "string" then
+                            handle(blobtype,blobpath,v,k)
+                        else
+                            for _,vv in pairs(v) do -- ipairs?
+                                handle(blobtype,blobpath,vv,k)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+function resolvers.locate_format(name)
+    local barename, fmtname = name:gsub("%.%a+$",""), ""
+    if resolvers.usecache then
+        local path = file.join(caches.setpath("formats")) -- maybe platform
+        fmtname = file.join(path,barename..".fmt") or ""
+    end
+    if fmtname == "" then
+        fmtname = resolvers.find_files(barename..".fmt")[1] or ""
+    end
+    fmtname = resolvers.clean_path(fmtname)
+    if fmtname ~= "" then
+        local barename = file.removesuffix(fmtname)
+        local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui"
+        if lfs.isfile(luiname) then
+            return barename, luiname
+        elseif lfs.isfile(lucname) then
+            return barename, lucname
+        elseif lfs.isfile(luaname) then
+            return barename, luaname
+        end
+    end
+    return nil, nil
+end
+
+function resolvers.boolean_variable(str,default)
+    local b = resolvers.expansion(str)
+    if b == "" then
+        return default
+    else
+        b = toboolean(b)
+        return (b == nil and default) or b
+    end
+end
+
+texconfig.kpse_init = false
+
+kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } )
+
+-- for a while
+
+input = resolvers
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This module deals with caching data. It sets up the paths and
+implements loaders and savers for tables. Best is to set the
+following variable. When not set, the usual paths will be
+checked. Personally I prefer the (users) temporary path.

+
+
+TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
+
+
+
Currently we do no locking when we write files. This is no real
+problem because most caching involves fonts and the chance of them
+being written at the same time is small. We also need to extend
+luatools with a recache feature.

+--ldx]]--
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache = false  trackers.register("resolvers.cache", function(v) trace_cache = v end)
+
+caches = caches or { }
+
+caches.path     = caches.path or nil
+caches.base     = caches.base or "luatex-cache"
+caches.more     = caches.more or "context"
+caches.direct   = false -- true is faster but may need huge amounts of memory
+caches.tree     = false
+caches.paths    = caches.paths or nil
+caches.force    = false
+caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" }
+
+function caches.cleanname(name)
+    return (gsub(lower(name),"[^%w%d]+","-"))
+end
+
+function caches.temp()
+    local cachepath = nil
+    local function check(list,isenv)
+        if not cachepath then
+            for k=1,#list do
+                local v = list[k]
+                cachepath = (isenv and (os.env[v] or "")) or v or ""
+                if cachepath == "" then
+                    -- next
+                else
+                    cachepath = resolvers.clean_path(cachepath)
+                    if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory"
+                        break
+                    elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
+                        dir.mkdirs(cachepath)
+                        if lfs.isdir(cachepath) and file.iswritable(cachepath) then
+                            break
+                        end
+                    end
+                end
+                cachepath = nil
+            end
+        end
+    end
+    check(resolvers.clean_path_list("TEXMFCACHE") or { })
+    check(caches.defaults,true)
+    if not cachepath then
+        print("\nfatal error: there is no valid (writable) cache path defined\n")
+        os.exit()
+    elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory"
+        print(format("\nfatal error: cache path %s is not a directory\n",cachepath))
+        os.exit()
+    end
+    cachepath = file.collapse_path(cachepath)
+    function caches.temp()
+        return cachepath
+    end
+    return cachepath
+end
+
+function caches.configpath()
+    return table.concat(resolvers.instance.cnffiles,";")
+end
+
+function caches.hashed(tree)
+    return md5.hex(gsub(lower(tree),"[\\\/]+","/"))
+end
+
+function caches.treehash()
+    local tree = caches.configpath()
+    if not tree or tree == "" then
+        return false
+    else
+        return caches.hashed(tree)
+    end
+end
+
+function caches.setpath(...)
+    if not caches.path then
+        if not caches.path then
+            caches.path = caches.temp()
+        end
+        caches.path = resolvers.clean_path(caches.path) -- to be sure
+        caches.tree = caches.tree or caches.treehash()
+        if caches.tree then
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
+        else
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
+        end
+    end
+    if not caches.path then
+        caches.path = '.'
+    end
+    caches.path = resolvers.clean_path(caches.path)
+    if not table.is_empty({...}) then
+        local pth = dir.mkdirs(caches.path,...)
+        return pth
+    end
+    caches.path = dir.expand_name(caches.path)
+    return caches.path
+end
+
+function caches.definepath(category,subcategory)
+    return function()
+        return caches.setpath(category,subcategory)
+    end
+end
+
+function caches.setluanames(path,name)
+    return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
+end
+
+function caches.loaddata(path,name)
+    local tmaname, tmcname = caches.setluanames(path,name)
+    local loader = loadfile(tmcname) or loadfile(tmaname)
+    if loader then
+        return loader()
+    else
+        return false
+    end
+end
+
+--~ function caches.loaddata(path,name)
+--~     local tmaname, tmcname = caches.setluanames(path,name)
+--~     return dofile(tmcname) or dofile(tmaname)
+--~ end
+
+function caches.iswritable(filepath,filename)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    return file.iswritable(tmaname)
+end
+
+function caches.savedata(filepath,filename,data,raw)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    local reduce, simplify = true, true
+    if raw then
+        reduce, simplify = false, false
+    end
+    if caches.direct then
+        file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex
+    else
+        table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true
+    end
+    local cleanup = resolvers.boolean_variable("PURGECACHE", false)
+    local strip = resolvers.boolean_variable("LUACSTRIP", true)
+    utils.lua.compile(tmaname, tmcname, cleanup, strip)
+end
+
+-- here we use the cache for format loading (texconfig.[formatname|jobname])
+
+--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then
+if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then
+    if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc
+    texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt")
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+resolvers.finders = resolvers.finders or { }
+resolvers.openers = resolvers.openers or { }
+resolvers.loaders = resolvers.loaders or { }
+
+resolvers.finders.notfound  = { nil }
+resolvers.openers.notfound  = { nil }
+resolvers.loaders.notfound  = { false, nil, 0 }
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-out'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+outputs = outputs or { }
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-con'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache      = false  trackers.register("resolvers.cache",      function(v) trace_cache      = v end)
+local trace_containers = false  trackers.register("resolvers.containers", function(v) trace_containers = v end)
+local trace_storage    = false  trackers.register("resolvers.storage",    function(v) trace_storage    = v end)
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+--[[ldx--
+
Once we found ourselves defining similar cache constructs
+several times, containers were introduced. Containers are used
+to collect tables in memory and reuse them when possible based
+on (unique) hashes (to be provided by the calling function).

+
+
Caching to disk is disabled by default. Version numbers are
+stored in the saved table which makes it possible to change the
+table structures without bothering about the disk cache.

+
+
Examples of usage can be found in the font related code.

+--ldx]]--
+
+containers = containers or { }
+
+containers.usecache = true
+
+local function report(container,tag,name)
+    if trace_cache or trace_containers then
+        logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid')
+    end
+end
+
+local allocated = { }
+
+-- tracing
+
+function containers.define(category, subcategory, version, enabled)
+    return function()
+        if category and subcategory then
+            local c = allocated[category]
+            if not c then
+                c  = { }
+                allocated[category] = c
+            end
+            local s = c[subcategory]
+            if not s then
+                s = {
+                    category = category,
+                    subcategory = subcategory,
+                    storage = { },
+                    enabled = enabled,
+                    version = version or 1.000,
+                    trace = false,
+                    path = caches and caches.setpath(category,subcategory),
+                }
+                c[subcategory] = s
+            end
+            return s
+        else
+            return nil
+        end
+    end
+end
+
+function containers.is_usable(container, name)
+    return container.enabled and caches and caches.iswritable(container.path, name)
+end
+
+function containers.is_valid(container, name)
+    if name and name ~= "" then
+        local storage = container.storage[name]
+        return storage and not table.is_empty(storage) and storage.cache_version == container.version
+    else
+        return false
+    end
+end
+
+function containers.read(container,name)
+    if container.enabled and caches and not container.storage[name] and containers.usecache then
+        container.storage[name] = caches.loaddata(container.path,name)
+        if containers.is_valid(container,name) then
+            report(container,"loaded",name)
+        else
+            container.storage[name] = nil
+        end
+    end
+    if container.storage[name] then
+        report(container,"reusing",name)
+    end
+    return container.storage[name]
+end
+
+function containers.write(container, name, data)
+    if data then
+        data.cache_version = container.version
+        if container.enabled and caches then
+            local unique, shared = data.unique, data.shared
+            data.unique, data.shared = nil, nil
+            caches.savedata(container.path, name, data)
+            report(container,"saved",name)
+            data.unique, data.shared = unique, shared
+        end
+        report(container,"stored",name)
+        container.storage[name] = data
+    end
+    return data
+end
+
+function containers.content(container,name)
+    return container.storage[name]
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-use'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+-- since we want to use the cache instead of the tree, we will now
+-- reimplement the saver.
+
+local save_data = resolvers.save_data
+local load_data = resolvers.load_data
+
+resolvers.cachepath = nil  -- public, for tracing
+resolvers.usecache  = true -- public, for tracing
+
+function resolvers.save_data(dataname)
+    save_data(dataname, function(cachename,dataname)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(cachename))
+        else
+            return file.join(cachename,dataname)
+        end
+    end)
+end
+
+function resolvers.load_data(pathname,dataname,filename)
+    load_data(pathname,dataname,filename,function(dataname,filename)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(pathname))
+        else
+            if not filename or (filename == "") then
+                filename = dataname
+            end
+            return file.join(pathname,filename)
+        end
+    end)
+end
+
+-- we will make a better format, maybe something xml or just text or lua
+
+resolvers.automounted = resolvers.automounted or { }
+
+function resolvers.automount(usecache)
+    local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT'))
+    if table.is_empty(mountpaths) and usecache then
+        mountpaths = { caches.setpath("mount") }
+    end
+    if not table.is_empty(mountpaths) then
+        statistics.starttiming(resolvers.instance)
+        for k, root in pairs(mountpaths) do
+            local f = io.open(root.."/url.tmi")
+            if f then
+                for line in f:lines() do
+                    if line then
+                        if line:find("^[%%#%-]") then -- or %W
+                            -- skip
+                        elseif line:find("^zip://") then
+                            if trace_locating then
+                                logs.report("fileio","mounting %s",line)
+                            end
+                            table.insert(resolvers.automounted,line)
+                            resolvers.usezipfile(line)
+                        end
+                    end
+                end
+                f:close()
+            end
+        end
+        statistics.stoptiming(resolvers.instance)
+    end
+end
+
+-- status info
+
+statistics.register("used config path", function() return caches.configpath()  end)
+statistics.register("used cache path",  function() return caches.temp() or "?" end)
+
+-- experiment (code will move)
+
+function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname
+    local enginebanner = status.list().banner
+    if formatbanner and enginebanner and sourcefile then
+        local luvname = file.replacesuffix(texname,"luv")
+        local luvdata = {
+            enginebanner = enginebanner,
+            formatbanner = formatbanner,
+            sourcehash   = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"),
+            sourcefile   = sourcefile,
+        }
+        io.savedata(luvname,table.serialize(luvdata,true))
+    end
+end
+
+function statistics.check_fmt_status(texname)
+    local enginebanner = status.list().banner
+    if enginebanner and texname then
+        local luvname = file.replacesuffix(texname,"luv")
+        if lfs.isfile(luvname) then
+            local luv = dofile(luvname)
+            if luv and luv.sourcefile then
+                local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown")
+                if luv.enginebanner and luv.enginebanner ~= enginebanner then
+                    return "engine mismatch"
+                end
+                if luv.sourcehash and luv.sourcehash ~= sourcehash then
+                    return "source mismatch"
+                end
+            else
+                return "invalid status file"
+            end
+        else
+            return "missing status file"
+        end
+    end
+    return true
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-kps'] = {
+    version   = 1.001,
+    comment   = "companion to luatools.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This file is used when we want the input handlers to behave like
+kpsewhich. What to do with the following:

+
+
+{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}
+$SELFAUTOLOC    : /usr/tex/bin/platform
+$SELFAUTODIR    : /usr/tex/bin
+$SELFAUTOPARENT : /usr/tex
+
+
+
How about just forgetting about them?

+--ldx]]--
+
+local suffixes = resolvers.suffixes
+local formats  = resolvers.formats
+
+suffixes['gf']                       = { 'gf' }
+suffixes['pk']                       = { 'pk' }
+suffixes['base']                     = { 'base' }
+suffixes['bib']                      = { 'bib' }
+suffixes['bst']                      = { 'bst' }
+suffixes['cnf']                      = { 'cnf' }
+suffixes['mem']                      = { 'mem' }
+suffixes['mf']                       = { 'mf' }
+suffixes['mfpool']                   = { 'pool' }
+suffixes['mft']                      = { 'mft' }
+suffixes['mppool']                   = { 'pool' }
+suffixes['graphic/figure']           = { 'eps', 'epsi' }
+suffixes['texpool']                  = { 'pool' }
+suffixes['PostScript header']        = { 'pro' }
+suffixes['ist']                      = { 'ist' }
+suffixes['web']                      = { 'web', 'ch' }
+suffixes['cweb']                     = { 'w', 'web', 'ch' }
+suffixes['cmap files']               = { 'cmap' }
+suffixes['lig files']                = { 'lig' }
+suffixes['bitmap font']              = { }
+suffixes['MetaPost support']         = { }
+suffixes['TeX system documentation'] = { }
+suffixes['TeX system sources']       = { }
+suffixes['dvips config']             = { }
+suffixes['type42 fonts']             = { }
+suffixes['web2c files']              = { }
+suffixes['other text files']         = { }
+suffixes['other binary files']       = { }
+suffixes['opentype fonts']           = { 'otf' }
+
+suffixes['fmt']                      = { 'fmt' }
+suffixes['texmfscripts']             = { 'rb','lua','py','pl' }
+
+suffixes['pdftex config']            = { }
+suffixes['Troff fonts']              = { }
+
+suffixes['ls-R']                     = { }
+
+--[[ldx--
+
If you wondered abou tsome of the previous mappings, how about
+the next bunch:

+--ldx]]--
+
+formats['bib']                      = ''
+formats['bst']                      = ''
+formats['mft']                      = ''
+formats['ist']                      = ''
+formats['web']                      = ''
+formats['cweb']                     = ''
+formats['MetaPost support']         = ''
+formats['TeX system documentation'] = ''
+formats['TeX system sources']       = ''
+formats['Troff fonts']              = ''
+formats['dvips config']             = ''
+formats['graphic/figure']           = ''
+formats['ls-R']                     = ''
+formats['other text files']         = ''
+formats['other binary files']       = ''
+
+formats['gf']                       = ''
+formats['pk']                       = ''
+formats['base']                     = 'MFBASES'
+formats['cnf']                      = ''
+formats['mem']                      = 'MPMEMS'
+formats['mf']                       = 'MFINPUTS'
+formats['mfpool']                   = 'MFPOOL'
+formats['mppool']                   = 'MPPOOL'
+formats['texpool']                  = 'TEXPOOL'
+formats['PostScript header']        = 'TEXPSHEADERS'
+formats['cmap files']               = 'CMAPFONTS'
+formats['type42 fonts']             = 'T42FONTS'
+formats['web2c files']              = 'WEB2C'
+formats['pdftex config']            = 'PDFTEXCONFIG'
+formats['texmfscripts']             = 'TEXMFSCRIPTS'
+formats['bitmap font']              = ''
+formats['lig files']                = 'LIGFONTS'
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-aux'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+local trace_verbose = false  trackers.register("resolvers.verbose", function(v) trace_verbose = v end)
+
+function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix
+    local scriptpath = "scripts/context/lua"
+    newname = file.addsuffix(newname,"lua")
+    local oldscript = resolvers.clean_path(oldname)
+    if trace_verbose then
+        logs.report("fileio","to be replaced old script %s", oldscript)
+    end
+    local newscripts = resolvers.find_files(newname) or { }
+    if #newscripts == 0 then
+        if trace_verbose then
+            logs.report("fileio","unable to locate new script")
+        end
+    else
+        for i=1,#newscripts do
+            local newscript = resolvers.clean_path(newscripts[i])
+            if trace_verbose then
+                logs.report("fileio","checking new script %s", newscript)
+            end
+            if oldscript == newscript then
+                if trace_verbose then
+                    logs.report("fileio","old and new script are the same")
+                end
+            elseif not find(newscript,scriptpath) then
+                if trace_verbose then
+                    logs.report("fileio","new script should come from %s",scriptpath)
+                end
+            elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then
+                if trace_verbose then
+                    logs.report("fileio","invalid new script name")
+                end
+            else
+                local newdata = io.loaddata(newscript)
+                if newdata then
+                    if trace_verbose then
+                        logs.report("fileio","old script content replaced by new content")
+                    end
+                    io.savedata(oldscript,newdata)
+                    break
+                elseif trace_verbose then
+                    logs.report("fileio","unable to load new script")
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-lst'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- used in mtxrun
+
+local find, concat, upper, format = string.find, table.concat, string.upper, string.format
+
+resolvers.listers = resolvers.listers or { }
+
+local function tabstr(str)
+    if type(str) == 'table' then
+        return concat(str," | ")
+    else
+        return str
+    end
+end
+
+local function list(list,report)
+    local instance = resolvers.instance
+    local pat = upper(pattern or "","")
+    local report = report or texio.write_nl
+    for _,key in pairs(table.sortedkeys(list)) do
+        if instance.pattern == "" or find(upper(key),pat) then
+            if instance.kpseonly then
+                if instance.kpsevars[key] then
+                    report(format("%s=%s",key,tabstr(list[key])))
+                end
+            else
+                report(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key])))
+            end
+        end
+    end
+end
+
+function resolvers.listers.variables () list(resolvers.instance.variables ) end
+function resolvers.listers.expansions() list(resolvers.instance.expansions) end
+
+function resolvers.listers.configurations(report)
+    local report = report or texio.write_nl
+    local instance = resolvers.instance
+    for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do
+        if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then
+            report(format("%s\n",key))
+            for i,c in ipairs(instance.order) do
+                local str = c[key]
+                if str then
+                    report(format("\t%s\t%s",i,str))
+                end
+            end
+            report("")
+        end
+    end
+end
+
+
+end -- of closure
+-- end library merge
+
+-- We initialize some characteristics of this program. We need to
+-- do this before we load the libraries, else own.name will not be
+-- properly set (handy for selfcleaning the file). It's an ugly
+-- looking piece of code.
+
+own = { }
+
+own.libs = { -- todo: check which ones are really needed
+    'l-string.lua',
+    'l-lpeg.lua',
+    'l-table.lua',
+    'l-io.lua',
+    'l-number.lua',
+    'l-set.lua',
+    'l-os.lua',
+    'l-file.lua',
+    'l-md5.lua',
+    'l-url.lua',
+    'l-dir.lua',
+    'l-boolean.lua',
+    'l-unicode.lua',
+    'l-math.lua',
+    'l-utils.lua',
+    'trac-tra.lua',
+    'luat-env.lua',
+    'trac-inf.lua',
+    'trac-log.lua',
+    'data-res.lua',
+    'data-tmp.lua',
+--  'data-pre.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-tex.lua',
+--  'data-bin.lua',
+--  'data-zip.lua',
+--  'data-crl.lua',
+--  'data-lua.lua',
+    'data-kps.lua', -- so that we can replace kpsewhich
+    'data-aux.lua', -- updater
+    'data-lst.lua', -- lister
+}
+
+-- We need this hack till luatex is fixed.
+
+if arg and arg[0] == 'luatex' and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+-- End of hack.
+
+own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua'
+own.path = string.match(own.name,"^(.+)[\\/].-$") or "."
+own.list = { '.' }
+
+if own.path ~= '.' then
+    table.insert(own.list,own.path)
+end
+
+table.insert(own.list,own.path.."/../../../tex/context/base")
+table.insert(own.list,own.path.."/mtx")
+table.insert(own.list,own.path.."/../sources")
+
+function locate_libs()
+    for _, lib in pairs(own.libs) do
+        for _, pth in pairs(own.list) do
+            local filename = string.gsub(pth .. "/" .. lib,"\\","/")
+            local codeblob = loadfile(filename)
+            if codeblob then
+                codeblob()
+                own.list = { pth } -- speed up te search
+                break
+            end
+        end
+    end
+end
+
+if not resolvers then
+    locate_libs()
+end
+
+if not resolvers then
+    print("")
+    print("Luatools is unable to start up due to lack of libraries. You may")
+    print("try to run 'lua luatools.lua --selfmerge' in the path where this")
+    print("script is located (normally under ..../scripts/context/lua) which")
+    print("will make luatools library independent.")
+    os.exit()
+end
+
+logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false)
+
+local instance = resolvers.reset()
+
+resolvers.defaultlibs = { -- not all are needed
+    'l-string.lua',
+    'l-lpeg.lua',
+    'l-table.lua',
+    'l-boolean.lua',
+    'l-number.lua',
+    'l-unicode.lua',
+    'l-os.lua',
+    'l-io.lua',
+    'l-file.lua',
+    'l-md5.lua',
+    'l-url.lua',
+    'l-dir.lua',
+    'l-utils.lua',
+    'l-dimen.lua',
+    'trac-inf.lua',
+    'trac-tra.lua',
+    'trac-log.lua',
+    'luat-env.lua', -- here ?
+    'data-res.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-tmp.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-pre.lua',
+    'data-tex.lua',
+    'data-bin.lua',
+--  'data-zip.lua',
+--  'data-clr.lua',
+    'data-lua.lua',
+    'data-ctx.lua',
+    'luat-fio.lua',
+    'luat-cnf.lua',
+}
+
+instance.engine     =     environment.arguments["engine"]   or 'luatex'
+instance.progname   =     environment.arguments["progname"] or 'context'
+instance.luaname    =     environment.arguments["luafile"]  or "" -- environment.ownname or ""
+instance.lualibs    =     environment.arguments["lualibs"]  or table.concat(resolvers.defaultlibs,",")
+instance.allresults =     environment.arguments["all"]      or false
+instance.pattern    =     environment.arguments["pattern"]  or nil
+instance.sortdata   =     environment.arguments["sort"]     or false
+instance.kpseonly   = not environment.arguments["all"]      or false
+instance.my_format  =     environment.arguments["format"]   or instance.format
+
+if type(instance.pattern) == 'boolean' then
+    logs.simple("invalid pattern specification")
+    instance.pattern = nil
+end
+
+if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end
+
+runners  = runners  or { }
+messages = messages or { }
+
+messages.no_ini_file = [[
+There is no lua initialization file found. This file can be forced by the
+"--progname" directive, or specified with "--luaname", or it is derived
+automatically from the formatname (aka jobname). It may be that you have
+to regenerate the file database using "luatools --generate".
+]]
+
+messages.help = [[
+--generate        generate file database
+--variables       show configuration variables
+--expansions      show expanded variables
+--configurations  show configuration order
+--expand-braces   expand complex variable
+--expand-path     expand variable (resolve paths)
+--expand-var      expand variable (resolve references)
+--show-path       show path expansion of ...
+--var-value       report value of variable
+--find-file       report file location
+--find-path       report path of file
+--make or --ini   make luatex format
+--run or --fmt=   run luatex format
+--luafile=str     lua inifile (default is .lua)
+--lualibs=list    libraries to assemble (optional when --compile)
+--compile         assemble and compile lua inifile
+--verbose         give a bit more info
+--all             show all found files
+--sort            sort cached data
+--engine=str      target engine
+--progname=str    format or backend
+--pattern=str     filter variables
+]]
+
+function runners.make_format(texname)
+    local instance = resolvers.instance
+    if texname and texname ~= "" then
+        if resolvers.usecache then
+            local path = file.join(caches.setpath("formats")) -- maybe platform
+            if path and lfs then
+                lfs.chdir(path)
+            end
+        end
+        local barename = texname:gsub("%.%a+$","")
+        if barename == texname then
+            texname = texname .. ".tex"
+        end
+        local fullname = resolvers.find_files(texname)[1] or ""
+        if fullname == "" then
+            logs.simple("no tex file with name: %s",texname)
+        else
+            local luaname, lucname, luapath, lualibs = "", "", "", { }
+            -- the following is optional, since context.lua can also
+            -- handle this collect and compile business
+            if environment.arguments["compile"] then
+                if luaname == "" then luaname = barename end
+                logs.simple("creating initialization file: %s",luaname)
+                luapath = file.dirname(luaname)
+                if luapath == "" then
+                    luapath = file.dirname(texname)
+                end
+                if luapath == "" then
+                    luapath = file.dirname(resolvers.find_files(texname)[1] or "")
+                end
+                lualibs = string.split(instance.lualibs,",")
+                luaname = file.basename(barename .. ".lua")
+                lucname = file.basename(barename .. ".luc")
+                -- todo: when this fails, we can just copy the merged libraries from
+                -- luatools since they are normally the same, at least for context
+                if lualibs[1] then
+                    local firstlib = file.join(luapath,lualibs[1])
+                    if not lfs.isfile(firstlib) then
+                        local foundname = resolvers.find_files(lualibs[1])[1]
+                        if foundname then
+                            logs.simple("located library path: %s",luapath)
+                            luapath = file.dirname(foundname)
+                        end
+                    end
+                end
+                logs.simple("using library path: %s",luapath)
+                logs.simple("using lua libraries: %s",table.join(lualibs," "))
+                utils.merger.selfcreate(lualibs,luapath,luaname)
+                local strip = resolvers.boolean_variable("LUACSTRIP", true)
+                if utils.lua.compile(luaname,lucname,false,strip) and io.exists(lucname) then
+                    luaname = lucname
+                    logs.simple("using compiled initialization file: %s",lucname)
+                else
+                    logs.simple("using uncompiled initialization file: %s",luaname)
+                end
+            else
+                for _, v in pairs({instance.luaname, instance.progname, barename}) do
+                    v = string.gsub(v..".lua","%.lua%.lua$",".lua")
+                    if v and (v ~= "") then
+                        luaname = resolvers.find_files(v)[1] or ""
+                        if luaname ~= "" then
+                            break
+                        end
+                    end
+                end
+            end
+            if environment.arguments["noluc"] then
+                luaname = luaname:gsub("%.luc$",".lua") -- make this an option
+            end
+            if luaname == "" then
+                if logs.verbose then
+                    logs.simplelines(messages.no_ini_file)
+                    logs.simple("texname : %s",texname)
+                    logs.simple("luaname : %s",instance.luaname)
+                    logs.simple("progname: %s",instance.progname)
+                    logs.simple("barename: %s",barename)
+                end
+            else
+                logs.simple("using lua initialization file: %s",luaname)
+                local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem")
+                if mp and #mp > 0 then
+                    for _, name in ipairs(mp) do
+                        logs.simple("removing related mplib format %s", file.basename(name))
+                        os.remove(name)
+                    end
+                end
+                local flags = {
+                    "--ini",
+                    "--lua=" .. string.quote(luaname)
+                }
+                local bs = (os.platform == "unix" and "\\\\") or "\\" -- todo: make a function
+                local command = "luatex ".. table.concat(flags," ")  .. " " .. string.quote(fullname) .. " " .. bs .. "dump"
+                logs.simple("running command: %s\n",command)
+                os.spawn(command)
+                -- todo: do a dummy run that generates the related metafun and mfplain formats
+            end
+        end
+    else
+        logs.simple("no tex file given")
+    end
+end
+
+function runners.run_format(name,data,more)
+ -- hm, rather old code here; we can now use the file.whatever functions
+    if name and (name ~= "") then
+        local barename = name:gsub("%.%a+$","")
+        local fmtname = ""
+        if resolvers.usecache then
+            local path = file.join(caches.setpath("formats")) -- maybe platform
+            fmtname = file.join(path,barename..".fmt") or ""
+        end
+        if fmtname == "" then
+            fmtname = resolvers.find_files(barename..".fmt")[1] or ""
+        end
+        fmtname = resolvers.clean_path(fmtname)
+        barename = fmtname:gsub("%.%a+$","")
+        if fmtname == "" then
+            logs.simple("no format with name: %s",name)
+        else
+            local luaname = barename .. ".luc"
+            local f = io.open(luaname)
+            if not f then
+                luaname = barename .. ".lua"
+                f = io.open(luaname)
+            end
+            if f then
+                f:close()
+                local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. (more ~= "" and string.quote(more) or "")
+                logs.simple("running command: %s",command)
+                os.spawn(command)
+            else
+                logs.simple("using format name: %s",fmtname)
+                logs.simple("no luc/lua with name: %s",barename)
+            end
+        end
+    end
+end
+
+local ok = true
+
+-- private option --noluc for testing errors in the stub
+
+if environment.arguments["find-file"] then
+    resolvers.load()
+    instance.format  = environment.arguments["format"] or instance.format
+    if instance.pattern then
+        instance.allresults = true
+        resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format)
+    else
+        resolvers.for_files(resolvers.find_files, environment.files, instance.my_format)
+    end
+elseif environment.arguments["find-path"] then
+    resolvers.load()
+    local path = resolvers.find_file(environment.files[1], instance.my_format)
+    if logs.verbose then
+        logs.simple(file.dirname(path))
+    else
+        print(file.dirname(path))
+    end
+elseif environment.arguments["run"] then
+    resolvers.load("nofiles") -- ! no need for loading databases
+    logs.setverbose(true)
+    runners.run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "")
+elseif environment.arguments["fmt"] then
+    resolvers.load("nofiles") -- ! no need for loading databases
+    logs.setverbose(true)
+    runners.run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "")
+elseif environment.arguments["expand-braces"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.expand_braces, environment.files)
+elseif environment.arguments["expand-path"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.expand_path, environment.files)
+elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.expand_var, environment.files)
+elseif environment.arguments["show-path"] or environment.arguments["path-value"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.show_path, environment.files)
+elseif environment.arguments["var-value"] or environment.arguments["show-value"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.var_value, environment.files)
+elseif environment.arguments["format-path"] then
+    resolvers.load()
+    logs.simple(caches.setpath("format"))
+elseif instance.pattern then -- brrr
+    resolvers.load()
+    instance.format = environment.arguments["format"] or instance.format
+    instance.allresults = true
+    resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format)
+elseif environment.arguments["generate"] then
+    instance.renewcache = true
+    logs.setverbose(true)
+    resolvers.load()
+elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then
+    resolvers.load()
+    logs.setverbose(true)
+    runners.make_format(environment.files[1] or "")
+elseif environment.arguments["selfmerge"] then
+    utils.merger.selfmerge(own.name,own.libs,own.list)
+elseif environment.arguments["selfclean"] then
+    utils.merger.selfclean(own.name)
+elseif environment.arguments["selfupdate"] then
+    resolvers.load()
+    logs.setverbose(true)
+    resolvers.update_script(own.name,"luatools")
+elseif environment.arguments["variables"] or environment.arguments["show-variables"] then
+    resolvers.load("nofiles")
+    resolvers.listers.variables()
+elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then
+    resolvers.load("nofiles")
+    resolvers.listers.expansions()
+elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then
+    resolvers.load("nofiles")
+    resolvers.listers.configurations()
+elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then
+    logs.help(messages.help)
+else
+    resolvers.load()
+    resolvers.for_files(resolvers.find_files, environment.files, instance.my_format)
+end
+
+if logs.verbose then
+    logs.simpleline()
+    logs.simple("runtime: %0.3f seconds",os.runtime())
+end
+
+if os.platform == "unix" then
+    io.write("\n")
+end
diff --git a/scripts/context/stubs/mswin/makempy.bat b/scripts/context/stubs/mswin/makempy.bat
index e339058c6..03eaa8a28 100755
--- a/scripts/context/stubs/mswin/makempy.bat
+++ b/scripts/context/stubs/mswin/makempy.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart makempy.pl %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute makempy.pl %*
+endlocal
diff --git a/scripts/context/stubs/mswin/metatex.cmd b/scripts/context/stubs/mswin/metatex.cmd
new file mode 100644
index 000000000..858f28f8f
--- /dev/null
+++ b/scripts/context/stubs/mswin/metatex.cmd
@@ -0,0 +1,5 @@
+@echo off
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --script metatex %*
+endlocal
diff --git a/scripts/context/stubs/mswin/mpstools.bat b/scripts/context/stubs/mswin/mpstools.bat
index df1732e17..8bd25674c 100755
--- a/scripts/context/stubs/mswin/mpstools.bat
+++ b/scripts/context/stubs/mswin/mpstools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart mpstools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute mpstools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/mptopdf.bat b/scripts/context/stubs/mswin/mptopdf.bat
index 242854337..f29881763 100755
--- a/scripts/context/stubs/mswin/mptopdf.bat
+++ b/scripts/context/stubs/mswin/mptopdf.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart mptopdf.pl %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute mptopdf.pl %*
+endlocal
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
new file mode 100644
index 000000000..0af429bf1
--- /dev/null
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -0,0 +1,10190 @@
+#!/usr/bin/env texlua
+
+if not modules then modules = { } end modules ['mtxrun'] = {
+    version   = 1.001,
+    comment   = "runner, lua replacement for texmfstart.rb",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+
+-- one can make a stub:
+--
+-- #!/bin/sh
+-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly mtxrun.lua "$@"
+
+-- filename : mtxrun.lua
+-- comment  : companion to context.tex
+-- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
+-- copyright: PRAGMA ADE / ConTeXt Development Team
+-- license  : see context related readme files
+
+-- This script is based on texmfstart.rb but does not use kpsewhich to
+-- locate files. Although kpse is a library it never came to opening up
+-- its interface to other programs (esp scripting languages) and so we
+-- do it ourselves. The lua variant evolved out of an experimental ruby
+-- one. Interesting is that using a scripting language instead of c does
+-- not have a speed penalty. Actually the lua variant is more efficient,
+-- especially when multiple calls to kpsewhich are involved. The lua
+-- library also gives way more control.
+
+-- to be done / considered
+--
+-- support for --exec or make it default
+-- support for jar files (or maybe not, never used, too messy)
+-- support for $RUBYINPUTS cum suis (if still needed)
+-- remember for subruns: _CTX_K_V_#{original}_
+-- remember for subruns: _CTX_K_S_#{original}_
+-- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb]
+
+texlua = true
+
+-- begin library merge
+
+
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-string'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep
+
+if not string.split then
+
+    -- this will be overloaded by a faster lpeg variant
+
+    function string:split(pattern)
+        if #self > 0 then
+            local t = { }
+            for s in gmatch(self..pattern,"(.-)"..pattern) do
+                t[#t+1] = s
+            end
+            return t
+        else
+            return { }
+        end
+    end
+
+end
+
+local chr_to_esc = {
+    ["%"] = "%%",
+    ["."] = "%.",
+    ["+"] = "%+", ["-"] = "%-", ["*"] = "%*",
+    ["^"] = "%^", ["$"] = "%$",
+    ["["] = "%[", ["]"] = "%]",
+    ["("] = "%(", [")"] = "%)",
+    ["{"] = "%{", ["}"] = "%}"
+}
+
+string.chr_to_esc = chr_to_esc
+
+function string:esc() -- variant 2
+    return (gsub(self,"(.)",chr_to_esc))
+end
+
+function string:unquote()
+    return (gsub(self,"^([\"\'])(.*)%1$","%2"))
+end
+
+function string:quote() -- we could use format("%q")
+    return '"' .. self:unquote() .. '"'
+end
+
+function string:count(pattern) -- variant 3
+    local n = 0
+    for _ in gmatch(self,pattern) do
+        n = n + 1
+    end
+    return n
+end
+
+function string:limit(n,sentinel)
+    if #self > n then
+        sentinel = sentinel or " ..."
+        return sub(self,1,(n-#sentinel)) .. sentinel
+    else
+        return self
+    end
+end
+
+function string:strip()
+    return (gsub(self,"^%s*(.-)%s*$", "%1"))
+end
+
+function string:is_empty()
+    return not find(find,"%S")
+end
+
+function string:enhance(pattern,action)
+    local ok, n = true, 0
+    while ok do
+        ok = false
+        self = gsub(self,pattern, function(...)
+            ok, n = true, n + 1
+            return action(...)
+        end)
+    end
+    return self, n
+end
+
+local chr_to_hex, hex_to_chr = { }, { }
+
+for i=0,255 do
+    local c, h = char(i), format("%02X",i)
+    chr_to_hex[c], hex_to_chr[h] = h, c
+end
+
+function string:to_hex()
+    return (gsub(self or "","(.)",chr_to_hex))
+end
+
+function string:from_hex()
+    return (gsub(self or "","(..)",hex_to_chr))
+end
+
+if not string.characters then
+
+    local function nextchar(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, str:sub(index,index)
+    end
+    function string:characters()
+        return nextchar, self, 0
+    end
+    local function nextbyte(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, byte(str:sub(index,index))
+    end
+    function string:bytes()
+        return nextbyte, self, 0
+    end
+
+end
+
+-- we can use format for this (neg n)
+
+function string:rpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self .. self.rep(chr or " ",m)
+    else
+        return self
+    end
+end
+
+function string:lpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self.rep(chr or " ",m) .. self
+    else
+        return self
+    end
+end
+
+string.padd = string.rpadd
+
+function is_number(str) -- tonumber
+    return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1
+end
+
+--~ print(is_number("1"))
+--~ print(is_number("1.1"))
+--~ print(is_number(".1"))
+--~ print(is_number("-0.1"))
+--~ print(is_number("+0.1"))
+--~ print(is_number("-.1"))
+--~ print(is_number("+.1"))
+
+function string:split_settings() -- no {} handling, see l-aux for lpeg variant
+    if find(self,"=") then
+        local t = { }
+        for k,v in gmatch(self,"(%a+)=([^%,]*)") do
+            t[k] = v
+        end
+        return t
+    else
+        return nil
+    end
+end
+
+local patterns_escapes = {
+    ["-"] = "%-",
+    ["."] = "%.",
+    ["+"] = "%+",
+    ["*"] = "%*",
+    ["%"] = "%%",
+    ["("] = "%)",
+    [")"] = "%)",
+    ["["] = "%[",
+    ["]"] = "%]",
+}
+
+function string:pattesc()
+    return (gsub(self,".",patterns_escapes))
+end
+
+function string:tohash()
+    local t = { }
+    for s in gmatch(self,"([^, ]+)") do -- lpeg
+        t[s] = true
+    end
+    return t
+end
+
+local pattern = lpeg.Ct(lpeg.C(1)^0)
+
+function string:totable()
+    return pattern:match(self)
+end
+
+--~ for _, str in ipairs {
+--~     "1234567123456712345671234567",
+--~     "a\tb\tc",
+--~     "aa\tbb\tcc",
+--~     "aaa\tbbb\tccc",
+--~     "aaaa\tbbbb\tcccc",
+--~     "aaaaa\tbbbbb\tccccc",
+--~     "aaaaaa\tbbbbbb\tcccccc",
+--~ } do print(string.tabtospace(str)) end
+
+function string.tabtospace(str,tab)
+    -- we don't handle embedded newlines
+    while true do
+        local s = find(str,"\t")
+        if s then
+            if not tab then tab = 7 end -- only when found
+            local d = tab-(s-1)%tab
+            if d > 0 then
+                str = gsub(str,"\t",rep(" ",d),1)
+            else
+                str = gsub(str,"\t","",1)
+            end
+        else
+            break
+        end
+    end
+    return str
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-lpeg'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
+
+--~ l-lpeg.lua :
+
+--~ lpeg.digit         = lpeg.R('09')^1
+--~ lpeg.sign          = lpeg.S('+-')^1
+--~ lpeg.cardinal      = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.integer       = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.float         = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1)
+--~ lpeg.number        = lpeg.float + lpeg.integer
+--~ lpeg.oct           = lpeg.P("0") * lpeg.R('07')^1
+--~ lpeg.hex           = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1
+--~ lpeg.uppercase     = lpeg.P("AZ")
+--~ lpeg.lowercase     = lpeg.P("az")
+
+--~ lpeg.eol           = lpeg.S('\r\n\f')^1 -- includes formfeed
+--~ lpeg.space         = lpeg.S(' ')^1
+--~ lpeg.nonspace      = lpeg.P(1-lpeg.space)^1
+--~ lpeg.whitespace    = lpeg.S(' \r\n\f\t')^1
+--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1
+
+local hash = { }
+
+function lpeg.anywhere(pattern) --slightly adapted from website
+    return P { P(pattern) + 1 * lpeg.V(1) }
+end
+
+function lpeg.startswith(pattern) --slightly adapted
+    return P(pattern)
+end
+
+function lpeg.splitter(pattern, action)
+    return (((1-P(pattern))^1)/action+1)^0
+end
+
+-- variant:
+
+--~ local parser = lpeg.Ct(lpeg.splitat(newline))
+
+local crlf     = P("\r\n")
+local cr       = P("\r")
+local lf       = P("\n")
+local space    = S(" \t\f\v")  -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local newline  = crlf + cr + lf
+local spacing  = space^0 * newline
+
+local empty    = spacing * Cc("")
+local nonempty = Cs((1-spacing)^1) * spacing^-1
+local content  = (empty + nonempty)^1
+
+local capture = Ct(content^0)
+
+function string:splitlines()
+    return capture:match(self)
+end
+
+lpeg.linebyline = content -- better make a sublibrary
+
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps->what->more"))  -- oeps what more
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps->what->more"))  -- oeps what->more
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps"))              -- oeps
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps"))              -- oeps
+
+local splitters_s, splitters_m = { }, { }
+
+local function splitat(separator,single)
+    local splitter = (single and splitters_s[separator]) or splitters_m[separator]
+    if not splitter then
+        separator = P(separator)
+        if single then
+            local other, any = C((1 - separator)^0), P(1)
+            splitter = other * (separator * C(any^0) + "")
+            splitters_s[separator] = splitter
+        else
+            local other = C((1 - separator)^0)
+            splitter = other * (separator * other)^0
+            splitters_m[separator] = splitter
+        end
+    end
+    return splitter
+end
+
+lpeg.splitat = splitat
+
+local cache = { }
+
+function string:split(separator)
+    local c = cache[separator]
+    if not c then
+        c = Ct(splitat(separator))
+        cache[separator] = c
+    end
+    return c:match(self)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-table'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+table.join = table.concat
+
+local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove
+local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump
+local getmetatable, setmetatable = getmetatable, setmetatable
+local type, next, tostring, ipairs = type, next, tostring, ipairs
+
+function table.strip(tab)
+    local lst = { }
+    for i=1,#tab do
+        local s = gsub(tab[i],"^%s*(.-)%s*$","%1")
+        if s == "" then
+            -- skip this one
+        else
+            lst[#lst+1] = s
+        end
+    end
+    return lst
+end
+
+local function sortedkeys(tab)
+    local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+        if kind == 3 then
+            -- no further check
+        else
+            local tkey = type(key)
+            if tkey == "string" then
+            --  if kind == 2 then kind = 3 else kind = 1 end
+                kind = (kind == 2 and 3) or 1
+            elseif tkey == "number" then
+            --  if kind == 1 then kind = 3 else kind = 2 end
+                kind = (kind == 1 and 3) or 2
+            else
+                kind = 3
+            end
+        end
+    end
+    if kind == 0 or kind == 3 then
+        sort(srt,function(a,b) return (tostring(a) < tostring(b)) end)
+    else
+        sort(srt)
+    end
+    return srt
+end
+
+local function sortedhashkeys(tab) -- fast one
+    local srt = { }
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+    end
+    sort(srt)
+    return srt
+end
+
+table.sortedkeys     = sortedkeys
+table.sortedhashkeys = sortedhashkeys
+
+function table.sortedpairs(t)
+    local s = sortedhashkeys(t) -- maybe just sortedkeys
+    local n = 0
+    local function kv(s)
+        n = n + 1
+        local k = s[n]
+        return k, t[k]
+    end
+    return kv, s
+end
+
+function table.append(t, list)
+    for _,v in next, list do
+        insert(t,v)
+    end
+end
+
+function table.prepend(t, list)
+    for k,v in next, list do
+        insert(t,k,v)
+    end
+end
+
+function table.merge(t, ...) -- first one is target
+    t = t or {}
+    local lst = {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            t[k] = v
+        end
+    end
+    return t
+end
+
+function table.merged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            tmp[k] = v
+        end
+    end
+    return tmp
+end
+
+function table.imerge(t, ...)
+    local lst = {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            t[#t+1] = nst[j]
+        end
+    end
+    return t
+end
+
+function table.imerged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            tmp[#tmp+1] = nst[j]
+        end
+    end
+    return tmp
+end
+
+local function fastcopy(old) -- fast one
+    if old then
+        local new = { }
+        for k,v in next, old do
+            if type(v) == "table" then
+                new[k] = fastcopy(v) -- was just table.copy
+            else
+                new[k] = v
+            end
+        end
+        -- optional second arg
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
+        end
+        return new
+    else
+        return { }
+    end
+end
+
+local function copy(t, tables) -- taken from lua wiki, slightly adapted
+    tables = tables or { }
+    local tcopy = {}
+    if not tables[t] then
+        tables[t] = tcopy
+    end
+    for i,v in next, t do -- brrr, what happens with sparse indexed
+        if type(i) == "table" then
+            if tables[i] then
+                i = tables[i]
+            else
+                i = copy(i, tables)
+            end
+        end
+        if type(v) ~= "table" then
+            tcopy[i] = v
+        elseif tables[v] then
+            tcopy[i] = tables[v]
+        else
+            tcopy[i] = copy(v, tables)
+        end
+    end
+    local mt = getmetatable(t)
+    if mt then
+        setmetatable(tcopy,mt)
+    end
+    return tcopy
+end
+
+table.fastcopy = fastcopy
+table.copy     = copy
+
+-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack)
+
+function table.sub(t,i,j)
+    return { unpack(t,i,j) }
+end
+
+function table.replace(a,b)
+    for k,v in next, b do
+        a[k] = v
+    end
+end
+
+-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice)
+
+function table.is_empty(t)
+    return not t or not next(t)
+end
+
+function table.one_entry(t)
+    local n = next(t)
+    return n and not next(t,n)
+end
+
+function table.starts_at(t)
+    return ipairs(t,1)(t,0)
+end
+
+function table.tohash(t,value)
+    local h = { }
+    if t then
+        if value == nil then value = true end
+        for _, v in next, t do -- no ipairs here
+            h[v] = value
+        end
+    end
+    return h
+end
+
+function table.fromhash(t)
+    local h = { }
+    for k, v in next, t do -- no ipairs here
+        if v then h[#h+1] = k end
+    end
+    return h
+end
+
+--~ print(table.serialize(t), "\n")
+--~ print(table.serialize(t,"name"), "\n")
+--~ print(table.serialize(t,false), "\n")
+--~ print(table.serialize(t,true), "\n")
+--~ print(table.serialize(t,"name",true), "\n")
+--~ print(table.serialize(t,"name",true,true), "\n")
+
+table.serialize_functions = true
+table.serialize_compact   = true
+table.serialize_inline    = true
+
+local noquotes, hexify, handle, reduce, compact, inline, functions
+
+local reserved = table.tohash { -- intercept a language flaw, no reserved words as key
+    'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
+    'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
+}
+
+local function simple_table(t)
+    if #t > 0 then
+        local n = 0
+        for _,v in next, t do
+            n = n + 1
+        end
+        if n == #t then
+            local tt = { }
+            for i=1,#t do
+                local v = t[i]
+                local tv = type(v)
+                if tv == "number" then
+                    if hexify then
+                        tt[#tt+1] = format("0x%04X",v)
+                    else
+                        tt[#tt+1] = tostring(v) -- tostring not needed
+                    end
+                elseif tv == "boolean" then
+                    tt[#tt+1] = tostring(v)
+                elseif tv == "string" then
+                    tt[#tt+1] = format("%q",v)
+                else
+                    tt = nil
+                    break
+                end
+            end
+            return tt
+        end
+    end
+    return nil
+end
+
+-- Because this is a core function of mkiv I moved some function calls
+-- inline.
+--
+-- twice as fast in a test:
+--
+-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) )
+
+local function do_serialize(root,name,depth,level,indexed)
+    if level > 0 then
+        depth = depth .. " "
+        if indexed then
+            handle(format("%s{",depth))
+        elseif name then
+        --~ handle(format("%s%s={",depth,key(name)))
+            if type(name) == "number" then -- or find(k,"^%d+$") then
+                if hexify then
+                    handle(format("%s[0x%04X]={",depth,name))
+                else
+                    handle(format("%s[%s]={",depth,name))
+                end
+            elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then
+                handle(format("%s%s={",depth,name))
+            else
+                handle(format("%s[%q]={",depth,name))
+            end
+        else
+            handle(format("%s{",depth))
+        end
+    end
+    if root and next(root) then
+        local first, last = nil, 0 -- #root cannot be trusted here
+        if compact then
+            -- NOT: for k=1,#root do (we need to quit at nil)
+            for k,v in ipairs(root) do -- can we use next?
+                if not first then first = k end
+                last = last + 1
+            end
+        end
+        local sk = sortedkeys(root)
+        for i=1,#sk do
+            local k = sk[i]
+            local v = root[k]
+            --~ if v == root then
+                -- circular
+            --~ else
+            local t = type(v)
+            if compact and first and type(k) == "number" and k >= first and k <= last then
+                if t == "number" then
+                    if hexify then
+                        handle(format("%s 0x%04X,",depth,v))
+                    else
+                        handle(format("%s %s,",depth,v))
+                    end
+                elseif t == "string" then
+                    if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                        handle(format("%s %s,",depth,v))
+                    else
+                        handle(format("%s %q,",depth,v))
+                    end
+                elseif t == "table" then
+                    if not next(v) then
+                        handle(format("%s {},",depth))
+                    elseif inline then -- and #t > 0
+                        local st = simple_table(v)
+                        if st then
+                            handle(format("%s { %s },",depth,concat(st,", ")))
+                        else
+                            do_serialize(v,k,depth,level+1,true)
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1,true)
+                    end
+                elseif t == "boolean" then
+                    handle(format("%s %s,",depth,tostring(v)))
+                elseif t == "function" then
+                    if functions then
+                        handle(format('%s loadstring(%q),',depth,dump(v)))
+                    else
+                        handle(format('%s "function",',depth))
+                    end
+                else
+                    handle(format("%s %q,",depth,tostring(v)))
+                end
+            elseif k == "__p__" then -- parent
+                if false then
+                    handle(format("%s __p__=nil,",depth))
+                end
+            elseif t == "number" then
+            --~ if hexify then
+            --~     handle(format("%s %s=0x%04X,",depth,key(k),v))
+            --~ else
+            --~     handle(format("%s %s=%s,",depth,key(k),v))
+            --~ end
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,v))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    if hexify then
+                        handle(format("%s %s=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s %s=%s,",depth,k,v))
+                    end
+                else
+                    if hexify then
+                        handle(format("%s [%q]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                end
+            elseif t == "string" then
+                if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                --~ handle(format("%s %s=%s,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%s,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%s,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%s,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                else
+                --~ handle(format("%s %s=%q,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%q,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%q,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%q,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%q,",depth,k,v))
+                    end
+                end
+            elseif t == "table" then
+                if not next(v) then
+                    --~ handle(format("%s %s={},",depth,key(k)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]={},",depth,k))
+                        else
+                            handle(format("%s [%s]={},",depth,k))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s={},",depth,k))
+                    else
+                        handle(format("%s [%q]={},",depth,k))
+                    end
+                elseif inline then
+                    local st = simple_table(v)
+                    if st then
+                    --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", ")))
+                        if type(k) == "number" then -- or find(k,"^%d+$") then
+                            if hexify then
+                                handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", ")))
+                            else
+                                handle(format("%s [%s]={ %s },",depth,k,concat(st,", ")))
+                            end
+                        elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                            handle(format("%s %s={ %s },",depth,k,concat(st,", ")))
+                        else
+                            handle(format("%s [%q]={ %s },",depth,k,concat(st,", ")))
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1)
+                    end
+                else
+                    do_serialize(v,k,depth,level+1)
+                end
+            elseif t == "boolean" then
+            --~ handle(format("%s %s=%s,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%s,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%s,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%s,",depth,k,tostring(v)))
+                end
+            elseif t == "function" then
+                if functions then
+                    --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v)))
+                        else
+                            handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v)))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=loadstring(%q),",depth,k,dump(v)))
+                    else
+                        handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v)))
+                    end
+                end
+            else
+                --~ handle(format("%s %s=%q,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%q,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%q,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%q,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%q,",depth,k,tostring(v)))
+                end
+            end
+            --~ end
+        end
+    end
+   if level > 0 then
+        handle(format("%s},",depth))
+    end
+end
+
+-- replacing handle by a direct t[#t+1] = ... (plus test) is not much
+-- faster (0.03 on 1.00 for zapfino.tma)
+
+local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
+    noquotes = _noquotes
+    hexify = _hexify
+    handle = _handle or print
+    reduce = _reduce or false
+    compact = table.serialize_compact
+    inline  = compact and table.serialize_inline
+    functions = table.serialize_functions
+    local tname = type(name)
+    if tname == "string" then
+        if name == "return" then
+            handle("return {")
+        else
+            handle(name .. "={")
+        end
+    elseif tname == "number" then
+        if hexify then
+            handle(format("[0x%04X]={",name))
+        else
+            handle("[" .. name .. "]={")
+        end
+    elseif tname == "boolean" then
+        if name then
+            handle("return {")
+        else
+            handle("{")
+        end
+    else
+        handle("t={")
+    end
+    if root and next(root) then
+        do_serialize(root,name,"",0,indexed)
+    end
+    handle("}")
+end
+
+--~ name:
+--~
+--~ true     : return     { }
+--~ false    :            { }
+--~ nil      : t        = { }
+--~ string   : string   = { }
+--~ 'return' : return     { }
+--~ number   : [number] = { }
+
+function table.serialize(root,name,reduce,noquotes,hexify)
+    local t = { }
+    local function flush(s)
+        t[#t+1] = s
+    end
+    serialize(root,name,flush,reduce,noquotes,hexify)
+    return concat(t,"\n")
+end
+
+function table.tohandle(handle,root,name,reduce,noquotes,hexify)
+    serialize(root,name,handle,reduce,noquotes,hexify)
+end
+
+-- sometimes tables are real use (zapfino extra pro is some 85M) in which
+-- case a stepwise serialization is nice; actually, we could consider:
+--
+-- for line in table.serializer(root,name,reduce,noquotes) do
+--    ...(line)
+-- end
+--
+-- so this is on the todo list
+
+table.tofile_maxtab = 2*1024
+
+function table.tofile(filename,root,name,reduce,noquotes,hexify)
+    local f = io.open(filename,'w')
+    if f then
+        local maxtab = table.tofile_maxtab
+        if maxtab > 1 then
+            local t = { }
+            local function flush(s)
+                t[#t+1] = s
+                if #t > maxtab then
+                    f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice
+                    t = { }
+                end
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+            f:write(concat(t,"\n"),"\n")
+        else
+            local function flush(s)
+                f:write(s,"\n")
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+        end
+        f:close()
+    end
+end
+
+local function flatten(t,f,complete)
+    for i=1,#t do
+        local v = t[i]
+        if type(v) == "table" then
+            if complete or type(v[1]) == "table" then
+                flatten(v,f,complete)
+            else
+                f[#f+1] = v
+            end
+        else
+            f[#f+1] = v
+        end
+    end
+end
+
+function table.flatten(t)
+    local f = { }
+    flatten(t,f,true)
+    return f
+end
+
+function table.unnest(t) -- bad name
+    local f = { }
+    flatten(t,f,false)
+    return f
+end
+
+table.flatten_one_level = table.unnest
+
+-- the next three may disappear
+
+function table.remove_value(t,value) -- todo: n
+    if value then
+        for i=1,#t do
+            if t[i] == value then
+                remove(t,i)
+                -- remove all, so no: return
+            end
+        end
+    end
+end
+
+function table.insert_before_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i,str)
+                    return
+                end
+            end
+        end
+        insert(t,1,str)
+    elseif value then
+        insert(t,1,value)
+    end
+end
+
+function table.insert_after_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i+1,str)
+                    return
+                end
+            end
+        end
+        t[#t+1] = str
+    elseif value then
+        t[#t+1] = value
+    end
+end
+
+local function are_equal(a,b,n,m) -- indexed
+    if #a == #b then
+        n = n or 1
+        m = m or #a
+        for i=n,m do
+            local ai, bi = a[i], b[i]
+            if ai==bi then
+                -- same
+            elseif type(ai)=="table" and type(bi)=="table" then
+                if not are_equal(ai,bi) then
+                    return false
+                end
+            else
+                return false
+            end
+        end
+        return true
+    else
+        return false
+    end
+end
+
+local function identical(a,b) -- assumes same structure
+    for ka, va in next, a do
+        local vb = b[k]
+        if va == vb then
+            -- same
+        elseif type(va) == "table" and  type(vb) == "table" then
+            if not identical(va,vb) then
+                return false
+            end
+        else
+            return false
+        end
+    end
+    return true
+end
+
+table.are_equal = are_equal
+table.identical = identical
+
+-- maybe also make a combined one
+
+function table.compact(t)
+    if t then
+        for k,v in next, t do
+            if not next(v) then
+                t[k] = nil
+            end
+        end
+    end
+end
+
+function table.contains(t, v)
+    if t then
+        for i=1, #t do
+            if t[i] == v then
+                return i
+            end
+        end
+    end
+    return false
+end
+
+function table.count(t)
+    local n, e = 0, next(t)
+    while e do
+        n, e = n + 1, next(t,e)
+    end
+    return n
+end
+
+function table.swapped(t)
+    local s = { }
+    for k, v in next, t do
+        s[v] = k
+    end
+    return s
+end
+
+--~ function table.are_equal(a,b)
+--~     return table.serialize(a) == table.serialize(b)
+--~ end
+
+function table.clone(t,p) -- t is optional or nil or table
+    if not p then
+        t, p = { }, t or { }
+    elseif not t then
+        t = { }
+    end
+    setmetatable(t, { __index = function(_,key) return p[key] end })
+    return t
+end
+
+function table.hexed(t,seperator)
+    local tt = { }
+    for i=1,#t do tt[i] = format("0x%04X",t[i]) end
+    return concat(tt,seperator or " ")
+end
+
+function table.reverse_hash(h)
+    local r = { }
+    for k,v in next, h do
+        r[v] = lower(gsub(k," ",""))
+    end
+    return r
+end
+
+function table.reverse(t)
+    local tt = { }
+    if #t > 0 then
+        for i=#t,1,-1 do
+            tt[#tt+1] = t[i]
+        end
+    end
+    return tt
+end
+
+--~ function table.keys(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return k
+--~ end
+
+--~ function table.keys_as_string(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return concat(k,"")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-io'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local byte = string.byte
+
+if string.find(os.getenv("PATH"),";") then
+    io.fileseparator, io.pathseparator = "\\", ";"
+else
+    io.fileseparator, io.pathseparator = "/" , ":"
+end
+
+function io.loaddata(filename,textmode)
+    local f = io.open(filename,(textmode and 'r') or 'rb')
+    if f then
+        local data = f:read('*all')
+    --  garbagecollector.check(data)
+        f:close()
+        return data
+    else
+        return nil
+    end
+end
+
+function io.savedata(filename,data,joiner)
+    local f = io.open(filename,"wb")
+    if f then
+        if type(data) == "table" then
+            f:write(table.join(data,joiner or ""))
+        elseif type(data) == "function" then
+            data(f)
+        else
+            f:write(data)
+        end
+        f:close()
+        return true
+    else
+        return false
+    end
+end
+
+function io.exists(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return false
+    else
+        assert(f:close())
+        return true
+    end
+end
+
+function io.size(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return 0
+    else
+        local s = f:seek("end")
+        assert(f:close())
+        return s
+    end
+end
+
+function io.noflines(f)
+    local n = 0
+    for _ in f:lines() do
+        n = n + 1
+    end
+    f:seek('set',0)
+    return n
+end
+
+local nextchar = {
+    [ 4] = function(f)
+        return f:read(1,1,1,1)
+    end,
+    [ 2] = function(f)
+        return f:read(1,1)
+    end,
+    [ 1] = function(f)
+        return f:read(1)
+    end,
+    [-2] = function(f)
+        local a, b = f:read(1,1)
+        return b, a
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        return d, c, b, a
+    end
+}
+
+function io.characters(f,n)
+    if f then
+        return nextchar[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+local nextbyte = {
+    [4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(a), byte(b), byte(c), byte(d)
+        else
+            return nil, nil, nil, nil
+        end
+    end,
+    [2] = function(f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(a), byte(b)
+        else
+            return nil, nil
+        end
+    end,
+    [1] = function (f)
+        local a = f:read(1)
+        if a then
+            return byte(a)
+        else
+            return nil
+        end
+    end,
+    [-2] = function (f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(b), byte(a)
+        else
+            return nil, nil
+        end
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(d), byte(c), byte(b), byte(a)
+        else
+            return nil, nil, nil, nil
+        end
+    end
+}
+
+function io.bytes(f,n)
+    if f then
+        return nextbyte[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+function io.ask(question,default,options)
+    while true do
+        io.write(question)
+        if options then
+            io.write(string.format(" [%s]",table.concat(options,"|")))
+        end
+        if default then
+            io.write(string.format(" [%s]",default))
+        end
+        io.write(string.format(" "))
+        local answer = io.read()
+        answer = answer:gsub("^%s*(.*)%s*$","%1")
+        if answer == "" and default then
+            return default
+        elseif not options then
+            return answer
+        else
+            for _,v in pairs(options) do
+                if v == answer then
+                    return answer
+                end
+            end
+            local pattern = "^" .. answer
+            for _,v in pairs(options) do
+                if v:find(pattern) then
+                    return v
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-number'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+number = number or { }
+
+-- a,b,c,d,e,f = number.toset(100101)
+
+function number.toset(n)
+    return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)")
+end
+
+function number.toevenhex(n)
+    local s = format("%X",n)
+    if #s % 2 == 0 then
+        return s
+    else
+        return "0" .. s
+    end
+end
+
+-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5%
+-- on
+--
+-- for i=1,1000000 do
+--     local a,b,c,d,e,f,g,h = number.toset(12345678)
+--     local a,b,c,d         = number.toset(1234)
+--     local a,b,c           = number.toset(123)
+-- end
+--
+-- of course dedicated "(.)(.)(.)(.)" matches are even faster
+
+local one = lpeg.C(1-lpeg.S(''))^1
+
+function number.toset(n)
+    return one:match(tostring(n))
+end
+
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-set'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+set = set or { }
+
+local nums   = { }
+local tabs   = { }
+local concat = table.concat
+
+set.create = table.tohash
+
+function set.tonumber(t)
+    if next(t) then
+        local s = ""
+    --  we could save mem by sorting, but it slows down
+        for k, v in pairs(t) do
+            if v then
+            --  why bother about the leading space
+                s = s .. " " .. k
+            end
+        end
+        if not nums[s] then
+            tabs[#tabs+1] = t
+            nums[s] = #tabs
+        end
+        return nums[s]
+    else
+        return 0
+    end
+end
+
+function set.totable(n)
+    if n == 0 then
+        return { }
+    else
+        return tabs[n] or { }
+    end
+end
+
+function set.contains(n,s)
+    if type(n) == "table" then
+        return n[s]
+    elseif n == 0 then
+        return false
+    else
+        local t = tabs[n]
+        return t and t[s]
+    end
+end
+
+--~ local c = set.create{'aap','noot','mies'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ local c = set.create{'zus','wim','jet'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ print(t['jet'])
+--~ print(set.contains(t,'jet'))
+--~ print(set.contains(t,'aap'))
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-os'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lub.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+function os.resultof(command)
+    return io.popen(command,"r"):read("*all")
+end
+
+if not os.exec  then os.exec  = os.execute end
+if not os.spawn then os.spawn = os.execute end
+
+--~ os.type : windows | unix (new, we already guessed os.platform)
+--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new)
+
+if not io.fileseparator then
+    if find(os.getenv("PATH"),";") then
+        io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows"
+    else
+        io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix"
+    end
+end
+
+os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix"
+
+function os.launch(str)
+    if os.platform == "windows" then
+        os.execute("start " .. str) -- os.spawn ?
+    else
+        os.execute(str .. " &")     -- os.spawn ?
+    end
+end
+
+if not os.setenv then
+    function os.setenv() return false end
+end
+
+if not os.times then
+    -- utime  = user time
+    -- stime  = system time
+    -- cutime = children user time
+    -- cstime = children system time
+    function os.times()
+        return {
+            utime  = os.gettimeofday(), -- user
+            stime  = 0,                 -- system
+            cutime = 0,                 -- children user
+            cstime = 0,                 -- children system
+        }
+    end
+end
+
+os.gettimeofday = os.gettimeofday or os.clock
+
+local startuptime = os.gettimeofday()
+
+function os.runtime()
+    return os.gettimeofday() - startuptime
+end
+
+--~ print(os.gettimeofday()-os.time())
+--~ os.sleep(1.234)
+--~ print (">>",os.runtime())
+--~ print(os.date("%H:%M:%S",os.gettimeofday()))
+--~ print(os.date("%H:%M:%S",os.time()))
+
+os.arch = os.arch or function()
+    local a = os.resultof("uname -m") or "linux"
+    os.arch = function()
+        return a
+    end
+    return a
+end
+
+local platform
+
+function os.currentplatform(name,default)
+    if not platform then
+        local name = os.name or os.platform or name -- os.name is built in, os.platform is mine
+        if not name then
+            platform = default or "linux"
+        elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then
+            if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then
+                platform = "mswin-64"
+            else
+                platform = "mswin"
+            end
+        else
+            local architecture = os.arch()
+            if name == "linux" then
+                if find(architecture,"x86_64") then
+                    platform = "linux-64"
+                elseif find(architecture,"ppc") then
+                    platform = "linux-ppc"
+                else
+                    platform = "linux"
+                end
+            elseif name == "macosx" then
+                if find(architecture,"i386") then
+                    platform = "osx-intel"
+                else
+                    platform = "osx-ppc"
+                end
+            elseif name == "sunos" then
+                if find(architecture,"sparc") then
+                    platform = "solaris-sparc"
+                else -- if architecture == 'i86pc'
+                    platform = "solaris-intel"
+                end
+            elseif name == "freebsd" then
+                if find(architecture,"amd64") then
+                    platform = "freebsd-amd64"
+                else
+                    platform = "freebsd"
+                end
+            else
+                platform = default or name
+            end
+        end
+        function os.currentplatform()
+            return platform
+        end
+    end
+    return platform
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-file'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- needs a cleanup
+
+file = file or { }
+
+local concat = table.concat
+local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub
+
+function file.removesuffix(filename)
+    return (gsub(filename,"%.[%a%d]+$",""))
+end
+
+function file.addsuffix(filename, suffix)
+    if not find(filename,"%.[%a%d]+$") then
+        return filename .. "." .. suffix
+    else
+        return filename
+    end
+end
+
+function file.replacesuffix(filename, suffix)
+    return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix
+end
+
+function file.dirname(name,default)
+    return match(name,"^(.+)[/\\].-$") or (default or "")
+end
+
+function file.basename(name)
+    return match(name,"^.+[/\\](.-)$") or name
+end
+
+function file.nameonly(name)
+    return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$",""))
+end
+
+function file.extname(name)
+    return match(name,"^.+%.([^/\\]-)$") or  ""
+end
+
+file.suffix = file.extname
+
+--~ print(file.join("x/","/y"))
+--~ print(file.join("http://","/y"))
+--~ print(file.join("http://a","/y"))
+--~ print(file.join("http:///a","/y"))
+--~ print(file.join("//nas-1","/y"))
+
+function file.join(...)
+    local pth = concat({...},"/")
+    pth = gsub(pth,"\\","/")
+    local a, b = match(pth,"^(.*://)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    a, b = match(pth,"^(//)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    return (gsub(pth,"//+","/"))
+end
+
+function file.iswritable(name)
+    local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,"."))
+    return a and a.permissions:sub(2,2) == "w"
+end
+
+function file.isreadable(name)
+    local a = lfs.attributes(name)
+    return a and a.permissions:sub(1,1) == "r"
+end
+
+file.is_readable = file.isreadable
+file.is_writable = file.iswritable
+
+-- todo: lpeg
+
+function file.split_path(str)
+    local t = { }
+    str = gsub(str,"\\", "/")
+    str = gsub(str,"(%a):([;/])", "%1\001%2")
+    for name in gmatch(str,"([^;:]+)") do
+        if name ~= "" then
+            t[#t+1] = gsub(name,"\001",":")
+        end
+    end
+    return t
+end
+
+function file.join_path(tab)
+    return concat(tab,io.pathseparator) -- can have trailing //
+end
+
+function file.collapse_path(str)
+    str = gsub(str,"/%./","/")
+    local n, m = 1, 1
+    while n > 0 or m > 0 do
+        str, n = gsub(str,"[^/%.]+/%.%.$","")
+        str, m = gsub(str,"[^/%.]+/%.%./","")
+    end
+    str = gsub(str,"([^/])/$","%1")
+    str = gsub(str,"^%./","")
+    str = gsub(str,"/%.$","")
+    if str == "" then str = "." end
+    return str
+end
+
+--~ print(file.collapse_path("a/./b/.."))
+--~ print(file.collapse_path("a/aa/../b/bb"))
+--~ print(file.collapse_path("a/../.."))
+--~ print(file.collapse_path("a/.././././b/.."))
+--~ print(file.collapse_path("a/./././b/.."))
+--~ print(file.collapse_path("a/b/c/../.."))
+
+function file.robustname(str)
+    return (gsub(str,"[^%a%d%/%-%.\\]+","-"))
+end
+
+file.readdata = io.loaddata
+file.savedata = io.savedata
+
+function file.copy(oldname,newname)
+    file.savedata(newname,io.loaddata(oldname))
+end
+
+-- lpeg variants, slightly faster, not always
+
+--~ local period    = lpeg.P(".")
+--~ local slashes   = lpeg.S("\\/")
+--~ local noperiod  = 1-period
+--~ local noslashes = 1-slashes
+--~ local name      = noperiod^1
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1
+
+--~ function file.extname(name)
+--~     return pattern:match(name) or ""
+--~ end
+
+--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1)
+
+--~ function file.removesuffix(name)
+--~     return pattern:match(name)
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1
+
+--~ function file.basename(name)
+--~     return pattern:match(name) or name
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1
+
+--~ function file.dirname(name)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2)
+--~     else
+--~         return ""
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.addsuffix(name, suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.replacesuffix(name,suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2) .. "." .. suffix
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1
+
+--~ function file.nameonly(name)
+--~     local a, b = pattern:match(name)
+--~     if b then
+--~         return name:sub(a,b-2)
+--~     elseif a then
+--~         return name:sub(a)
+--~     else
+--~         return name
+--~     end
+--~ end
+
+--~ local test = file.extname
+--~ local test = file.basename
+--~ local test = file.dirname
+--~ local test = file.addsuffix
+--~ local test = file.replacesuffix
+--~ local test = file.nameonly
+
+--~ print(1,test("./a/b/c/abd.def.xxx","!!!"))
+--~ print(2,test("./../b/c/abd.def.xxx","!!!"))
+--~ print(3,test("a/b/c/abd.def.xxx","!!!"))
+--~ print(4,test("a/b/c/def.xxx","!!!"))
+--~ print(5,test("a/b/c/def","!!!"))
+--~ print(6,test("def","!!!"))
+--~ print(7,test("def.xxx","!!!"))
+
+--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim)
+
+-- also rewrite previous
+
+local letter    = lpeg.R("az","AZ") + lpeg.S("_-+")
+local separator = lpeg.P("://")
+
+local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/")
+local rootbased = lpeg.P("/") + letter*lpeg.P(":")
+
+-- ./name ../name  /name c: :// name/name
+
+function file.is_qualified_path(filename)
+    return qualified:match(filename)
+end
+
+function file.is_rootbased_path(filename)
+    return rootbased:match(filename)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-md5'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This also provides file checksums and checkers.
+
+local gsub, format, byte = string.gsub, string.format, string.byte
+
+local function convert(str,fmt)
+    return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end))
+end
+
+if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end
+if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end
+if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end
+
+--~ if not md5.HEX then
+--~     local function remap(chr) return format("%02X",byte(chr)) end
+--~     function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.hex then
+--~     local function remap(chr) return format("%02x",byte(chr)) end
+--~     function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.dec then
+--~     local function remap(chr) return format("%03i",byte(chr)) end
+--~     function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+
+file.needs_updating_threshold = 1
+
+function file.needs_updating(oldname,newname) -- size modification access change
+    local oldtime = lfs.attributes(oldname, modification)
+    local newtime = lfs.attributes(newname, modification)
+    if newtime >= oldtime then
+        return false
+    elseif oldtime - newtime < file.needs_updating_threshold then
+        return false
+    else
+        return true
+    end
+end
+
+function file.checksum(name)
+    if md5 then
+        local data = io.loaddata(name)
+        if data then
+            return md5.HEX(data)
+        end
+    end
+    return nil
+end
+
+function file.loadchecksum(name)
+    if md5 then
+        local data = io.loaddata(name .. ".md5")
+        return data and data:gsub("%s","")
+    end
+    return nil
+end
+
+function file.savechecksum(name, checksum)
+    if not checksum then checksum = file.checksum(name) end
+    if checksum then
+        io.savedata(name .. ".md5",checksum)
+        return checksum
+    end
+    return nil
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-dir'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type = type
+local find, gmatch = string.find, string.gmatch
+
+dir = dir or { }
+
+-- optimizing for no string.find (*) does not save time
+
+local attributes = lfs.attributes
+local walkdir    = lfs.dir
+
+local function glob_pattern(path,patt,recurse,action)
+    local ok, scanner
+    if path == "/" then
+        ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
+    else
+        ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
+    end
+    if ok and type(scanner) == "function" then
+        if not find(path,"/$") then path = path .. '/' end
+        for name in scanner do
+            local full = path .. name
+            local mode = attributes(full,'mode')
+            if mode == 'file' then
+                if find(full,patt) then
+                    action(full)
+                end
+            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
+                glob_pattern(full,patt,recurse,action)
+            end
+        end
+    end
+end
+
+dir.glob_pattern = glob_pattern
+
+local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V
+
+local pattern = Ct {
+    [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3),
+    [2] = C(((1-S("*?/"))^0 * P("/"))^0),
+    [3] = C(P(1)^0)
+}
+
+local filter = Cs ( (
+    P("**") / ".*" +
+    P("*")  / "[^/]*" +
+    P("?")  / "[^/]" +
+    P(".")  / "%%." +
+    P("+")  / "%%+" +
+    P("-")  / "%%-" +
+    P(1)
+)^0 )
+
+local function glob(str,t)
+    if type(str) == "table" then
+        local t = t or { }
+        for s=1,#str do
+            glob(str[s],t)
+        end
+        return t
+    elseif lfs.isfile(str) then
+        local t = t or { }
+        t[#t+1] = str
+        return t
+    else
+        local split = pattern:match(str)
+        if split then
+            local t = t or { }
+            local action = action or function(name) t[#t+1] = name end
+            local root, path, base = split[1], split[2], split[3]
+            local recurse = find(base,"%*%*")
+            local start = root .. path
+            local result = filter:match(start .. base)
+            glob_pattern(start,result,recurse,action)
+            return t
+        else
+            return { }
+        end
+    end
+end
+
+dir.glob = glob
+
+--~ list = dir.glob("**/*.tif")
+--~ list = dir.glob("/**/*.tif")
+--~ list = dir.glob("./**/*.tif")
+--~ list = dir.glob("oeps/**/*.tif")
+--~ list = dir.glob("/oeps/**/*.tif")
+
+local function globfiles(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func -- alas, we need this indirect way
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if recurse then
+                    globfiles(path .. "/" .. name,recurse,func,files)
+                end
+            elseif mode == "file" then
+                if func then
+                    if func(name) then
+                        files[#files+1] = path .. "/" .. name
+                    end
+                else
+                    files[#files+1] = path .. "/" .. name
+                end
+            end
+        end
+    end
+    return files
+end
+
+dir.globfiles = globfiles
+
+-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
+-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
+-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
+-- t = dir.glob("f:/minimal/tex/**/*")
+-- print(dir.ls("f:/minimal/tex/**/*"))
+-- print(dir.ls("*.tex"))
+
+function dir.ls(pattern)
+    return table.concat(glob(pattern),"\n")
+end
+
+--~ mkdirs("temp")
+--~ mkdirs("a/b/c")
+--~ mkdirs(".","/a/b/c")
+--~ mkdirs("a","b","c")
+
+local make_indeed = true -- false
+
+if string.find(os.getenv("PATH"),";") then
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        local first, middle, last
+        local drive = false
+        first, middle, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            -- empty network path == local path
+        else
+            first, last = str:match("^(//)/*(.-)$")
+            if first then
+                middle, last = str:match("([^/]+)/+(.-)$")
+                if middle then
+                    pth = "//" .. middle
+                else
+                    pth = "//" .. last
+                    last = ""
+                end
+            else
+                first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$")
+                if first then
+                    pth, drive = first .. middle, true
+                else
+                    middle, last = str:match("^(/*)(.-)$")
+                    if not middle then
+                        last = str
+                    end
+                end
+            end
+        end
+        for s in gmatch(last,"[^/]+") do
+            if pth == "" then
+                pth = s
+            elseif drive then
+                pth, drive = pth .. s, false
+            else
+                pth = pth .. "/" .. s
+            end
+            if make_indeed and not lfs.isdir(pth) then
+                lfs.mkdir(pth)
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("a:"))
+--~         print(dir.mkdirs("a:/b/c"))
+--~         print(dir.mkdirs("a:b/c"))
+--~         print(dir.mkdirs("a:/bbb/c"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        local first, nothing, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            first = lfs.currentdir() .. "/"
+            first = first:gsub("\\","/")
+        end
+        if not first then
+            first, last = str:match("^(//)/*(.*)$")
+        end
+        if not first then
+            first, last = str:match("^([a-zA-Z]:)(.*)$")
+            if first and not find(last,"^/") then
+                local d = lfs.currentdir()
+                if lfs.chdir(first) then
+                    first = lfs.currentdir()
+                    first = first:gsub("\\","/")
+                end
+                lfs.chdir(d)
+            end
+        end
+        if not first then
+            first, last = lfs.currentdir(), str
+            first = first:gsub("\\","/")
+        end
+        last = last:gsub("//","/")
+        last = last:gsub("/%./","/")
+        last = last:gsub("^/*","")
+        first = first:gsub("/*$","")
+        if last == "" then
+            return first
+        else
+            return first .. "/" .. last
+        end
+    end
+
+else
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        str = str:gsub("/+","/")
+        if find(str,"^/") then
+            pth = "/"
+            for s in gmatch(str,"[^/]+") do
+                local first = (pth == "/")
+                if first then
+                    pth = pth .. s
+                else
+                    pth = pth .. "/" .. s
+                end
+                if make_indeed and not first and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        else
+            pth = "."
+            for s in gmatch(str,"[^/]+") do
+                pth = pth .. "/" .. s
+                if make_indeed and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        if not find(str,"^/") then
+            str = lfs.currentdir() .. "/" .. str
+        end
+        str = str:gsub("//","/")
+        str = str:gsub("/%./","/")
+        return str
+    end
+
+end
+
+dir.makedirs = dir.mkdirs
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-boolean'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+boolean = boolean or { }
+
+local type, tonumber = type, tonumber
+
+function boolean.tonumber(b)
+    if b then return 1 else return 0 end
+end
+
+function toboolean(str,tolerant)
+    if tolerant then
+        local tstr = type(str)
+        if tstr == "string" then
+            return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t"
+        elseif tstr == "number" then
+            return tonumber(str) ~= 0
+        elseif tstr == "nil" then
+            return false
+        else
+            return str
+        end
+    elseif str == "true" then
+        return true
+    elseif str == "false" then
+        return false
+    else
+        return str
+    end
+end
+
+function string.is_boolean(str)
+    if type(str) == "string" then
+        if str == "true" or str == "yes" or str == "on" or str == "t" then
+            return true
+        elseif str == "false" or str == "no" or str == "off" or str == "f" then
+            return false
+        end
+    end
+    return nil
+end
+
+function boolean.alwaystrue()
+    return true
+end
+
+function boolean.falsetrue()
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-math'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan
+
+if not math.round then
+    function math.round(x)
+        return floor(x + 0.5)
+    end
+end
+
+if not math.div then
+    function math.div(n,m)
+        return floor(n/m)
+    end
+end
+
+if not math.mod then
+    function math.mod(n,m)
+        return n % m
+    end
+end
+
+local pipi = 2*math.pi/360
+
+function math.sind(d)
+    return sin(d*pipi)
+end
+
+function math.cosd(d)
+    return cos(d*pipi)
+end
+
+function math.tand(d)
+    return tan(d*pipi)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-utils'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- hm, quite unreadable
+
+if not utils        then utils        = { } end
+if not utils.merger then utils.merger = { } end
+if not utils.lua    then utils.lua    = { } end
+
+utils.merger.m_begin = "begin library merge"
+utils.merger.m_end   = "end library merge"
+utils.merger.pattern =
+    "%c+" ..
+    "%-%-%s+" .. utils.merger.m_begin ..
+    "%c+(.-)%c+" ..
+    "%-%-%s+" .. utils.merger.m_end ..
+    "%c+"
+
+function utils.merger._self_fake_()
+    return
+        "-- " .. "created merged file" .. "\n\n" ..
+        "-- " .. utils.merger.m_begin  .. "\n\n" ..
+        "-- " .. utils.merger.m_end    .. "\n\n"
+end
+
+function utils.report(...)
+    print(...)
+end
+
+utils.merger.strip_comment = true
+
+function utils.merger._self_load_(name)
+    local f, data = io.open(name), ""
+    if f then
+        utils.report("reading merge from %s",name)
+        data = f:read("*all")
+        f:close()
+    else
+        utils.report("unknown file to merge %s",name)
+    end
+    if data and utils.merger.strip_comment then
+        -- saves some 20K
+        data = data:gsub("%-%-~[^\n\r]*[\r\n]", "")
+    end
+    return data or ""
+end
+
+function utils.merger._self_save_(name, data)
+    if data ~= "" then
+        local f = io.open(name,'w')
+        if f then
+            utils.report("saving merge from %s",name)
+            f:write(data)
+            f:close()
+        end
+    end
+end
+
+function utils.merger._self_swap_(data,code)
+    if data ~= "" then
+        return (data:gsub(utils.merger.pattern, function(s)
+            return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n"
+        end, 1))
+    else
+        return ""
+    end
+end
+
+--~ stripper:
+--~
+--~ data = string.gsub(data,"%-%-~[^\n]*\n","")
+--~ data = string.gsub(data,"\n\n+","\n")
+
+function utils.merger._self_libs_(libs,list)
+    local result, f, frozen = { }, nil, false
+    result[#result+1] = "\n"
+    if type(libs) == 'string' then libs = { libs } end
+    if type(list) == 'string' then list = { list } end
+    local foundpath = nil
+    for _, lib in ipairs(libs) do
+        for _, pth in ipairs(list) do
+            pth = string.gsub(pth,"\\","/") -- file.clean_path
+            utils.report("checking library path %s",pth)
+            local name = pth .. "/" .. lib
+            if lfs.isfile(name) then
+                foundpath = pth
+            end
+        end
+        if foundpath then break end
+    end
+    if foundpath then
+        utils.report("using library path %s",foundpath)
+        local right, wrong = { }, { }
+        for _, lib in ipairs(libs) do
+            local fullname = foundpath .. "/" .. lib
+            if lfs.isfile(fullname) then
+            --  right[#right+1] = lib
+                utils.report("merging library %s",fullname)
+                result[#result+1] = "do -- create closure to overcome 200 locals limit"
+                result[#result+1] = io.loaddata(fullname,true)
+                result[#result+1] = "end -- of closure"
+            else
+            --  wrong[#wrong+1] = lib
+                utils.report("no library %s",fullname)
+            end
+        end
+        if #right > 0 then
+            utils.report("merged libraries: %s",table.concat(right," "))
+        end
+        if #wrong > 0 then
+            utils.report("skipped libraries: %s",table.concat(wrong," "))
+        end
+    else
+        utils.report("no valid library path found")
+    end
+    return table.concat(result, "\n\n")
+end
+
+function utils.merger.selfcreate(libs,list,target)
+    if target then
+        utils.merger._self_save_(
+            target,
+            utils.merger._self_swap_(
+                utils.merger._self_fake_(),
+                utils.merger._self_libs_(libs,list)
+            )
+        )
+    end
+end
+
+function utils.merger.selfmerge(name,libs,list,target)
+    utils.merger._self_save_(
+        target or name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            utils.merger._self_libs_(libs,list)
+        )
+    )
+end
+
+function utils.merger.selfclean(name)
+    utils.merger._self_save_(
+        name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            ""
+        )
+    )
+end
+
+function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true
+ -- utils.report("compiling",luafile,"into",lucfile)
+    os.remove(lucfile)
+    local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile)
+    if strip ~= false then
+        command = "-s " .. command
+    end
+    local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0)
+    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+     -- utils.report("removing",luafile)
+        os.remove(luafile)
+    end
+    return done
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-tab'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
The parser used here is inspired by the variant discussed in the lua book, but
+handles comment and processing instructions, has a different structure, provides
+parent access; a first version used different trickery but was less optimized to we
+went this route. First we had a find based parser, now we have an  based one.
+The find based parser can be found in l-xml-edu.lua along with other older code.

+
+
Expecially the lpath code is experimental, we will support some of xpath, but
+only things that make sense for us; as compensation it is possible to hook in your
+own functions. Apart from preprocessing content for  we also need
+this module for process management, like handling  and 
+files.

+
+
+a/b/c /*/c
+a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n)
+a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n)
+
+
+
Beware, the interface may change. For instance at, ns, tg, dt may get more
+verbose names. Once the code is stable we will also remove some tracing and
+optimize the code.

+--ldx]]--
+
+xml = xml or { }
+
+--~ local xml = xml
+
+local concat, remove, insert = table.concat, table.remove, table.insert
+local type, next, setmetatable = type, next, setmetatable
+local format, lower, find = string.format, string.lower, string.find
+
+--[[ldx--
+
This module can be used stand alone but also inside  in
+which case it hooks into the tracker code. Therefore we provide a few
+functions that set the tracers.

+--ldx]]--
+
+local trace_remap = false
+
+if trackers then
+    trackers.register("xml.remap", function(v) trace_remap = v end)
+end
+
+function xml.settrace(str,value)
+    if str == "remap" then
+        trace_remap = value or false
+    end
+end
+
+--[[ldx--
+
First a hack to enable namespace resolving. A namespace is characterized by
+a . The following function associates a namespace prefix with a
+pattern. We use , which in this case is more than twice as fast as a
+find based solution where we loop over an array of patterns. Less code and
+much cleaner.

+--ldx]]--
+
+xml.xmlns = xml.xmlns or { }
+
+local check = lpeg.P(false)
+local parse = check
+
+--[[ldx--
+
The next function associates a namespace prefix with an . This
+normally happens independent of parsing.

+
+
+xml.registerns("mml","mathml")
+
+--ldx]]--
+
+function xml.registerns(namespace, pattern) -- pattern can be an lpeg
+    check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace
+    parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) }
+end
+
+--[[ldx--
+
The next function also registers a namespace, but this time we map a
+given namespace prefix onto a registered one, using the given
+. This used for attributes like xmlns:m.

+
+
+xml.checkns("m","http://www.w3.org/mathml")
+
+--ldx]]--
+
+function xml.checkns(namespace,url)
+    local ns = parse:match(lower(url))
+    if ns and namespace ~= ns then
+        xml.xmlns[namespace] = ns
+    end
+end
+
+--[[ldx--
+
Next we provide a way to turn an  into a registered
+namespace. This used for the xmlns attribute.

+
+
+resolvedns = xml.resolvens("http://www.w3.org/mathml")
+
+
+This returns mml.
+--ldx]]--
+
+function xml.resolvens(url)
+     return parse:match(lower(url)) or ""
+end
+
+--[[ldx--
+
A namespace in an element can be remapped onto the registered
+one efficiently by using the xml.xmlns table.

+--ldx]]--
+
+--[[ldx--
+
This version uses . We follow the same approach as before, stack and top and
+such. This version is about twice as fast which is mostly due to the fact that
+we don't have to prepare the stream for cdata, doctype etc etc. This variant is
+is dedicated to Luigi Scarso, who challenged me with 40 megabyte  files that
+took 12.5 seconds to load (1.5 for file io and the rest for tree building). With
+the  implementation we got that down to less 7.3 seconds. Loading the 14
+ interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.

+
+
Next comes the parser. The rather messy doctype definition comes in many
+disguises so it is no surprice that later on have to dedicate quite some
+ code to it.

+
+
+
+
+
+
+
+
+
+
+
The code may look a bit complex but this is mostly due to the fact that we
+resolve namespaces and attach metatables. There is only one public function:

+
+
+local x = xml.convert(somestring)
+
+
+
An optional second boolean argument tells this function not to create a root
+element.

+--ldx]]--
+
+xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes
+
+-- not just one big nested table capture (lpeg overflow)
+
+local nsremap, resolvens = xml.xmlns, xml.resolvens
+
+local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {}
+
+local mt = { __tostring = xml.text }
+
+function xml.check_error(top,toclose)
+    return ""
+end
+
+local strip   = false
+local cleanup = false
+
+function xml.set_text_cleanup(fnc)
+    cleanup = fnc
+end
+
+local function add_attribute(namespace,tag,value)
+    if cleanup and #value > 0 then
+        value = cleanup(value) -- new
+    end
+    if tag == "xmlns" then
+        xmlns[#xmlns+1] = resolvens(value)
+        at[tag] = value
+    elseif namespace == "xmlns" then
+        xml.checkns(tag,value)
+        at["xmlns:" .. tag] = value
+    else
+        at[tag] = value
+    end
+end
+
+local function add_begin(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
+    top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] }
+    setmetatable(top, mt)
+    dt = top.dt
+    stack[#stack+1] = top
+    at = { }
+end
+
+local function add_end(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local toclose = remove(stack)
+    top = stack[#stack]
+    if #stack < 1 then
+        errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "")
+    elseif toclose.tg ~= tag then -- no namespace check
+        errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "")
+    end
+    dt = top.dt
+    dt[#dt+1] = toclose
+    dt[0] = top
+    if toclose.at.xmlns then
+        remove(xmlns)
+    end
+end
+
+local function add_empty(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
+    top = stack[#stack]
+    dt = top.dt
+    local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top }
+    dt[#dt+1] = t
+    setmetatable(t, mt)
+    if at.xmlns then
+        remove(xmlns)
+    end
+    at = { }
+end
+
+local function add_text(text)
+    if cleanup and #text > 0 then
+        dt[#dt+1] = cleanup(text)
+    else
+        dt[#dt+1] = text
+    end
+end
+
+local function add_special(what, spacing, text)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    if strip and (what == "@cm@" or what == "@dt@") then
+        -- forget it
+    else
+        dt[#dt+1] = { special=true, ns="", tg=what, dt={text} }
+    end
+end
+
+local function set_message(txt)
+    errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","")
+end
+
+local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V
+
+local space            = S(' \r\n\t')
+local open             = P('<')
+local close            = P('>')
+local squote           = S("'")
+local dquote           = S('"')
+local equal            = P('=')
+local slash            = P('/')
+local colon            = P(':')
+local valid            = R('az', 'AZ', '09') + S('_-.')
+local name_yes         = C(valid^1) * colon * C(valid^1)
+local name_nop         = C(P(true)) * C(valid^1)
+local name             = name_yes + name_nop
+
+local utfbom           = P('\000\000\254\255') + P('\255\254\000\000') +
+                         P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture
+
+local spacing          = C(space^0)
+local justtext         = C((1-open)^1)
+local somespace        = space^1
+local optionalspace    = space^0
+
+local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
+local attribute        = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute
+local attributes       = attribute^0
+
+local text             = justtext / add_text
+local balanced         = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example
+
+local emptyelement     = (spacing * open         * name * attributes * optionalspace * slash * close) / add_empty
+local beginelement     = (spacing * open         * name * attributes * optionalspace         * close) / add_begin
+local endelement       = (spacing * open * slash * name              * optionalspace         * close) / add_end
+
+local begincomment     = open * P("!--")
+local endcomment       = P("--") * close
+local begininstruction = open * P("?")
+local endinstruction   = P("?") * close
+local begincdata       = open * P("![CDATA[")
+local endcdata         = P("]]") * close
+
+local someinstruction  = C((1 - endinstruction)^0)
+local somecomment      = C((1 - endcomment    )^0)
+local somecdata        = C((1 - endcdata      )^0)
+
+local function entity(k,v) entities[k] = v end
+
+local begindoctype     = open * P("!DOCTYPE")
+local enddoctype       = close
+local beginset         = P("[")
+local endset           = P("]")
+local doctypename      = C((1-somespace)^0)
+local elementdoctype   = optionalspace * P("Packaging data in an xml like table is done with the following
+function. Maybe it will go away (when not used).

+--ldx]]--
+
+function xml.is_valid(root)
+    return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er
+end
+
+function xml.package(tag,attributes,data)
+    local ns, tg = tag:match("^(.-):?([^:]+)$")
+    local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} }
+    setmetatable(t, mt)
+    return t
+end
+
+function xml.is_valid(root)
+    return root and not root.error
+end
+
+xml.error_handler = (logs and logs.report) or (input and logs.report) or print
+
+--[[ldx--
+
We cannot load an  from a filehandle so we need to load
+the whole file first. The function accepts a string representing
+a filename or a file handle.

+--ldx]]--
+
+function xml.load(filename)
+    if type(filename) == "string" then
+        local f = io.open(filename,'r')
+        if f then
+            local root = xml.convert(f:read("*all"))
+            f:close()
+            return root
+        else
+            return xml.convert("")
+        end
+    elseif filename then -- filehandle
+        return xml.convert(filename:read("*all"))
+    else
+        return xml.convert("")
+    end
+end
+
+--[[ldx--
+
When we inject new elements, we need to convert strings to
+valid trees, which is what the next function does.

+--ldx]]--
+
+function xml.toxml(data)
+    if type(data) == "string" then
+        local root = { xml.convert(data,true) }
+        return (#root > 1 and root) or root[1]
+    else
+        return data
+    end
+end
+
+--[[ldx--
+
For copying a tree we use a dedicated function instead of the
+generic table copier. Since we know what we're dealing with we
+can speed up things a bit. The second argument is not to be used!

+--ldx]]--
+
+function copy(old,tables)
+    if old then
+        tables = tables or { }
+        local new = { }
+        if not tables[old] then
+            tables[old] = new
+        end
+        for k,v in pairs(old) do
+            new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v
+        end
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
+        end
+        return new
+    else
+        return { }
+    end
+end
+
+xml.copy = copy
+
+--[[ldx--
+
In  serializing the tree or parts of the tree is a major
+actitivity which is why the following function is pretty optimized resulting
+in a few more lines of code than needed. The variant that uses the formatting
+function for all components is about 15% slower than the concatinating
+alternative.

+--ldx]]--
+
+-- todo: add  when not present
+
+local fallbackhandle = (tex and tex.sprint) or io.write
+
+local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands)
+    if not e then
+        return
+    elseif not nocommands then
+        local ec = e.command
+        if ec ~= nil then -- we can have all kind of types
+            if e.special then
+                local etg, edt = e.tg, e.dt
+                local spc = specialconverter and specialconverter[etg]
+                if spc then
+                    local result = spc(edt[1])
+                    if result then
+                        handle(result)
+                        return
+                    else
+                        -- no need to handle any further
+                    end
+                end
+            end
+            local xc = xml.command
+            if xc then
+                xc(e,ec)
+                return
+            end
+        end
+    end
+    handle = handle or fallbackhandle
+    local etg = e.tg
+    if etg then
+        if e.special then
+            local edt = e.dt
+            local spc = specialconverter and specialconverter[etg]
+            if spc then
+                local result = spc(edt[1])
+                if result then
+                    handle(result)
+                else
+                    -- no need to handle any further
+                end
+            elseif etg == "@pi@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@cm@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@cd@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@dt@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@rt@" then
+                serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands)
+            end
+        else
+            local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn
+            local ats = eat and next(eat) and { } -- type test maybe faster
+            if ats then
+                if attributeconverter then
+                    for k,v in next, eat do
+                        ats[#ats+1] = format('%s=%q',k,attributeconverter(v))
+                    end
+                else
+                    for k,v in next, eat do
+                        ats[#ats+1] = format('%s=%q',k,v)
+                    end
+                end
+            end
+            if ern and trace_remap and ern ~= ens then
+                ens = ern
+            end
+            if ens ~= "" then
+                if edt and #edt > 0 then
+                    if ats then
+                    --  handle(format("<%s:%s %s>",ens,etg,concat(ats," ")))
+                        handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">")
+                    else
+                    --  handle(format("<%s:%s>",ens,etg))
+                        handle("<" .. ens .. ":" .. etg .. ">")
+                    end
+                    for i=1,#edt do
+                        local e = edt[i]
+                        if type(e) == "string" then
+                            if textconverter then
+                                handle(textconverter(e))
+                            else
+                                handle(e)
+                            end
+                        else
+                            serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                        end
+                    end
+                --  handle(format("",ens,etg))
+                    handle("")
+                else
+                    if ats then
+                    --  handle(format("<%s:%s %s/>",ens,etg,concat(ats," ")))
+                        handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>")
+                    else
+                    --  handle(format("<%s:%s/>",ens,etg))
+                        handle("<" .. ens .. ":" .. etg .. "/>")
+                    end
+                end
+            else
+                if edt and #edt > 0 then
+                    if ats then
+                    --  handle(format("<%s %s>",etg,concat(ats," ")))
+                        handle("<" .. etg .. " " .. concat(ats," ") .. ">")
+                    else
+                    --  handle(format("<%s>",etg))
+                        handle("<" .. etg .. ">")
+                    end
+                    for i=1,#edt do
+                        local ei = edt[i]
+                        if type(ei) == "string" then
+                            if textconverter then
+                                handle(textconverter(ei))
+                            else
+                                handle(ei)
+                            end
+                        else
+                            serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                        end
+                    end
+                --  handle(format("",etg))
+                    handle("")
+                else
+                    if ats then
+                    --  handle(format("<%s %s/>",etg,concat(ats," ")))
+                        handle("<" .. etg .. " " .. concat(ats," ") .. "/>")
+                    else
+                    --  handle(format("<%s/>",etg))
+                        handle("<" .. etg .. "/>")
+                    end
+                end
+            end
+        end
+    elseif type(e) == "string" then
+        if textconverter then
+            handle(textconverter(e))
+        else
+            handle(e)
+        end
+    else
+        for i=1,#e do
+            local ei = e[i]
+            if type(ei) == "string" then
+                if textconverter then
+                    handle(textconverter(ei))
+                else
+                    handle(ei)
+                end
+            else
+                serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
+            end
+        end
+    end
+end
+
+xml.serialize = serialize
+
+function xml.checkbom(root) -- can be made faster
+    if root.ri then
+        local dt, found = root.dt, false
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then
+                found = true
+                break
+            end
+        end
+        if not found then
+            insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } )
+            insert(dt, 2, "\n" )
+        end
+    end
+end
+
+--[[ldx--
+
At the cost of some 25% runtime overhead you can first convert the tree to a string
+and then handle the lot.

+--ldx]]--
+
+function xml.tostring(root) -- 25% overhead due to collecting
+    if root then
+        if type(root) == 'string' then
+            return root
+        elseif next(root) then -- next is faster than type (and >0 test)
+            local result = { }
+            serialize(root,function(s) result[#result+1] = s end)
+            return concat(result,"")
+        end
+    end
+    return ""
+end
+
+--[[ldx--
+
The next function operated on the content only and needs a handle function
+that accepts a string.

+--ldx]]--
+
+function xml.string(e,handle)
+    if not handle or (e.special and e.tg ~= "@rt@") then
+        -- nothing
+    elseif e.tg then
+        local edt = e.dt
+        if edt then
+            for i=1,#edt do
+                xml.string(edt[i],handle)
+            end
+        end
+    else
+        handle(e)
+    end
+end
+
+--[[ldx--
+
How you deal with saving data depends on your preferences. For a 40 MB database
+file the timing on a 2.3 Core Duo are as follows (time in seconds):

+
+
+1.3 : load data from file to string
+6.1 : convert string into tree
+5.3 : saving in file using xmlsave
+6.8 : converting to string using xml.tostring
+3.6 : saving converted string in file
+
+
+
The save function is given below.

+--ldx]]--
+
+function xml.save(root,name)
+    local f = io.open(name,"w")
+    if f then
+        xml.serialize(root,function(s) f:write(s) end)
+        f:close()
+    end
+end
+
+--[[ldx--
+
A few helpers:

+--ldx]]--
+
+function xml.body(root)
+    return (root.ri and root.dt[root.ri]) or root
+end
+
+function xml.text(root)
+    return (root and xml.tostring(root)) or ""
+end
+
+function xml.content(root) -- bugged
+    return (root and root.dt and xml.tostring(root.dt)) or ""
+end
+
+function xml.isempty(root, pattern)
+    if pattern == "" or pattern == "*" then
+        pattern = nil
+    end
+    if pattern then
+        -- todo
+        return false
+    else
+        return not root or not root.dt or #root.dt == 0 or root.dt == ""
+    end
+end
+
+--[[ldx--
+
The next helper erases an element but keeps the table as it is,
+and since empty strings are not serialized (effectively) it does
+not harm. Copying the table would take more time. Usage:

+
+
+dt[k] = xml.empty() or xml.empty(dt,k)
+
+--ldx]]--
+
+function xml.empty(dt,k)
+    if dt and k then
+        dt[k] = ""
+        return dt[k]
+    else
+        return ""
+    end
+end
+
+--[[ldx--
+
The next helper assigns a tree (or string). Usage:

+
+
+dt[k] = xml.assign(root) or xml.assign(dt,k,root)
+
+--ldx]]--
+
+function xml.assign(dt,k,root)
+    if dt and k then
+        dt[k] = (type(root) == "table" and xml.body(root)) or root
+        return dt[k]
+    else
+        return xml.body(root)
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-pth'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local concat, remove, insert = table.concat, table.remove, table.insert
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find
+
+--[[ldx--
+
This module can be used stand alone but also inside  in
+which case it hooks into the tracker code. Therefore we provide a few
+functions that set the tracers. Here we overload a previously defined
+function.

+--ldx]]--
+
+local trace_lpath = false
+
+if trackers then
+    trackers.register("xml.lpath", function(v) trace_lpath = v end)
+end
+
+local settrace = xml.settrace -- lxml-tab
+
+function xml.settrace(str,value)
+    if str == "lpath" then
+        trace_lpath = value or false
+    else
+        settrace(str,value) -- lxml-tab
+    end
+end
+
+--[[ldx--
+
We've now arrived at an intersting part: accessing the tree using a subset
+of  and since we're not compatible we call it . We
+will explain more about its usage in other documents.

+--ldx]]--
+
+local lpathcalls  = 0 -- statistics
+local lpathcached = 0 -- statistics
+
+xml.functions   = xml.functions   or { }
+xml.expressions = xml.expressions or { }
+
+local functions   = xml.functions
+local expressions = xml.expressions
+
+local actions = {
+    [10] = "stay",
+    [11] = "parent",
+    [12] = "subtree root",
+    [13] = "document root",
+    [14] = "any",
+    [15] = "many",
+    [16] = "initial",
+    [20] = "match",
+    [21] = "match one of",
+    [22] = "match and attribute eq",
+    [23] = "match and attribute ne",
+    [24] = "match one of and attribute eq",
+    [25] = "match one of and attribute ne",
+    [27] = "has attribute",
+    [28] = "has value",
+    [29] = "fast match",
+    [30] = "select",
+    [31] = "expression",
+    [40] = "processing instruction",
+}
+
+-- a rather dumb lpeg
+
+local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
+
+-- instead of using functions we just parse a few names which saves a call
+-- later on
+
+local lp_position  = P("position()") / "ps"
+local lp_index     = P("index()")    / "id"
+local lp_text      = P("text()")     / "tx"
+local lp_name      = P("name()")     / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')"
+local lp_tag       = P("tag()")      / "tg" -- (rt.tg or '')
+local lp_ns        = P("ns()")       / "ns" -- (rt.ns or '')
+local lp_noequal   = P("!=")         / "~=" + P("<=") + P(">=") + P("==")
+local lp_doequal   = P("=")          / "=="
+local lp_attribute = P("@")          / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')")
+
+local lp_lua_function  = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling
+    return t .. "("
+end
+
+local lp_function  = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling
+    if expressions[t] then
+        return "expressions." .. t .. "("
+    else
+        return "expressions.error("
+    end
+end
+
+local lparent  = lpeg.P("(")
+local rparent  = lpeg.P(")")
+local noparent = 1 - (lparent+rparent)
+local nested   = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent}
+local value    = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"}
+
+-- if we use a dedicated namespace then we don't need to pass rt and k
+
+local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s)
+    if expressions[t] then
+        if s then
+            return "expressions." .. t .. "(r,k," .. s ..")"
+        else
+            return "expressions." .. t .. "(r,k)"
+        end
+    else
+        return "expressions.error(" .. t .. ")"
+    end
+end
+
+local converter = lpeg.Cs ( (
+    lp_position +
+    lp_index +
+    lp_text + lp_name + -- fast one
+    lp_special +
+    lp_noequal + lp_doequal +
+    lp_attribute +
+    lp_lua_function +
+    lp_function +
+1 )^1 )
+
+-- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1
+
+local template = [[
+    return function(expressions,r,d,k,e,dt,ns,tg,id,ps)
+        local at, tx = e.at or { }, dt[1] or ""
+        return %s
+    end
+]]
+
+local function make_expression(str)
+    str = converter:match(str)
+    return str, loadstring(format(template,str))()
+end
+
+local map = { }
+
+local space                    = S(' \r\n\t')
+local squote                   = S("'")
+local dquote                   = S('"')
+local lparent                  = P('(')
+local rparent                  = P(')')
+local atsign                   = P('@')
+local lbracket                 = P('[')
+local rbracket                 = P(']')
+local exclam                   = P('!')
+local period                   = P('.')
+local eq                       = P('==') + P('=')
+local ne                       = P('<>') + P('!=')
+local star                     = P('*')
+local slash                    = P('/')
+local colon                    = P(':')
+local bar                      = P('|')
+local hat                      = P('^')
+local valid                    = R('az', 'AZ', '09') + S('_-')
+local name_yes                 = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:*
+local name_nop                 = Cc("*") * C(valid^1)
+local name                     = name_yes + name_nop
+local number                   = C((S('+-')^0 * R('09')^1)) / tonumber
+local names                    = (bar^0 * name)^1
+local morenames                = name * (bar^0 * name)^1
+local instructiontag           = P('pi::')
+local spacing                  = C(space^0)
+local somespace                = space^1
+local optionalspace            = space^0
+local text                     = C(valid^0)
+local value                    = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
+local empty                    = 1-slash
+
+local is_eq                    = lbracket * atsign * name * eq * value * rbracket
+local is_ne                    = lbracket * atsign * name * ne * value * rbracket
+local is_attribute             = lbracket * atsign * name              * rbracket
+local is_value                 = lbracket *          value             * rbracket
+local is_number                = lbracket *          number            * rbracket
+
+local nobracket                = 1-(lbracket+rbracket)  -- must be improved
+local is_expression            = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket
+
+local is_expression            = lbracket * (C(nobracket^1))/make_expression * rbracket
+
+local is_one                   =          name
+local is_none                  = exclam * name
+local is_one_of                =          ((lparent * names * rparent) + morenames)
+local is_none_of               = exclam * ((lparent * names * rparent) + morenames)
+
+local stay                     = (period                )
+local parent                   = (period * period       ) / function(   ) map[#map+1] = { 11             } end
+local subtreeroot              = (slash + hat           ) / function(   ) map[#map+1] = { 12             } end
+local documentroot             = (hat * hat             ) / function(   ) map[#map+1] = { 13             } end
+local any                      = (star                  ) / function(   ) map[#map+1] = { 14             } end
+local many                     = (star * star           ) / function(   ) map[#map+1] = { 15             } end
+local initial                  = (hat * hat * hat       ) / function(   ) map[#map+1] = { 16             } end
+
+local match                    = (is_one                ) / function(...) map[#map+1] = { 20, true , ... } end
+local match_one_of             = (is_one_of             ) / function(...) map[#map+1] = { 21, true , ... } end
+local dont_match               = (is_none               ) / function(...) map[#map+1] = { 20, false, ... } end
+local dont_match_one_of        = (is_none_of            ) / function(...) map[#map+1] = { 21, false, ... } end
+
+local match_and_eq             = (is_one     * is_eq    ) / function(...) map[#map+1] = { 22, true , ... } end
+local match_and_ne             = (is_one     * is_ne    ) / function(...) map[#map+1] = { 23, true , ... } end
+local dont_match_and_eq        = (is_none    * is_eq    ) / function(...) map[#map+1] = { 22, false, ... } end
+local dont_match_and_ne        = (is_none    * is_ne    ) / function(...) map[#map+1] = { 23, false, ... } end
+
+local match_one_of_and_eq      = (is_one_of  * is_eq    ) / function(...) map[#map+1] = { 24, true , ... } end
+local match_one_of_and_ne      = (is_one_of  * is_ne    ) / function(...) map[#map+1] = { 25, true , ... } end
+local dont_match_one_of_and_eq = (is_none_of * is_eq    ) / function(...) map[#map+1] = { 24, false, ... } end
+local dont_match_one_of_and_ne = (is_none_of * is_ne    ) / function(...) map[#map+1] = { 25, false, ... } end
+
+local has_attribute            = (is_one  * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end
+local has_value                = (is_one  * is_value    ) / function(...) map[#map+1] = { 28, true , ... } end
+local dont_has_attribute       = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end
+local dont_has_value           = (is_none * is_value    ) / function(...) map[#map+1] = { 28, false, ... } end
+local position                 = (is_one  * is_number   ) / function(...) map[#map+1] = { 30, true,  ... } end
+local dont_position            = (is_none * is_number   ) / function(...) map[#map+1] = { 30, false, ... } end
+
+local expression               = (is_one  * is_expression)/ function(...) map[#map+1] = { 31, true,  ... } end
+local dont_expression          = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end
+
+local self_expression          = (         is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
+                                                                          map[#map+1] = { 31, true,  "*", "*", ... } end
+local dont_self_expression     = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
+                                                                          map[#map+1] = { 31, false, "*", "*", ... } end
+
+local instruction              = (instructiontag * text ) / function(...) map[#map+1] = { 40,        ... } end
+local nothing                  = (empty                 ) / function(   ) map[#map+1] = { 15             } end -- 15 ?
+local crap                     = (1-slash)^1
+
+-- a few ugly goodies:
+
+local docroottag               = P('^^')             / function(   ) map[#map+1] = { 12             } end
+local subroottag               = P('^')              / function(   ) map[#map+1] = { 13             } end
+local roottag                  = P('root::')         / function(   ) map[#map+1] = { 12             } end
+local parenttag                = P('parent::')       / function(   ) map[#map+1] = { 11             } end
+local childtag                 = P('child::')
+local selftag                  = P('self::')
+
+-- there will be more and order will be optimized
+
+local selector = (
+    instruction +
+--  many + any + -- brrr, not here !
+    parent + stay +
+    dont_position + position +
+    dont_match_one_of_and_eq + dont_match_one_of_and_ne +
+    match_one_of_and_eq + match_one_of_and_ne +
+    dont_match_and_eq + dont_match_and_ne +
+    match_and_eq + match_and_ne +
+    dont_expression + expression +
+    dont_self_expression + self_expression +
+    has_attribute + has_value +
+    dont_match_one_of + match_one_of +
+    dont_match + match +
+    many + any +
+    crap + empty
+)
+
+local grammar = P { "startup",
+    startup  = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"),
+    followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1,
+}
+
+local function compose(str)
+    if not str or str == "" then
+        -- wildcard
+        return true
+    elseif str == '/' then
+        -- root
+        return false
+    else
+        map = { }
+        grammar:match(str)
+        if #map == 0 then
+            return true
+        else
+            local m = map[1][1]
+            if #map == 1 then
+                if m == 14 or m == 15 then
+                    -- wildcard
+                    return true
+                elseif m == 12 then
+                    -- root
+                    return false
+                end
+            elseif #map == 2 and m == 12 and map[2][1] == 20 then
+            --  return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } }
+                map[2][1] = 29
+                return { map[2] }
+            end
+            if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then
+                insert(map, 1, { 16 })
+            end
+        --  print(gsub(table.serialize(map),"[ \n]+"," "))
+            return map
+        end
+    end
+end
+
+local cache = { }
+
+function xml.lpath(pattern,trace)
+    lpathcalls = lpathcalls + 1
+    if type(pattern) == "string" then
+        local result = cache[pattern]
+        if result == nil then -- can be false which is valid -)
+            result = compose(pattern)
+            cache[pattern] = result
+            lpathcached = lpathcached + 1
+        end
+        if trace or trace_lpath then
+            xml.lshow(result)
+        end
+        return result
+    else
+        return pattern
+    end
+end
+
+function xml.cached_patterns()
+    return cache
+end
+
+--  we run out of locals (limited to 200)
+--
+--  local fallbackreport = (texio and texio.write) or io.write
+
+function xml.lshow(pattern,report)
+--      report = report or fallbackreport
+    report = report or (texio and texio.write) or io.write
+    local lp = xml.lpath(pattern)
+    if lp == false then
+        report(" -: root\n")
+    elseif lp == true then
+        report(" -: wildcard\n")
+    else
+        if type(pattern) == "string" then
+            report(format("pattern: %s\n",pattern))
+        end
+        for k=1,#lp do
+            local v = lp[k]
+            if #v > 1 then
+                local t = { }
+                for i=2,#v do
+                    local vv = v[i]
+                    if type(vv) == "string" then
+                        t[#t+1] = (vv ~= "" and vv) or "#"
+                    elseif type(vv) == "boolean" then
+                        t[#t+1] = (vv and "==") or "<>"
+                    end
+                end
+                report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," ")))
+            else
+                report(format("%2i: %s %s\n", k,v[1],actions[v[1]]))
+            end
+        end
+    end
+end
+
+function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e
+    local t = { ... }
+--      local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport
+    local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write
+    if e == nil then
+        report("\n")
+    elseif type(e) ~= "table" then
+        report(tostring(e))
+    elseif e.tg then
+        report(tostring(e) .. "\n")
+    else
+        for i=1,#e do
+            report(tostring(e[i]) .. "\n")
+        end
+    end
+end
+
+--[[ldx--
+
An  is converted to a table with instructions for traversing the
+tree. Hoever, simple cases are signaled by booleans. Because we don't know in
+advance what we want to do with the found element the handle gets three arguments:

+
+
+r : the root element of the data table
+d : the data table of the result
+t : the index in the data table of the result
+
+
+
 Access to the root and data table makes it possible to construct insert and delete
+functions.

+--ldx]]--
+
+local functions   = xml.functions
+local expressions = xml.expressions
+
+expressions.contains = string.find
+expressions.find     = string.find
+expressions.upper    = string.upper
+expressions.lower    = string.lower
+expressions.number   = tonumber
+expressions.boolean  = toboolean
+
+expressions.oneof = function(s,...) -- slow
+    local t = {...} for i=1,#t do if s == t[i] then return true end end return false
+end
+
+expressions.error = function(str)
+    xml.error_handler("unknown function in lpath expression",str or "?")
+    return false
+end
+
+functions.text = function(root,k,n) -- unchecked, maybe one deeper
+    local t = type(t)
+    if t == "string" then
+        return t
+    else -- todo n
+        local rdt = root.dt
+        return (rdt and rdt[k]) or root[k] or ""
+    end
+end
+
+functions.name = function(d,k,n) -- ns + tg
+    local found = false
+    n = n or 0
+    if not k then
+        -- not found
+    elseif n == 0 then
+        local dk = d[k]
+        found = dk and (type(dk) == "table") and dk
+    elseif n < 0 then
+        for i=k-1,1,-1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == -1 then
+                    found = di
+                    break
+                else
+                    n = n + 1
+                end
+            end
+        end
+    else
+        for i=k+1,#d,1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == 1 then
+                    found = di
+                    break
+                else
+                    n = n - 1
+                end
+            end
+        end
+    end
+    if found then
+        local ns, tg = found.rn or found.ns or "", found.tg
+        if ns ~= "" then
+            return ns .. ":" .. tg
+        else
+            return tg
+        end
+    else
+        return ""
+    end
+end
+
+functions.tag = function(d,k,n) -- only tg
+    local found = false
+    n = n or 0
+    if not k then
+        -- not found
+    elseif n == 0 then
+        local dk = d[k]
+        found = dk and (type(dk) == "table") and dk
+    elseif n < 0 then
+        for i=k-1,1,-1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == -1 then
+                    found = di
+                    break
+                else
+                    n = n + 1
+                end
+            end
+        end
+    else
+        for i=k+1,#d,1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == 1 then
+                    found = di
+                    break
+                else
+                    n = n - 1
+                end
+            end
+        end
+    end
+    return (found and found.tg) or ""
+end
+
+expressions.text = functions.text
+expressions.name = functions.name
+expressions.tag  = functions.tag
+
+local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces
+    if not root then -- error
+        return false
+    elseif pattern == false then -- root
+        handle(root,root.dt,root.ri)
+        return false
+    elseif pattern == true then -- wildcard
+        local rootdt = root.dt
+        if rootdt then
+            local start, stop, step = 1, #rootdt, 1
+            if reverse then
+                start, stop, step = stop, start, -1
+            end
+            for k=start,stop,step do
+                if handle(root,rootdt,root.ri or k)            then return false end
+                if not traverse(rootdt[k],true,handle,reverse) then return false end
+            end
+        end
+        return false
+    elseif root.dt then
+        index = index or 1
+        local action = pattern[index]
+        local command = action[1]
+        if command == 29 then -- fast case /oeps
+            local rootdt = root.dt
+            for k=1,#rootdt do
+                local e = rootdt[k]
+                local tg = e.tg
+                if e.tg then
+                    local ns = e.rn or e.ns
+                    local ns_a, tg_a = action[3], action[4]
+                    local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a)
+                    if not action[2] then matched = not matched end
+                    if matched then
+                        if handle(root,rootdt,k) then return false end
+                    end
+                end
+            end
+        elseif command == 11 then -- parent
+            local ep = root.__p__ or parent
+            if index < #pattern then
+                if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+            elseif handle(root,rootdt,k) then
+                return false
+            end
+        else
+            if (command == 16 or command == 12) and index == 1 then -- initial
+            --  wildcard = true
+                wildcard = command == 16 -- ok?
+                index = index + 1
+                action = pattern[index]
+                command = action and action[1] or 0 -- something is wrong
+            end
+            if command == 11 then -- parent
+                local ep = root.__p__ or parent
+                if index < #pattern then
+                    if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+                elseif handle(root,rootdt,k) then
+                    return false
+                end
+            else
+                local rootdt = root.dt
+                local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1
+                if command == 30 then
+                    if action[5] < 0 then
+                        start, stop, step = stop, start, -1
+                        dn = -1
+                    end
+                elseif reverse and index == #pattern then
+                    start, stop, step = stop, start, -1
+                end
+                local idx = 0
+                local hsh = { } -- this will slooow down the lot
+                for k=start,stop,step do -- we used to have functions for all but a case is faster
+                    local e = rootdt[k]
+                    local ns, tg = e.rn or e.ns, e.tg
+                    if tg then
+                     -- we can optimize this for simple searches, but it probably does not pay off
+                        hsh[tg] = (hsh[tg] or 0) + 1
+                        idx = idx + 1
+                        if command == 30 then
+                            local ns_a, tg_a = action[3], action[4]
+                            if tg == tg_a then
+                                matched = ns_a == "*" or ns == ns_a
+                            elseif tg_a == '*' then
+                                matched, multiple = ns_a == "*" or ns == ns_a, true
+                            else
+                                matched = false
+                            end
+                            if not action[2] then matched = not matched end
+                            if matched then
+                                n = n + dn
+                                if n == action[5] then
+                                    if index == #pattern then
+                                        if handle(root,rootdt,root.ri or k) then return false end
+                                    else
+                                        if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                    end
+                                    break
+                                end
+                            elseif wildcard then
+                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                            end
+                        else
+                            local matched, multiple = false, false
+                            if command == 20 then -- match
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                            elseif command == 21 then -- match one of
+                                multiple = true
+                                for i=3,#action,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
+                                    end
+                                end
+                                if not action[2] then matched = not matched end
+                            elseif command == 22 then -- eq
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                matched = matched and e.at[action[6]] == action[7]
+                            elseif command == 23 then -- ne
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = mached and e.at[action[6]] ~= action[7]
+                            elseif command == 24 then -- one of eq
+                                multiple = true
+                                for i=3,#action-2,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
+                                    end
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[#action-1]] == action[#action]
+                            elseif command == 25 then -- one of ne
+                                multiple = true
+                                for i=3,#action-2,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
+                                    end
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[#action-1]] ~= action[#action]
+                            elseif command == 27 then -- has attribute
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[5]]
+                            elseif command == 28 then -- has value
+                                local edt, ns_a, tg_a = e.dt, action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and edt and edt[1] == action[5]
+                            elseif command == 31 then
+                                local edt, ns_a, tg_a = e.dt, action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                if matched then
+                                    matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1)
+                                end
+                            end
+                            if matched then -- combine tg test and at test
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                    if wildcard then
+                                        if multiple then
+                                            if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                                        else
+                                         -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml
+                                            if not traverse(e,pattern,handle,reverse,index,root) then return false end
+                                        end
+                                    end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                end
+                            elseif command == 14 then -- any
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                end
+                            elseif command == 15 then -- many
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end
+                                end
+                            -- not here : 11
+                            elseif command == 11 then -- parent
+                                local ep = e.__p__ or parent
+                                if index < #pattern then
+                                    if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end
+                                elseif handle(root,rootdt,k) then
+                                    return false
+                                end
+                            elseif command == 40 and e.special and tg == "@pi@" then -- pi
+                                local pi = action[2]
+                                if pi ~= "" then
+                                    local pt = e.dt[1]
+                                    if pt and pt:find(pi) then
+                                        if handle(root,rootdt,k) then
+                                            return false
+                                        end
+                                    end
+                                elseif handle(root,rootdt,k) then
+                                    return false
+                                end
+                            elseif wildcard then
+                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                            end
+                        end
+                    else
+                        -- not here : 11
+                        if command == 11 then -- parent
+                            local ep = e.__p__ or parent
+                            if index < #pattern then
+                                if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+                            elseif handle(root,rootdt,k) then
+                                return false
+                            end
+                            break -- else loop
+                        end
+                    end
+                end
+            end
+        end
+    end
+    return true
+end
+
+xml.traverse = traverse
+
+--[[ldx--
+
Next come all kind of locators and manipulators. The most generic function here
+is xml.filter(root,pattern). All registers functions in the filters namespace
+can be path of a search path, as in:

+
+
+local r, d, k = xml.filter(root,"/a/b/c/position(4)"
+
+--ldx]]--
+
+local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert
+
+xml.filters = { }
+
+function xml.filters.default(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.attributes(root,pattern,arguments)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
+    local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
+    if ekat then
+        if arguments then
+            return ekat[arguments] or "", rt, dt, dk
+        else
+            return ekat, rt, dt, dk
+        end
+    else
+        return { }, rt, dt, dk
+    end
+end
+
+function xml.filters.reverse(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.count(root,pattern,everything)
+    local n = 0
+    traverse(root, lpath(pattern), function(r,d,t)
+        if everything or type(d[t]) == "table" then
+            n = n + 1
+        end
+    end)
+    return n
+end
+
+function xml.filters.elements(root, pattern) -- == all
+    local t = { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local e = d[k]
+        if e then
+            t[#t+1] = e
+        end
+    end)
+    return t
+end
+
+function xml.filters.texts(root, pattern)
+    local t = { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local e = d[k]
+        if e and e.dt then
+            t[#t+1] = e.dt
+        end
+    end)
+    return t
+end
+
+function xml.filters.first(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.last(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.index(root,pattern,arguments)
+    local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1
+    if i and i ~= 0 then
+        if i < 0 then
+            reverse, i = true, -i
+        end
+        traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse)
+        if i == 0 then
+            return dt and dt[dk], rt, dt, dk
+        end
+    end
+    return nil, nil, nil, nil
+end
+
+function xml.filters.attribute(root,pattern,arguments)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
+    local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
+    return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or ""
+end
+
+function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow
+    local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments)
+    if dtk then -- n
+        local dtkdt = dtk.dt
+        if not dtkdt then
+            return "", rt, dt, dk
+        elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then
+            return dtkdt[1], rt, dt, dk
+        else
+            return xml.tostring(dtkdt), rt, dt, dk
+        end
+    else
+        return "", rt, dt, dk
+    end
+end
+
+function xml.filters.tag(root,pattern,n)
+    local tag = ""
+    traverse(root, lpath(pattern), function(r,d,k)
+        tag = xml.functions.tag(d,k,n and tonumber(n))
+        return true
+    end)
+    return tag
+end
+
+function xml.filters.name(root,pattern,n)
+    local tag = ""
+    traverse(root, lpath(pattern), function(r,d,k)
+        tag = xml.functions.name(d,k,n and tonumber(n))
+        return true
+    end)
+    return tag
+end
+
+--[[ldx--
+
For splitting the filter function from the path specification, we can
+use string matching or lpeg matching. Here the difference in speed is
+neglectable but the lpeg variant is more robust.

+--ldx]]--
+
+--  not faster but hipper ... although ... i can't get rid of the trailing / in the path
+
+local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
+
+local slash     = P('/')
+local name      = (R("az","AZ","--","__"))^1
+local path      = C(((1-slash)^0 * slash)^1)
+local argument  = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" }
+local action    = Cc(1) * path * C(name) * argument
+local attribute = Cc(2) * path * P('@') * C(name)
+local direct    = Cc(3) * Cc("../*") * slash^0 * C(name) * argument
+
+local parser    = direct + action + attribute
+
+local filters          = xml.filters
+local attribute_filter = xml.filters.attributes
+local default_filter   = xml.filters.default
+
+-- todo: also hash, could be gc'd
+
+function xml.filter(root,pattern)
+    local kind, a, b, c = parser:match(pattern)
+    if kind == 1 or kind == 3 then
+        return (filters[b] or default_filter)(root,a,c)
+    elseif kind == 2 then
+        return attribute_filter(root,a,b)
+    else
+        return default_filter(root,pattern)
+    end
+end
+
+--~     slightly faster, but first we need a proper test file
+--~
+--~     local hash = { }
+--~
+--~     function xml.filter(root,pattern)
+--~         local h = hash[pattern]
+--~         if not h then
+--~             local kind, a, b, c = parser:match(pattern)
+--~             if kind == 1 then
+--~                 h = { kind, filters[b] or default_filter, a, b, c }
+--~             elseif kind == 2 then
+--~                 h = { kind, attribute_filter, a, b, c }
+--~             else
+--~                 h = { kind, default_filter, a, b, c }
+--~             end
+--~             hash[pattern] = h
+--~         end
+--~         local kind = h[1]
+--~         if kind == 1 then
+--~             return h[2](root,h[2],h[4])
+--~         elseif kind == 2 then
+--~             return h[2](root,h[2],h[3])
+--~         else
+--~             return h[2](root,pattern)
+--~         end
+--~     end
+
+--[[ldx--
+
The following functions collect elements and texts.

+--ldx]]--
+
+-- still somewhat bugged
+
+function xml.collect_elements(root, pattern, ignorespaces)
+    local rr, dd = { }, { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d and d[k]
+        if dk then
+            if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then
+                -- ignore
+            else
+                local n = #rr+1
+                rr[n], dd[n] = r, dk
+            end
+        end
+    end)
+    return dd, rr
+end
+
+function xml.collect_texts(root, pattern, flatten)
+    local t = { } -- no r collector
+    traverse(root, lpath(pattern), function(r,d,k)
+        if d then
+            local ek = d[k]
+            local tx = ek and ek.dt
+            if flatten then
+                if tx then
+                    t[#t+1] = xml.tostring(tx) or ""
+                else
+                    t[#t+1] = ""
+                end
+            else
+                t[#t+1] = tx or ""
+            end
+        else
+            t[#t+1] = ""
+        end
+    end)
+    return t
+end
+
+function xml.collect_tags(root, pattern, nonamespace)
+    local t = { }
+    xml.traverse(root, xml.lpath(pattern), function(r,d,k)
+        local dk = d and d[k]
+        if dk and type(dk) == "table" then
+            local ns, tg = e.ns, e.tg
+            if nonamespace then
+                t[#t+1] = tg -- if needed we can return an extra table
+            elseif ns == "" then
+                t[#t+1] = tg
+            else
+                t[#t+1] = ns .. ":" .. tg
+            end
+        end
+    end)
+    return #t > 0 and {}
+end
+
+--[[ldx--
+
Often using an iterators looks nicer in the code than passing handler
+functions. The  book describes how to use coroutines for that
+purpose (). This permits
+code like:

+
+
+for r, d, k in xml.elements(xml.load('text.xml'),"title") do
+    print(d[k])
+end
+
+
+
Which will print all the titles in the document. The iterator variant takes
+1.5 times the runtime of the function variant which is due to the overhead in
+creating the wrapper. So, instead of:

+
+
+function xml.filters.first(root,pattern)
+    for rt,dt,dk in xml.elements(root,pattern)
+        return dt and dt[dk], rt, dt, dk
+    end
+    return nil, nil, nil, nil
+end
+
+
+
We use the function variants in the filters.

+--ldx]]--
+
+local wrap, yield = coroutine.wrap, coroutine.yield
+
+function xml.elements(root,pattern,reverse)
+    return wrap(function() traverse(root, lpath(pattern), yield, reverse) end)
+end
+
+function xml.elements_only(root,pattern,reverse)
+    return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end)
+end
+
+function xml.each_element(root, pattern, handle, reverse)
+    local ok
+    traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse)
+    return ok
+end
+
+function xml.process_elements(root, pattern, handle)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dkdt = d[k].dt
+        if dkdt then
+            for i=1,#dkdt do
+                local v = dkdt[i]
+                if v.tg then handle(v) end
+            end
+        end
+    end)
+end
+
+function xml.process_attributes(root, pattern, handle)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local ek = d[k]
+        local a = ek.at or { }
+        handle(a)
+        if next(a) then -- next is faster than type (and >0 test)
+            ek.at = a
+        else
+            ek.at = nil
+        end
+    end)
+end
+
+--[[ldx--
+
We've now arrives at the functions that manipulate the tree.

+--ldx]]--
+
+function xml.inject_element(root, pattern, element, prepend)
+    if root and element then
+        local matches, collect = { }, nil
+        if type(element) == "string" then
+            element = convert(element,true)
+        end
+        if element then
+            collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
+            traverse(root, lpath(pattern), collect)
+            for i=1,#matches do
+                local m = matches[i]
+                local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil
+                if element.ri then
+                    element = element.dt[element.ri].dt
+                else
+                    element = element.dt
+                end
+                if r.ri then
+                    edt = r.dt[r.ri].dt
+                else
+                    edt = d and d[k] and d[k].dt
+                end
+                if edt then
+                    local be, af
+                    if prepend then
+                        be, af = xml.copy(element), edt
+                    else
+                        be, af = edt, xml.copy(element)
+                    end
+                    for i=1,#af do
+                        be[#be+1] = af[i]
+                    end
+                    if r.ri then
+                        r.dt[r.ri].dt = be
+                    else
+                        d[k].dt = be
+                    end
+                else
+                 -- r.dt = element.dt -- todo
+                end
+            end
+        end
+    end
+end
+
+-- todo: copy !
+
+function xml.insert_element(root, pattern, element, before) -- todo: element als functie
+    if root and element then
+        if pattern == "/" then
+            xml.inject_element(root, pattern, element, before)
+        else
+            local matches, collect = { }, nil
+            if type(element) == "string" then
+                element = convert(element,true)
+            end
+            if element and element.ri then
+                element = element.dt[element.ri]
+            end
+            if element then
+                collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
+                traverse(root, lpath(pattern), collect)
+                for i=#matches,1,-1 do
+                    local m = matches[i]
+                    local r, d, k, element = m[1], m[2], m[3], m[4]
+                    if not before then k = k + 1 end
+                    if element.tg then
+                        insert(d,k,element) -- untested
+--~                         elseif element.dt then
+--~                             for _,v in ipairs(element.dt) do -- i added
+--~                                 insert(d,k,v)
+--~                                 k = k + 1
+--~                             end
+--~                         end
+                    else
+                        local edt = element.dt
+                        if edt then
+                            for i=1,#edt do
+                                insert(d,k,edt[i])
+                                k = k + 1
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+xml.insert_element_after  =                 xml.insert_element
+xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end
+xml.inject_element_after  =                 xml.inject_element
+xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end
+
+function xml.delete_element(root, pattern)
+    local matches, deleted = { }, { }
+    local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end
+    traverse(root, lpath(pattern), collect)
+    for i=#matches,1,-1 do
+        local m = matches[i]
+        deleted[#deleted+1] = remove(m[2],m[3])
+    end
+    return deleted
+end
+
+function xml.replace_element(root, pattern, element)
+    if type(element) == "string" then
+        element = convert(element,true)
+    end
+    if element and element.ri then
+        element = element.dt[element.ri]
+    end
+    if element then
+        traverse(root, lpath(pattern), function(rm, d, k)
+            d[k] = element.dt -- maybe not clever enough
+        end)
+    end
+end
+
+local function load_data(name) -- == io.loaddata
+    local f, data = io.open(name), ""
+    if f then
+        data = f:read("*all",'b') -- 'b' ?
+        f:close()
+    end
+    return data
+end
+
+function xml.include(xmldata,pattern,attribute,recursive,loaddata)
+    -- parse="text" (default: xml), encoding="" (todo)
+    -- attribute = attribute or 'href'
+    pattern = pattern or 'include'
+    loaddata = loaddata or load_data
+    local function include(r,d,k)
+        local ek, name = d[k], nil
+        if not attribute or attribute == "" then
+            local ekdt = ek.dt
+            name = (type(ekdt) == "table" and ekdt[1]) or ekdt
+        end
+        if not name then
+            if ek.at then
+                for a in gmatch(attribute or "href","([^|]+)") do
+                    name = ek.at[a]
+                    if name then break end
+                end
+            end
+        end
+        local data = (name and name ~= "" and loaddata(name)) or ""
+        if data == "" then
+            xml.empty(d,k)
+        elseif ek.at["parse"] == "text" then -- for the moment hard coded
+            d[k] = xml.escaped(data)
+        else
+            local xi = xml.convert(data)
+            if not xi then
+                xml.empty(d,k)
+            else
+                if recursive then
+                    xml.include(xi,pattern,attribute,recursive,loaddata)
+                end
+                xml.assign(d,k,xi)
+            end
+        end
+    end
+    xml.each_element(xmldata, pattern, include)
+end
+
+function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space !
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dkdt = d[k].dt
+        if dkdt then -- can be optimized
+            local t = { }
+            for i=1,#dkdt do
+                local str = dkdt[i]
+                if type(str) == "string" then
+
+                    if str == "" then
+                        -- stripped
+                    else
+                        if nolines then
+                            str = gsub(str,"[ \n\r\t]+"," ")
+                        end
+                        if str == "" then
+                            -- stripped
+                        else
+                            t[#t+1] = str
+                        end
+                    end
+                else
+                    t[#t+1] = str
+                end
+            end
+            d[k].dt = t
+        end
+    end)
+end
+
+local function rename_space(root, oldspace, newspace) -- fast variant
+    local ndt = #root.dt
+    for i=1,ndt or 0 do
+        local e = root[i]
+        if type(e) == "table" then
+            if e.ns == oldspace then
+                e.ns = newspace
+                if e.rn then
+                    e.rn = newspace
+                end
+            end
+            local edt = e.dt
+            if edt then
+                rename_space(edt, oldspace, newspace)
+            end
+        end
+    end
+end
+
+xml.rename_space = rename_space
+
+function xml.remap_tag(root, pattern, newtg)
+    traverse(root, lpath(pattern), function(r,d,k)
+        d[k].tg = newtg
+    end)
+end
+function xml.remap_namespace(root, pattern, newns)
+    traverse(root, lpath(pattern), function(r,d,k)
+        d[k].ns = newns
+    end)
+end
+function xml.check_namespace(root, pattern, newns)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d[k]
+        if (not dk.rn or dk.rn == "") and dk.ns == "" then
+            dk.rn = newns
+        end
+    end)
+end
+function xml.remap_name(root, pattern, newtg, newns, newrn)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d[k]
+        dk.tg = newtg
+        dk.ns = newns
+        dk.rn = newrn
+    end)
+end
+
+function xml.filters.found(root,pattern,check_content)
+    local found = false
+    traverse(root, lpath(pattern), function(r,d,k)
+        if check_content then
+            local dk = d and d[k]
+            found = dk and dk.dt and next(dk.dt) and true
+        else
+            found = true
+        end
+        return true
+    end)
+    return found
+end
+
+--[[ldx--
+
Here are a few synonyms.

+--ldx]]--
+
+xml.filters.position = xml.filters.index
+
+xml.count    = xml.filters.count
+xml.index    = xml.filters.index
+xml.position = xml.filters.index
+xml.first    = xml.filters.first
+xml.last     = xml.filters.last
+xml.found    = xml.filters.found
+
+xml.each     = xml.each_element
+xml.process  = xml.process_element
+xml.strip    = xml.strip_whitespace
+xml.collect  = xml.collect_elements
+xml.all      = xml.collect_elements
+
+xml.insert   = xml.insert_element_after
+xml.inject   = xml.inject_element_after
+xml.after    = xml.insert_element_after
+xml.before   = xml.insert_element_before
+xml.delete   = xml.delete_element
+xml.replace  = xml.replace_element
+
+--[[ldx--
+
The following helper functions best belong to the lmxl-ini
+module. Some are here because we need then in the mk
+document and other manuals, others came up when playing with
+this module. Since this module is also used in  we've
+put them here instead of loading mode modules there then needed.

+--ldx]]--
+
+function xml.gsub(t,old,new)
+    local dt = t.dt
+    if dt then
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "string" then
+                dt[k] = gsub(v,old,new)
+            else
+                xml.gsub(v,old,new)
+            end
+        end
+    end
+end
+
+function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual
+    if d and k and d[k-1] and type(d[k-1]) == "string" then
+        local s = d[k-1]:match("\n(%s+)")
+        xml.gsub(dk,"\n"..string.rep(" ",#s),"\n")
+    end
+end
+
+function xml.serialize_path(root,lpath,handle)
+    local dk, r, d, k = xml.first(root,lpath)
+    dk = xml.copy(dk)
+    xml.strip_leading_spaces(dk,d,k)
+    xml.serialize(dk,handle)
+end
+
+--~ xml.escapes   = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' }
+--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end
+
+--~ function xml.escaped  (str) return (gsub(str,"(.)"   , xml.escapes  )) end
+--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end
+--~ function xml.cleansed (str) return (gsub(str,"<.->"  , ''           )) end -- "%b<>"
+
+local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
+
+-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
+--
+-- 1021:0335:0287:0247
+
+-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
+--
+-- 1559:0257:0288:0190 (last one suggested by roberto)
+
+--    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
+--    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+local normal  = (1 - S("<&>"))^0
+local special = P("<")/"<" + P(">")/">" + P("&")/"&"
+local escaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
+
+--    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+--    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
+local normal    = (1 - S"&")^0
+local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
+local unescaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
+
+local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
+
+function xml.escaped  (str) return escaped  :match(str) end
+function xml.unescaped(str) return unescaped:match(str) end
+function xml.cleansed (str) return cleansed :match(str) end
+
+function xml.join(t,separator,lastseparator)
+    if #t > 0 then
+        local result = { }
+        for k,v in pairs(t) do
+            result[k] = xml.tostring(v)
+        end
+        if lastseparator then
+            return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result]
+        else
+            return concat(result,separator)
+        end
+    else
+        return ""
+    end
+end
+
+function xml.statistics()
+    return {
+        lpathcalls = lpathcalls,
+        lpathcached = lpathcached,
+    }
+end
+
+--  xml.set_text_cleanup(xml.show_text_entities)
+--  xml.set_text_cleanup(xml.resolve_text_entities)
+
+--~ xml.lshow("/../../../a/(b|c)[@d='e']/f")
+--~ xml.lshow("/../../../a/!(b|c)[@d='e']/f")
+--~ xml.lshow("/../../../a/!b[@d!='e']/f")
+
+--~ x = xml.convert([[
+--~     
+--~         01
+--~         02
+--~         03
+--~         OK
+--~         05
+--~         06
+--~         ALSO OK
+--~     
+--~ ]])
+
+--~ xml.settrace("lpath",true)
+
+--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']"))
+--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]"))
+--~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']"))
+--~ xml.xshow(xml.all  (x,"b[number(@n)>2 and number(@n)<6]"))
+--~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]"))
+
+--~ str = [[
+--~ 
+--~ 
+--~     my secret
+--~ 
+--~ ]]
+
+--~ x = xml.convert([[
+--~     0102xx03OK
+--~ ]])
+--~ xml.xshow(xml.first(x,"b[tag(2) == 'x']"))
+--~ xml.xshow(xml.first(x,"b[tag(1) == 'x']"))
+--~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']"))
+--~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']"))
+
+--~ print(xml.filter(x,"b/tag(2)"))
+--~ print(xml.filter(x,"b/tag(1)"))
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-ent'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, gsub, find = string.format, string.gsub, string.find
+local utfchar = unicode.utf8.char
+
+--[[ldx--
+
We provide (at least here) two entity handlers. The more extensive
+resolver consults a hash first, tries to convert to  next,
+and finaly calls a handler when defines. When this all fails, the
+original entity is returned.

+--ldx]]--
+
+xml.entities = xml.entities or { } -- xml.entity_handler == function
+
+function xml.entity_handler(e)
+    return format("[%s]",e)
+end
+
+local function toutf(s)
+    return utfchar(tonumber(s,16))
+end
+
+local function utfize(root)
+    local d = root.dt
+    for k=1,#d do
+        local dk = d[k]
+        if type(dk) == "string" then
+        --  test prevents copying if no match
+            if find(dk,"&#x.-;") then
+                d[k] = gsub(dk,"&#x(.-);",toutf)
+            end
+        else
+            utfize(dk)
+        end
+    end
+end
+
+xml.utfize = utfize
+
+local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks
+    if find(e,"^#x") then
+        return utfchar(tonumber(e:sub(3),16))
+    elseif find(e,"^#") then
+        return utfchar(tonumber(e:sub(2)))
+    else
+        local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded)
+        if ee then
+            return ee
+        else
+            local h = xml.entity_handler
+            return (h and h(e)) or "&" .. e .. ";"
+        end
+    end
+end
+
+local function resolve_entities(root)
+    if not root.special or root.tg == "@rt@" then
+        local d = root.dt
+        for k=1,#d do
+            local dk = d[k]
+            if type(dk) == "string" then
+                if find(dk,"&.-;") then
+                    d[k] = gsub(dk,"&(.-);",resolve)
+                end
+            else
+                resolve_entities(dk)
+            end
+        end
+    end
+end
+
+xml.resolve_entities = resolve_entities
+
+function xml.utfize_text(str)
+    if find(str,"&#") then
+        return (gsub(str,"&#x(.-);",toutf))
+    else
+        return str
+    end
+end
+
+function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline
+    if find(str,"&") then
+        return (gsub(str,"&(.-);",resolve))
+    else
+        return str
+    end
+end
+
+function xml.show_text_entities(str)
+    if find(str,"&") then
+        return (gsub(str,"&(.-);","[%1]"))
+    else
+        return str
+    end
+end
+
+-- experimental, this will be done differently
+
+function xml.merge_entities(root)
+    local documententities = root.entities
+    local allentities = xml.entities
+    if documententities then
+        for k, v in next, documententities do
+            allentities[k] = v
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-mis'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local concat = table.concat
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, gsub = string.format, string.gsub
+
+--[[ldx--
+
The following helper functions best belong to the lmxl-ini
+module. Some are here because we need then in the mk
+document and other manuals, others came up when playing with
+this module. Since this module is also used in  we've
+put them here instead of loading mode modules there then needed.

+--ldx]]--
+
+function xml.gsub(t,old,new)
+    local dt = t.dt
+    if dt then
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "string" then
+                dt[k] = gsub(v,old,new)
+            else
+                xml.gsub(v,old,new)
+            end
+        end
+    end
+end
+
+function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual
+    if d and k and d[k-1] and type(d[k-1]) == "string" then
+        local s = d[k-1]:match("\n(%s+)")
+        xml.gsub(dk,"\n"..string.rep(" ",#s),"\n")
+    end
+end
+
+function xml.serialize_path(root,lpath,handle)
+    local dk, r, d, k = xml.first(root,lpath)
+    dk = xml.copy(dk)
+    xml.strip_leading_spaces(dk,d,k)
+    xml.serialize(dk,handle)
+end
+
+--~ xml.escapes   = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' }
+--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end
+
+--~ function xml.escaped  (str) return (gsub(str,"(.)"   , xml.escapes  )) end
+--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end
+--~ function xml.cleansed (str) return (gsub(str,"<.->"  , ''           )) end -- "%b<>"
+
+local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
+
+-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
+--
+-- 1021:0335:0287:0247
+
+-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
+--
+-- 1559:0257:0288:0190 (last one suggested by roberto)
+
+--    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
+--    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+local normal  = (1 - S("<&>"))^0
+local special = P("<")/"<" + P(">")/">" + P("&")/"&"
+local escaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
+
+--    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+--    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
+local normal    = (1 - S"&")^0
+local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
+local unescaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
+
+local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
+
+xml.escaped_pattern   = escaped
+xml.unescaped_pattern = unescaped
+xml.cleansed_pattern  = cleansed
+
+function xml.escaped  (str) return escaped  :match(str) end
+function xml.unescaped(str) return unescaped:match(str) end
+function xml.cleansed (str) return cleansed :match(str) end
+
+function xml.join(t,separator,lastseparator)
+    if #t > 0 then
+        local result = { }
+        for k,v in pairs(t) do
+            result[k] = xml.tostring(v)
+        end
+        if lastseparator then
+            return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result]
+        else
+            return concat(result,separator)
+        end
+    else
+        return ""
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-tra'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- the  tag is kind of generic and used for functions that are not
+-- bound to a variable, like node.new, node.copy etc (contrary to for instance
+-- node.has_attribute which is bound to a has_attribute local variable in mkiv)
+
+debugger = debugger or { }
+
+local counters = { }
+local names = { }
+local getinfo = debug.getinfo
+local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch
+
+-- one
+
+local function hook()
+    local f = getinfo(2,"f").func
+    local n = getinfo(2,"Sn")
+--  if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end
+    if f then
+        local cf = counters[f]
+        if cf == nil then
+            counters[f] = 1
+            names[f] = n
+        else
+            counters[f] = cf + 1
+        end
+    end
+end
+local function getname(func)
+    local n = names[func]
+    if n then
+        if n.what == "C" then
+            return n.name or ''
+        else
+            -- source short_src linedefined what name namewhat nups func
+            local name = n.name or n.namewhat or n.what
+            if not name or name == "" then name = "?" end
+            return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name)
+        end
+    else
+        return "unknown"
+    end
+end
+function debugger.showstats(printer,threshold)
+    printer   = printer or texio.write or print
+    threshold = threshold or 0
+    local total, grandtotal, functions = 0, 0, 0
+    printer("\n") -- ugly but ok
+ -- table.sort(counters)
+    for func, count in pairs(counters) do
+        if count > threshold then
+            local name = getname(func)
+            if not name:find("for generator") then
+                printer(format("%8i  %s", count, name))
+                total = total + count
+            end
+        end
+        grandtotal = grandtotal + count
+        functions = functions + 1
+    end
+    printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+end
+
+-- two
+
+--~ local function hook()
+--~     local n = getinfo(2)
+--~     if n.what=="C" and not n.name then
+--~         local f = tostring(debug.traceback())
+--~         local cf = counters[f]
+--~         if cf == nil then
+--~             counters[f] = 1
+--~             names[f] = n
+--~         else
+--~             counters[f] = cf + 1
+--~         end
+--~     end
+--~ end
+--~ function debugger.showstats(printer,threshold)
+--~     printer   = printer or texio.write or print
+--~     threshold = threshold or 0
+--~     local total, grandtotal, functions = 0, 0, 0
+--~     printer("\n") -- ugly but ok
+--~  -- table.sort(counters)
+--~     for func, count in pairs(counters) do
+--~         if count > threshold then
+--~             printer(format("%8i  %s", count, func))
+--~             total = total + count
+--~         end
+--~         grandtotal = grandtotal + count
+--~         functions = functions + 1
+--~     end
+--~     printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+--~ end
+
+-- rest
+
+function debugger.savestats(filename,threshold)
+    local f = io.open(filename,'w')
+    if f then
+        debugger.showstats(function(str) f:write(str) end,threshold)
+        f:close()
+    end
+end
+
+function debugger.enable()
+    debug.sethook(hook,"c")
+end
+
+function debugger.disable()
+    debug.sethook()
+--~ counters[debug.getinfo(2,"f").func] = nil
+end
+
+function debugger.tracing()
+    local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0
+    if n > 0 then
+        function debugger.tracing() return true  end ; return true
+    else
+        function debugger.tracing() return false end ; return false
+    end
+end
+
+--~ debugger.enable()
+
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+
+--~ debugger.disable()
+
+--~ print("")
+--~ debugger.showstats()
+--~ print("")
+--~ debugger.showstats(print,3)
+
+trackers = trackers or { }
+
+local data, done = { }, { }
+
+local function set(what,value)
+    for w in gmatch(lower(what),"[^, ]+") do
+        for d, f in next, data do
+            if done[d] then
+                -- prevent recursion due to wildcards
+            elseif find(d,w) then
+                done[d] = true
+                for i=1,#f do
+                    f[i](value)
+                end
+            end
+        end
+    end
+end
+
+local function reset()
+    for d, f in next, data do
+        for i=1,#f do
+            f[i](false)
+        end
+    end
+end
+
+function trackers.register(what,...)
+    what = lower(what)
+    local w = data[what]
+    if not w then
+        w = { }
+        data[what] = w
+    end
+    for _, fnc in next, { ... } do
+        local typ = type(fnc)
+        if typ == "function" then
+            w[#w+1] = fnc
+        elseif typ == "string" then
+            w[#w+1] = function(value) set(fnc,value,nesting) end
+        end
+    end
+end
+
+function trackers.enable(what)
+    done = { }
+    set(what,true)
+end
+
+function trackers.disable(what)
+    done = { }
+    if not what or what == "" then
+        trackers.reset(what)
+    else
+        set(what,false)
+    end
+end
+
+function trackers.reset(what)
+    done = { }
+    reset()
+end
+
+function trackers.list() -- pattern
+    local list = table.sortedkeys(data)
+    local user, system = { }, { }
+    for l=1,#list do
+        local what = list[l]
+        if find(what,"^%*") then
+            system[#system+1] = what
+        else
+            user[#user+1] = what
+        end
+    end
+    return user, system
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-env'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- A former version provided functionality for non embeded core
+-- scripts i.e. runtime library loading. Given the amount of
+-- Lua code we use now, this no longer makes sense. Much of this
+-- evolved before bytecode arrays were available and so a lot of
+-- code has disappeared already.
+
+local trace_verbose  = false  trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+
+local format = string.format
+
+-- precautions
+
+os.setlocale(nil,nil) -- useless feature and even dangerous in luatex
+
+function os.setlocale()
+    -- no way you can mess with it
+end
+
+-- dirty tricks
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then
+    profiler.start("luatex-profile.log")
+end
+
+-- environment
+
+environment             = environment or { }
+environment.arguments   = { }
+environment.files       = { }
+environment.sortedflags = nil
+
+if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end
+if not environment.version or environment.version == "" then             environment.version = "unknown"   end
+if not environment.jobname                              then             environment.jobname = "unknown"   end
+
+function environment.initialize_arguments(arg)
+    local arguments, files = { }, { }
+    environment.arguments, environment.files, environment.sortedflags = arguments, files, nil
+    for index, argument in pairs(arg) do
+        if index > 0 then
+            local flag, value = argument:match("^%-+(.+)=(.-)$")
+            if flag then
+                arguments[flag] = string.unquote(value or "")
+            else
+                flag = argument:match("^%-+(.+)")
+                if flag then
+                    arguments[flag] = true
+                else
+                    files[#files+1] = argument
+                end
+            end
+        end
+    end
+    environment.ownname = environment.ownname or arg[0] or 'unknown.lua'
+end
+
+function environment.setargument(name,value)
+    environment.arguments[name] = value
+end
+
+-- todo: defaults, better checks e.g on type (boolean versus string)
+--
+-- tricky: too many hits when we support partials unless we add
+-- a registration of arguments so from now on we have 'partial'
+
+function environment.argument(name,partial)
+    local arguments, sortedflags = environment.arguments, environment.sortedflags
+    if arguments[name] then
+        return arguments[name]
+    elseif partial then
+        if not sortedflags then
+            sortedflags = { }
+            for _,v in pairs(table.sortedkeys(arguments)) do
+                sortedflags[#sortedflags+1] = "^" .. v
+            end
+            environment.sortedflags = sortedflags
+        end
+        -- example of potential clash: ^mode ^modefile
+        for _,v in ipairs(sortedflags) do
+            if name:find(v) then
+                return arguments[v:sub(2,#v)]
+            end
+        end
+    end
+    return nil
+end
+
+function environment.split_arguments(separator) -- rather special, cut-off before separator
+    local done, before, after = false, { }, { }
+    for _,v in ipairs(environment.original_arguments) do
+        if not done and v == separator then
+            done = true
+        elseif done then
+            after[#after+1] = v
+        else
+            before[#before+1] = v
+        end
+    end
+    return before, after
+end
+
+function environment.reconstruct_commandline(arg,noquote)
+    arg = arg or environment.original_arguments
+    if noquote and #arg == 1 then
+        local a = arg[1]
+        a = resolvers.resolve(a)
+        a = a:unquote()
+        return a
+    elseif next(arg) then
+        local result = { }
+        for _,a in ipairs(arg) do -- ipairs 1 .. #n
+            a = resolvers.resolve(a)
+            a = a:unquote()
+            a = a:gsub('"','\\"') -- tricky
+            if a:find(" ") then
+                result[#result+1] = a:quote()
+            else
+                result[#result+1] = a
+            end
+        end
+        return table.join(result," ")
+    else
+        return ""
+    end
+end
+
+if arg then
+
+    -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later)
+    local newarg, instring = { }, false
+
+    for index, argument in ipairs(arg) do
+        if argument:find("^\"") then
+            newarg[#newarg+1] = argument:gsub("^\"","")
+            if not argument:find("\"$") then
+                instring = true
+            end
+        elseif argument:find("\"$") then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","")
+            instring = false
+        elseif instring then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument
+        else
+            newarg[#newarg+1] = argument
+        end
+    end
+    for i=1,-5,-1 do
+        newarg[i] = arg[i]
+    end
+
+    environment.initialize_arguments(newarg)
+    environment.original_arguments = newarg
+    environment.raw_arguments = arg
+
+    arg = { } -- prevent duplicate handling
+
+end
+
+-- weird place ... depends on a not yet loaded module
+
+function environment.texfile(filename)
+    return resolvers.find_file(filename,'tex')
+end
+
+function environment.luafile(filename)
+    local resolved = resolvers.find_file(filename,'tex') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    resolved = resolvers.find_file(filename,'texmfscripts') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    return resolvers.find_file(filename,'luatexlibs') or ""
+end
+
+environment.loadedluacode = loadfile -- can be overloaded
+
+--~ function environment.loadedluacode(name)
+--~     if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then
+--~         local chunk = loadstring(io.loaddata("texluac.luc"))
+--~         os.remove("texluac.luc")
+--~         return chunk
+--~     else
+--~         environment.loadedluacode = loadfile -- can be overloaded
+--~         return loadfile(name)
+--~     end
+--~ end
+
+function environment.luafilechunk(filename) -- used for loading lua bytecode in the format
+    filename = file.replacesuffix(filename, "lua")
+    local fullname = environment.luafile(filename)
+    if fullname and fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading file %s", fullname)
+        end
+        return environment.loadedluacode(fullname)
+    else
+        if trace_verbose then
+            logs.report("fileio","unknown file %s", filename)
+        end
+        return nil
+    end
+end
+
+-- the next ones can use the previous ones / combine
+
+function environment.loadluafile(filename, version)
+    local lucname, luaname, chunk
+    local basename = file.removesuffix(filename)
+    if basename == filename then
+        lucname, luaname = basename .. ".luc",  basename .. ".lua"
+    else
+        lucname, luaname = nil, basename -- forced suffix
+    end
+    -- when not overloaded by explicit suffix we look for a luc file first
+    local fullname = (lucname and environment.luafile(lucname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+    end
+    if chunk then
+        assert(chunk)()
+        if version then
+            -- we check of the version number of this chunk matches
+            local v = version -- can be nil
+            if modules and modules[filename] then
+                v = modules[filename].version -- new method
+            elseif versions and versions[filename] then
+                v = versions[filename]        -- old method
+            end
+            if v == version then
+                return true
+            else
+                if trace_verbose then
+                    logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version)
+                end
+                environment.loadluafile(filename)
+            end
+        else
+            return true
+        end
+    end
+    fullname = (luaname and environment.luafile(luaname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+        if not chunk then
+            if verbose then
+                logs.report("fileio","unknown file %s", filename)
+            end
+        else
+            assert(chunk)()
+            return true
+        end
+    end
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-inf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+local statusinfo, n, registered = { }, 0, { }
+
+statistics = statistics or { }
+
+statistics.enable    = true
+statistics.threshold = 0.05
+
+-- timing functions
+
+local clock = os.gettimeofday or os.clock
+
+function statistics.hastimer(instance)
+    return instance and instance.starttime
+end
+
+function statistics.starttiming(instance)
+    if instance then
+        local it = instance.timing
+        if not it then
+            it = 0
+        end
+        if it == 0 then
+            instance.starttime = clock()
+            if not instance.loadtime then
+                instance.loadtime = 0
+            end
+        end
+        instance.timing = it + 1
+    end
+end
+
+function statistics.stoptiming(instance, report)
+    if instance then
+        local it = instance.timing
+        if it > 1 then
+            instance.timing = it - 1
+        else
+            local starttime = instance.starttime
+            if starttime then
+                local stoptime = clock()
+                local loadtime = stoptime - starttime
+                instance.stoptime = stoptime
+                instance.loadtime = instance.loadtime + loadtime
+                if report then
+                    statistics.report("load time %0.3f",loadtime)
+                end
+                instance.timing = 0
+                return loadtime
+            end
+        end
+    end
+    return 0
+end
+
+function statistics.elapsedtime(instance)
+    return format("%0.3f",(instance and instance.loadtime) or 0)
+end
+
+function statistics.elapsedindeed(instance)
+    local t = (instance and instance.loadtime) or 0
+    return t > statistics.threshold
+end
+
+-- general function
+
+function statistics.register(tag,fnc)
+    if statistics.enable and type(fnc) == "function" then
+        local rt = registered[tag] or (#statusinfo + 1)
+        statusinfo[rt] = { tag, fnc }
+        registered[tag] = rt
+        if #tag > n then n = #tag end
+    end
+end
+
+function statistics.show(reporter)
+    if statistics.enable then
+        if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end
+        -- this code will move
+        local register = statistics.register
+        register("luatex banner", function()
+            return string.lower(status.banner)
+        end)
+        register("control sequences", function()
+            return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra)
+        end)
+        register("callbacks", function()
+            local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
+            return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
+        end)
+        register("current memory usage", statistics.memused)
+        register("runtime",statistics.runtime)
+--         --
+        for i=1,#statusinfo do
+            local s = statusinfo[i]
+            local r = s[2]()
+            if r then
+                reporter(s[1],r,n)
+            end
+        end
+        statistics.enable = false
+    end
+end
+
+function statistics.show_job_stat(tag,data,n)
+    texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data))
+end
+
+function statistics.memused() -- no math.round yet -)
+    local round = math.round or math.floor
+    return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000))
+end
+
+if statistics.runtime then
+    -- already loaded and set
+elseif luatex and luatex.starttime then
+    statistics.starttime = luatex.starttime
+    statistics.loadtime = 0
+    statistics.timing = 0
+else
+    statistics.starttiming(statistics)
+end
+
+function statistics.runtime()
+    statistics.stoptiming(statistics)
+    return statistics.formatruntime(statistics.elapsedtime(statistics))
+end
+
+function statistics.formatruntime(runtime)
+    return format("%s seconds", statistics.elapsedtime(statistics))
+end
+
+function statistics.timed(action,report)
+    local timer = { }
+    report = report or logs.simple
+    statistics.starttiming(timer)
+    action()
+    statistics.stoptiming(timer)
+    report("total runtime: %s",statistics.elapsedtime(timer))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-log'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this is old code that needs an overhaul
+
+local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+
+if texlua then
+    write_nl = print
+    write    = io.write
+end
+
+--[[ldx--
+
This is a prelude to a more extensive logging module. For the sake
+of parsing log files, in addition to the standard logging we will
+provide an  structured file. Actually, any logging that
+is hooked into callbacks will be \XML\ by default.

+--ldx]]--
+
+logs     = logs     or { }
+logs.xml = logs.xml or { }
+logs.tex = logs.tex or { }
+
+--[[ldx--
+
This looks pretty ugly but we need to speed things up a bit.

+--ldx]]--
+
+logs.moreinfo = [[
+more information about ConTeXt and the tools that come with it can be found at:
+
+maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
+webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
+wiki     : http://contextgarden.net
+]]
+
+logs.levels = {
+    ['error']   = 1,
+    ['warning'] = 2,
+    ['info']    = 3,
+    ['debug']   = 4,
+}
+
+logs.functions = {
+    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct',
+    'start_run', 'stop_run',
+    'start_page_number', 'stop_page_number',
+    'report_output_pages', 'report_output_log',
+    'report_tex_stat', 'report_job_stat',
+    'show_open', 'show_close', 'show_load',
+}
+
+logs.tracers = {
+}
+
+logs.level = 0
+logs.mode  = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"))
+
+function logs.set_level(level)
+    logs.level = logs.levels[level] or level
+end
+
+function logs.set_method(method)
+    for _, v in next, logs.functions do
+        logs[v] = logs[method][v] or function() end
+    end
+end
+
+-- tex logging
+
+function logs.tex.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(category .. " | " .. format(fmt,...))
+    else
+        write_nl(category .. " |")
+    end
+end
+
+function logs.tex.line(fmt,...) -- new
+    if fmt then
+        write_nl(format(fmt,...))
+    else
+        write_nl("")
+    end
+end
+
+local texcount = tex and tex.count
+
+function logs.tex.start_page_number()
+    local real, user, sub = texcount[0], texcount[1], texcount[2]
+    if real > 0 then
+        if user > 0 then
+            if sub > 0 then
+                write(format("[%s.%s.%s",real,user,sub))
+            else
+                write(format("[%s.%s",real,user))
+            end
+        else
+            write(format("[%s",real))
+        end
+    else
+        write("[-")
+    end
+end
+
+function logs.tex.stop_page_number()
+    write("]")
+end
+
+logs.tex.report_job_stat = statistics.show_job_stat
+
+-- xml logging
+
+function logs.xml.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",category,format(fmt,...)))
+    else
+        write_nl(format("",category))
+    end
+end
+function logs.xml.line(fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",format(fmt,...)))
+    else
+        write_nl("")
+    end
+end
+
+function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
+function logs.xml.stop () if logs.level > 0 then tw("") end end
+function logs.xml.push () if logs.level > 0 then tw("" ) end end
+
+function logs.xml.start_run()
+    write_nl("")
+    write_nl("") --  xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng'
+    write_nl("")
+end
+
+function logs.xml.stop_run()
+    write_nl("")
+end
+
+function logs.xml.start_page_number()
+    write_nl(format("
")
+    write_nl("")
+end
+
+function logs.xml.report_output_pages(p,b)
+    write_nl(format("", p))
+    write_nl(format("", b))
+    write_nl("")
+end
+
+function logs.xml.report_output_log()
+end
+
+function logs.xml.report_tex_stat(k,v)
+    texiowrite_nl("log",""..tostring(v).."")
+end
+
+local level = 0
+
+function logs.xml.show_open(name)
+    level = level + 1
+    texiowrite_nl(format("",level,name))
+end
+
+function logs.xml.show_close(name)
+    texiowrite(" ")
+    level = level - 1
+end
+
+function logs.xml.show_load(name)
+    texiowrite_nl(format("",level+1,name))
+end
+
+--
+
+local name, banner = 'report', 'context'
+
+local function report(category,fmt,...)
+    if fmt then
+        write_nl(format("%s | %s: %s",name,category,format(fmt,...)))
+    elseif category then
+        write_nl(format("%s | %s",name,category))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+local function simple(fmt,...)
+    if fmt then
+        write_nl(format("%s | %s",name,format(fmt,...)))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+function logs.setprogram(_name_,_banner_,_verbose_)
+    name, banner = _name_, _banner_
+    if _verbose_ then
+        trackers.enable("resolvers.verbose")
+    end
+    logs.set_method("tex")
+    logs.report = report -- also used in libraries
+    logs.simple = simple -- only used in scripts !
+    if utils then
+        utils.report = simple
+    end
+    logs.verbose = _verbose_
+end
+
+function logs.setverbose(what)
+    if what then
+        trackers.enable("resolvers.verbose")
+    else
+        trackers.disable("resolvers.verbose")
+    end
+    logs.verbose = what or false
+end
+
+function logs.extendbanner(_banner_,_verbose_)
+    banner = banner .. " | ".. _banner_
+    if _verbose_ ~= nil then
+        logs.setverbose(what)
+    end
+end
+
+logs.verbose = false
+logs.report  = logs.tex.report
+logs.simple  = logs.tex.report
+
+function logs.reportlines(str) -- todo: 
+    for line in str:gmatch("(.-)[\n\r]") do
+        logs.report(line)
+    end
+end
+
+function logs.reportline() -- for scripts too
+    logs.report()
+end
+
+logs.simpleline = logs.reportline
+
+function logs.help(message,option)
+    logs.report(banner)
+    logs.reportline()
+    logs.reportlines(message)
+    local moreinfo = logs.moreinfo or ""
+    if moreinfo ~= "" and option ~= "nomoreinfo" then
+        logs.reportline()
+        logs.reportlines(moreinfo)
+    end
+end
+
+logs.set_level('error')
+logs.set_method('tex')
+
+function logs.system(whereto,process,jobname,category,...)
+    for i=1,10 do
+        local f = io.open(whereto,"a")
+        if f then
+            f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)))
+            f:close()
+            break
+        else
+            sleep(0.1)
+        end
+    end
+end
+
+--~ local syslogname = "oeps.xxx"
+--~
+--~ for i=1,10 do
+--~     logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+    comment   = "companion to luat-lib.tex",
+}
+
+-- After a few years using the code the large luat-inp.lua file
+-- has been split up a bit. In the process some functionality was
+-- dropped:
+--
+-- * support for reading lsr files
+-- * selective scanning (subtrees)
+-- * some public auxiliary functions were made private
+--
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
+-- This lib is multi-purpose and can be loaded again later on so that
+-- additional functionality becomes available. We will split thislogs.report("fileio",
+-- module in components once we're done with prototyping. This is the
+-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing
+-- something in this module one can best check with Taco or Hans first; there
+-- is some nasty trickery going on that relates to traditional kpse support.
+
+-- To be considered: hash key lowercase, first entry in table filename
+-- (any case), rest paths (so no need for optimization). Or maybe a
+-- separate table that matches lowercase names to mixed case when
+-- present. In that case the lower() cases can go away. I will do that
+-- only when we run into problems with names ... well ... Iwona-Regular.
+
+-- Beware, loading and saving is overloaded in luat-tmp!
+
+local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
+local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
+local next, type = next, type
+
+local trace_locating, trace_detail, trace_verbose  = false, false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+trackers.register("resolvers.detail",   function(v) trace_detail   = v trackers.enable("resolvers.verbose,resolvers.detail") end)
+
+if not resolvers then
+    resolvers = {
+        suffixes     = { },
+        formats      = { },
+        dangerous    = { },
+        suffixmap    = { },
+        alternatives = { },
+        locators     = { },  -- locate databases
+        hashers      = { },  -- load databases
+        generators   = { },  -- generate databases
+    }
+end
+
+local resolvers = resolvers
+
+resolvers.locators  .notfound = { nil }
+resolvers.hashers   .notfound = { nil }
+resolvers.generators.notfound = { nil }
+
+resolvers.cacheversion = '1.0.1'
+resolvers.cnfname      = 'texmf.cnf'
+resolvers.luaname      = 'texmfcnf.lua'
+resolvers.homedir      = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~'
+resolvers.cnfdefault   = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
+
+local dummy_path_expr = "^!*unset/*$"
+
+local formats      = resolvers.formats
+local suffixes     = resolvers.suffixes
+local dangerous    = resolvers.dangerous
+local suffixmap    = resolvers.suffixmap
+local alternatives = resolvers.alternatives
+
+formats['afm'] = 'AFMFONTS'       suffixes['afm'] = { 'afm' }
+formats['enc'] = 'ENCFONTS'       suffixes['enc'] = { 'enc' }
+formats['fmt'] = 'TEXFORMATS'     suffixes['fmt'] = { 'fmt' }
+formats['map'] = 'TEXFONTMAPS'    suffixes['map'] = { 'map' }
+formats['mp']  = 'MPINPUTS'       suffixes['mp']  = { 'mp' }
+formats['ocp'] = 'OCPINPUTS'      suffixes['ocp'] = { 'ocp' }
+formats['ofm'] = 'OFMFONTS'       suffixes['ofm'] = { 'ofm', 'tfm' }
+formats['otf'] = 'OPENTYPEFONTS'  suffixes['otf'] = { 'otf' } -- 'ttf'
+formats['opl'] = 'OPLFONTS'       suffixes['opl'] = { 'opl' }
+formats['otp'] = 'OTPINPUTS'      suffixes['otp'] = { 'otp' }
+formats['ovf'] = 'OVFFONTS'       suffixes['ovf'] = { 'ovf', 'vf' }
+formats['ovp'] = 'OVPFONTS'       suffixes['ovp'] = { 'ovp' }
+formats['tex'] = 'TEXINPUTS'      suffixes['tex'] = { 'tex' }
+formats['tfm'] = 'TFMFONTS'       suffixes['tfm'] = { 'tfm' }
+formats['ttf'] = 'TTFONTS'        suffixes['ttf'] = { 'ttf', 'ttc' }
+formats['pfb'] = 'T1FONTS'        suffixes['pfb'] = { 'pfb', 'pfa' }
+formats['vf']  = 'VFFONTS'        suffixes['vf']  = { 'vf' }
+
+formats['fea'] = 'FONTFEATURES'   suffixes['fea'] = { 'fea' }
+formats['cid'] = 'FONTCIDMAPS'    suffixes['cid'] = { 'cid', 'cidmap' }
+
+formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new
+suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua'
+
+formats ['lua'] = 'LUAINPUTS' -- new
+suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' }
+
+-- backward compatible ones
+
+alternatives['map files']            = 'map'
+alternatives['enc files']            = 'enc'
+alternatives['cid files']            = 'cid'
+alternatives['fea files']            = 'fea'
+alternatives['opentype fonts']       = 'otf'
+alternatives['truetype fonts']       = 'ttf'
+alternatives['truetype collections'] = 'ttc'
+alternatives['type1 fonts']          = 'pfb'
+
+-- obscure ones
+
+formats ['misc fonts'] = ''
+suffixes['misc fonts'] = { }
+
+formats     ['sfd']                      = 'SFDFONTS'
+suffixes    ['sfd']                      = { 'sfd' }
+alternatives['subfont definition files'] = 'sfd'
+
+-- In practice we will work within one tds tree, but i want to keep
+-- the option open to build tools that look at multiple trees, which is
+-- why we keep the tree specific data in a table. We used to pass the
+-- instance but for practical pusposes we now avoid this and use a
+-- instance variable.
+
+-- here we catch a few new thingies (todo: add these paths to context.tmf)
+--
+-- FONTFEATURES  = .;$TEXMF/fonts/fea//
+-- FONTCIDMAPS   = .;$TEXMF/fonts/cid//
+
+-- we always have one instance active
+
+resolvers.instance = resolvers.instance or nil -- the current one (slow access)
+local instance = resolvers.instance or nil -- the current one (fast access)
+
+function resolvers.newinstance()
+
+    -- store once, freeze and faster (once reset we can best use
+    -- instance.environment) maybe better have a register suffix
+    -- function
+
+    for k, v in next, suffixes do
+        for i=1,#v do
+            local vi = v[i]
+            if vi then
+                suffixmap[vi] = k
+            end
+        end
+    end
+
+    -- because vf searching is somewhat dangerous, we want to prevent
+    -- too liberal searching esp because we do a lookup on the current
+    -- path anyway; only tex (or any) is safe
+
+    for k, v in next, formats do
+        dangerous[k] = true
+    end
+    dangerous.tex = nil
+
+    -- the instance
+
+    local newinstance = {
+        rootpath        = '',
+        treepath        = '',
+        progname        = 'context',
+        engine          = 'luatex',
+        format          = '',
+        environment     = { },
+        variables       = { },
+        expansions      = { },
+        files           = { },
+        remap           = { },
+        configuration   = { },
+        setup           = { },
+        order           = { },
+        found           = { },
+        foundintrees    = { },
+        kpsevars        = { },
+        hashes          = { },
+        cnffiles        = { },
+        luafiles        = { },
+        lists           = { },
+        remember        = true,
+        diskcache       = true,
+        renewcache      = false,
+        scandisk        = true,
+        cachepath       = nil,
+        loaderror       = false,
+        sortdata        = false,
+        savelists       = true,
+        cleanuppaths    = true,
+        allresults      = false,
+        pattern         = nil, -- lists
+        data            = { }, -- only for loading
+        force_suffixes  = true,
+        fakepaths       = { },
+    }
+
+    local ne = newinstance.environment
+
+    for k,v in next, os.env do
+        ne[k] = resolvers.bare_variable(v)
+    end
+
+    return newinstance
+
+end
+
+function resolvers.setinstance(someinstance)
+    instance = someinstance
+    resolvers.instance = someinstance
+    return someinstance
+end
+
+function resolvers.reset()
+    return resolvers.setinstance(resolvers.newinstance())
+end
+
+local function reset_hashes()
+    instance.lists = { }
+    instance.found = { }
+end
+
+local function check_configuration() -- not yet ok, no time for debugging now
+    local ie = instance.environment
+    local function fix(varname,default)
+        local proname = varname .. "." .. instance.progname or "crap"
+        local p, v = ie[proname], ie[varname]
+        if not ((p and p ~= "") or (v and v ~= "")) then
+            instance.variables[varname] = default -- or environment?
+        end
+    end
+    local name = os.name
+    if name == "windows" then
+        fix("OSFONTDIR", "c:/windows/fonts//")
+    elseif name == "macosx" then
+        fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//")
+    else
+        -- bad luck
+    end
+    fix("LUAINPUTS"   , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm
+    fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("LUATEXLIBS"  , ".;$TEXMF/luatex/lua//")
+end
+
+function resolvers.bare_variable(str) -- assumes str is a string
+    return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2"))
+end
+
+function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail'
+    if n then
+        trackers.disable("resolvers.*")
+        trackers.enable("resolvers."..n)
+    end
+end
+
+resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE"))
+
+function resolvers.osenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
+        if e == nil then
+         -- value = "" -- false
+        else
+            value = resolvers.bare_variable(e)
+        end
+        ie[key] = value
+    end
+    return value or ""
+end
+
+function resolvers.env(key)
+    return instance.environment[key] or resolvers.osenv(key)
+end
+
+--
+
+local function expand_vars(lst) -- simple vars
+    local variables, env = instance.variables, resolvers.env
+    local function resolve(a)
+        return variables[a] or env(a)
+    end
+    for k=1,#lst do
+        lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve)
+    end
+end
+
+local function expanded_var(var) -- simple vars
+    local function resolve(a)
+        return instance.variables[a] or resolvers.env(a)
+    end
+    return (gsub(var,"%$([%a%d%_%-]+)",resolve))
+end
+
+local function entry(entries,name)
+    if name and (name ~= "") then
+        name = gsub(name,'%$','')
+        local result = entries[name..'.'..instance.progname] or entries[name]
+        if result then
+            return result
+        else
+            result = resolvers.env(name)
+            if result then
+                instance.variables[name] = result
+                resolvers.expand_variables()
+                return instance.expansions[name] or ""
+            end
+        end
+    end
+    return ""
+end
+
+local function is_entry(entries,name)
+    if name and name ~= "" then
+        name = gsub(name,'%$','')
+        return (entries[name..'.'..instance.progname] or entries[name]) ~= nil
+    else
+        return false
+    end
+end
+
+-- {a,b,c,d}
+-- a,b,c/{p,q,r},d
+-- a,b,c/{p,q,r}/d/{x,y,z}//
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a{b,c}{d,e}f
+-- {a,b,c,d}
+-- {a,b,c/{p,q,r},d}
+-- {a,b,c/{p,q,r}/d/{x,y,z}//}
+-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
+-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
+-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
+
+-- this one is better and faster, but it took me a while to realize
+-- that this kind of replacement is cleaner than messy parsing and
+-- fuzzy concatenating we can probably gain a bit with selectively
+-- applying lpeg, but experiments with lpeg parsing this proved not to
+-- work that well; the parsing is ok, but dealing with the resulting
+-- table is a pain because we need to work inside-out recursively
+
+local function splitpathexpr(str, t, validate)
+    -- no need for further optimization as it is only called a
+    -- few times, we can use lpeg for the sub; we could move
+    -- the local functions outside the body
+    t = t or { }
+    str = gsub(str,",}",",@}")
+    str = gsub(str,"{,","{@,")
+ -- str = "@" .. str .. "@"
+    local ok, done
+    local function do_first(a,b)
+        local t = { }
+        for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_second(a,b)
+        local t = { }
+        for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_both(a,b)
+        local t = { }
+        for sa in gmatch(a,"[^,]+") do
+            for sb in gmatch(b,"[^,]+") do
+                t[#t+1] = sa .. sb
+            end
+        end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_three(a,b,c)
+        return a .. b.. c
+    end
+    while true do
+        done = false
+        while true do
+            str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both)
+            if ok > 0 then done = true else break end
+        end
+        str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three)
+        if ok > 0 then done = true end
+        if not done then break end
+    end
+    str = gsub(str,"[{}]", "")
+    str = gsub(str,"@","")
+    if validate then
+        for s in gmatch(str,"[^,]+") do
+            s = validate(s)
+            if s then t[#t+1] = s end
+        end
+    else
+        for s in gmatch(str,"[^,]+") do
+            t[#t+1] = s
+        end
+    end
+    return t
+end
+
+local function expanded_path_from_list(pathlist) -- maybe not a list, just a path
+    -- a previous version fed back into pathlist
+    local newlist, ok = { }, false
+    for k=1,#pathlist do
+        if find(pathlist[k],"[{}]") then
+            ok = true
+            break
+        end
+    end
+    if ok then
+        local function validate(s)
+            s = file.collapse_path(s)
+            return s ~= "" and not find(s,dummy_path_expr) and s
+        end
+        for k=1,#pathlist do
+            splitpathexpr(pathlist[k],newlist,validate)
+        end
+    else
+        for k=1,#pathlist do
+            for p in gmatch(pathlist[k],"([^,]+)") do
+                p = file.collapse_path(p)
+                if p ~= "" then newlist[#newlist+1] = p end
+            end
+        end
+    end
+    return newlist
+end
+
+-- we follow a rather traditional approach:
+--
+-- (1) texmf.cnf given in TEXMFCNF
+-- (2) texmf.cnf searched in default variable
+--
+-- also we now follow the stupid route: if not set then just assume *one*
+-- cnf file under texmf (i.e. distribution)
+
+resolvers.ownpath     = resolvers.ownpath or nil
+resolvers.ownbin      = resolvers.ownbin  or arg[-2] or arg[-1] or arg[0] or "luatex"
+resolvers.autoselfdir = true -- false may be handy for debugging
+
+function resolvers.getownpath()
+    if not resolvers.ownpath then
+        if resolvers.autoselfdir and os.selfdir then
+            resolvers.ownpath = os.selfdir
+        else
+            local binary = resolvers.ownbin
+            if os.platform == "windows" then
+                binary = file.replacesuffix(binary,"exe")
+            end
+            for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+                local b = file.join(p,binary)
+                if lfs.isfile(b) then
+                    -- we assume that after changing to the path the currentdir function
+                    -- resolves to the real location and use this side effect here; this
+                    -- trick is needed because on the mac installations use symlinks in the
+                    -- path instead of real locations
+                    local olddir = lfs.currentdir()
+                    if lfs.chdir(p) then
+                        local pp = lfs.currentdir()
+                        if trace_verbose and p ~= pp then
+                            logs.report("fileio","following symlink %s to %s",p,pp)
+                        end
+                        resolvers.ownpath = pp
+                        lfs.chdir(olddir)
+                    else
+                        if trace_verbose then
+                            logs.report("fileio","unable to check path %s",p)
+                        end
+                        resolvers.ownpath =  p
+                    end
+                    break
+                end
+            end
+        end
+        if not resolvers.ownpath then resolvers.ownpath = '.' end
+    end
+    return resolvers.ownpath
+end
+
+local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" }
+
+local function identify_own()
+    local ownpath = resolvers.getownpath() or lfs.currentdir()
+    local ie = instance.environment
+    if ownpath then
+        if resolvers.env('SELFAUTOLOC')    == "" then os.env['SELFAUTOLOC']    = file.collapse_path(ownpath) end
+        if resolvers.env('SELFAUTODIR')    == "" then os.env['SELFAUTODIR']    = file.collapse_path(ownpath .. "/..") end
+        if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end
+    else
+        logs.report("fileio","error: unable to locate ownpath")
+        os.exit()
+    end
+    if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end
+    if resolvers.env('TEXOS')    == "" then os.env['TEXOS']    = resolvers.env('SELFAUTODIR') end
+    if resolvers.env('TEXROOT')  == "" then os.env['TEXROOT']  = resolvers.env('SELFAUTOPARENT') end
+    if trace_verbose then
+        for i=1,#own_places do
+            local v = own_places[i]
+            logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown")
+        end
+    end
+    identify_own = function() end
+end
+
+function resolvers.identify_cnf()
+    if #instance.cnffiles == 0 then
+        -- fallback
+        identify_own()
+        -- the real search
+        resolvers.expand_variables()
+        local t = resolvers.split_path(resolvers.env('TEXMFCNF'))
+        t = expanded_path_from_list(t)
+        expand_vars(t) -- redundant
+        local function locate(filename,list)
+            for i=1,#t do
+                local ti = t[i]
+                local texmfcnf = file.collapse_path(file.join(ti,filename))
+                if lfs.isfile(texmfcnf) then
+                    list[#list+1] = texmfcnf
+                end
+            end
+        end
+        locate(resolvers.luaname,instance.luafiles)
+        locate(resolvers.cnfname,instance.cnffiles)
+    end
+end
+
+local function load_cnf_file(fname)
+    fname = resolvers.clean_path(fname)
+    local lname = file.replacesuffix(fname,'lua')
+    local f = io.open(lname)
+    if f then -- this will go
+        f:close()
+        local dname = file.dirname(fname)
+        if not instance.configuration[dname] then
+            resolvers.load_data(dname,'configuration',lname and file.basename(lname))
+            instance.order[#instance.order+1] = instance.configuration[dname]
+        end
+    else
+        f = io.open(fname)
+        if f then
+            if trace_verbose then
+                logs.report("fileio","loading %s", fname)
+            end
+            local line, data, n, k, v
+            local dname = file.dirname(fname)
+            if not instance.configuration[dname] then
+                instance.configuration[dname] = { }
+                instance.order[#instance.order+1] = instance.configuration[dname]
+            end
+            local data = instance.configuration[dname]
+            while true do
+                local line, n = f:read(), 0
+                if line then
+                    while true do -- join lines
+                        line, n = gsub(line,"\\%s*$", "")
+                        if n > 0 then
+                            line = line .. f:read()
+                        else
+                            break
+                        end
+                    end
+                    if not find(line,"^[%%#]") then
+                        local l = gsub(line,"%s*%%.*$","")
+                        local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$")
+                        if k and v and not data[k] then
+                            v = gsub(v,"[%%#].*",'')
+                            data[k] = gsub(v,"~","$HOME")
+                            instance.kpsevars[k] = true
+                        end
+                    end
+                else
+                    break
+                end
+            end
+            f:close()
+        elseif trace_verbose then
+            logs.report("fileio","skipping %s", fname)
+        end
+    end
+end
+
+local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in next, c do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = resolvers.bare_variable(v)
+                end
+            end
+        end
+    end
+end
+
+function resolvers.load_cnf()
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            load_cnf_file(fname)
+        end
+    end
+    -- instance.cnffiles contain complete names now !
+    if #instance.cnffiles == 0 then
+        if trace_verbose then
+            logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)")
+        end
+    else
+        instance.rootpath = instance.cnffiles[1]
+        for k,fname in ipairs(instance.cnffiles) do
+            instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        if instance.diskcache and not instance.renewcache then
+            resolvers.loadoldconfig(instance.cnffiles)
+            if instance.loaderror then
+                loadoldconfigdata()
+                resolvers.saveoldconfig()
+            end
+        else
+            loadoldconfigdata()
+            if instance.renewcache then
+                resolvers.saveoldconfig()
+            end
+        end
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+function resolvers.load_lua()
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        resolvers.loadnewconfig()
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+-- database loading
+
+function resolvers.load_hash()
+    resolvers.locatelists()
+    if instance.diskcache and not instance.renewcache then
+        resolvers.loadfiles()
+        if instance.loaderror then
+            resolvers.loadlists()
+            resolvers.savefiles()
+        end
+    else
+        resolvers.loadlists()
+        if instance.renewcache then
+            resolvers.savefiles()
+        end
+    end
+end
+
+function resolvers.append_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash append: %s",tag)
+    end
+    insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.prepend_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash prepend: %s",tag)
+    end
+    insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash
+--  local t = resolvers.expanded_path_list('TEXMF') -- full expansion
+    local t = resolvers.split_path(resolvers.env('TEXMF'))
+    insert(t,1,specification)
+    local newspec = concat(t,";")
+    if instance.environment["TEXMF"] then
+        instance.environment["TEXMF"] = newspec
+    elseif instance.variables["TEXMF"] then
+        instance.variables["TEXMF"] = newspec
+    else
+        -- weird
+    end
+    resolvers.expand_variables()
+    reset_hashes()
+end
+
+-- locators
+
+function resolvers.locatelists()
+    for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do
+        if trace_verbose then
+            logs.report("fileio","locating list of %s",path)
+        end
+        resolvers.locatedatabase(file.collapse_path(path))
+    end
+end
+
+function resolvers.locatedatabase(specification)
+    return resolvers.methodhandler('locators', specification)
+end
+
+function resolvers.locators.tex(specification)
+    if specification and specification ~= '' and lfs.isdir(specification) then
+        if trace_locating then
+            logs.report("fileio",'! tex locator found: %s',specification)
+        end
+        resolvers.append_hash('file',specification,filename)
+    elseif trace_locating then
+        logs.report("fileio",'? tex locator not found: %s',specification)
+    end
+end
+
+-- hashers
+
+function resolvers.hashdatabase(tag,name)
+    return resolvers.methodhandler('hashers',tag,name)
+end
+
+function resolvers.loadfiles()
+    instance.loaderror = false
+    instance.files = { }
+    if not instance.renewcache then
+        for _, hash in ipairs(instance.hashes) do
+            resolvers.hashdatabase(hash.tag,hash.name)
+            if instance.loaderror then break end
+        end
+    end
+end
+
+function resolvers.hashers.tex(tag,name)
+    resolvers.load_data(tag,'files')
+end
+
+-- generators:
+
+function resolvers.loadlists()
+    for _, hash in ipairs(instance.hashes) do
+        resolvers.generatedatabase(hash.tag)
+    end
+end
+
+function resolvers.generatedatabase(specification)
+    return resolvers.methodhandler('generators', specification)
+end
+
+-- starting with . or .. etc or funny char
+
+local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t"))
+
+function resolvers.generators.tex(specification)
+    local tag = specification
+    if trace_verbose then
+        logs.report("fileio","scanning path %s",specification)
+    end
+    instance.files[tag] = { }
+    local files = instance.files[tag]
+    local n, m, r = 0, 0, 0
+    local spec = specification .. '/'
+    local attributes = lfs.attributes
+    local directory = lfs.dir
+    local function action(path)
+        local full
+        if path then
+            full = spec .. path .. '/'
+        else
+            full = spec
+        end
+        for name in directory(full) do
+            if not weird:match(name) then
+                local mode = attributes(full..name,'mode')
+                if mode == 'file' then
+                    if path then
+                        n = n + 1
+                        local f = files[name]
+                        if f then
+                            if type(f) == 'string' then
+                                files[name] = { f, path }
+                            else
+                                f[#f+1] = path
+                            end
+                        else -- probably unique anyway
+                            files[name] = path
+                            local lower = lower(name)
+                            if name ~= lower then
+                                files["remap:"..lower] = name
+                                r = r + 1
+                            end
+                        end
+                    end
+                elseif mode == 'directory' then
+                    m = m + 1
+                    if path then
+                        action(path..'/'..name)
+                    else
+                        action(name)
+                    end
+                end
+            end
+        end
+    end
+    action()
+    if trace_verbose then
+        logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r)
+    end
+end
+
+-- savers, todo
+
+function resolvers.savefiles()
+    resolvers.save_data('files')
+end
+
+-- A config (optionally) has the paths split in tables. Internally
+-- we join them and split them after the expansion has taken place. This
+-- is more convenient.
+
+function resolvers.splitconfig()
+    for i,c in ipairs(instance) do
+        for k,v in pairs(c) do
+            if type(v) == 'string' then
+                local t = file.split_path(v)
+                if #t > 1 then
+                    c[k] = t
+                end
+            end
+        end
+    end
+end
+
+function resolvers.joinconfig()
+    for i,c in ipairs(instance.order) do
+        for k,v in pairs(c) do -- ipairs?
+            if type(v) == 'table' then
+                c[k] = file.join_path(v)
+            end
+        end
+    end
+end
+function resolvers.split_path(str)
+    if type(str) == 'table' then
+        return str
+    else
+        return file.split_path(str)
+    end
+end
+function resolvers.join_path(str)
+    if type(str) == 'table' then
+        return file.join_path(str)
+    else
+        return str
+    end
+end
+
+function resolvers.splitexpansions()
+    local ie = instance.expansions
+    for k,v in next, ie do
+        local t, h = { }, { }
+        for _,vv in ipairs(file.split_path(v)) do
+            if vv ~= "" and not h[vv] then
+                t[#t+1] = vv
+                h[vv] = true
+            end
+        end
+        if #t > 1 then
+            ie[k] = t
+        else
+            ie[k] = t[1]
+        end
+    end
+end
+
+-- end of split/join code
+
+function resolvers.saveoldconfig()
+    resolvers.splitconfig()
+    resolvers.save_data('configuration')
+    resolvers.joinconfig()
+end
+
+resolvers.configbanner = [[
+-- This is a Luatex configuration file created by 'luatools.lua' or
+-- 'luatex.exe' directly. For comment, suggestions and questions you can
+-- contact the ConTeXt Development Team. This configuration file is
+-- not copyrighted. [HH & TH]
+]]
+
+function resolvers.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local function dump(k,v,m) -- could be moved inline
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+        end
+    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sortedkeys(files)) do -- ipairs
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sortedkeys(fk)) do -- ipairs
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
+            end
+        end
+    else
+        for k, v in next, files do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in next, v do
+                    t[#t+1] = dump(kk,vv,"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,v,"\t")
+            end
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+function resolvers.save_data(dataname, makename) -- untested without cache overload
+    for cachename, files in next, instance[dataname] do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. ".lua", name .. ".luc"
+        if trace_verbose then
+            logs.report("fileio","preparing %s for %s",dataname,cachename)
+        end
+        for k, v in next, files do
+            if type(v) == "table" and #v == 1 then
+                files[k] = v[1]
+            end
+        end
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = resolvers.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local ok = io.savedata(luaname,resolvers.serialize(data))
+        if ok then
+            if trace_verbose then
+                logs.report("fileio","%s saved in %s",dataname,luaname)
+            end
+            if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip
+                if trace_verbose then
+                    logs.report("fileio","%s compiled to %s",dataname,lucname)
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname)
+                end
+                os.remove(lucname)
+            end
+        elseif trace_verbose then
+            logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname)
+        end
+    end
+end
+
+function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua")
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then
+            if trace_verbose then
+                logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = data.content
+        else
+            if trace_verbose then
+                logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
+        end
+    elseif trace_verbose then
+        logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+    end
+end
+
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
+
+function resolvers.resetconfig()
+    identify_own()
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function resolvers.loadnewconfig()
+    for _, cnf in ipairs(instance.luafiles) do
+        local pathname = file.dirname(cnf)
+        local filename = file.join(pathname,resolvers.luaname)
+        local blob = loadfile(filename)
+        if blob then
+            local data = blob()
+            if data then
+                if trace_verbose then
+                    logs.report("fileio","loading configuration file %s",filename)
+                end
+                if true then
+                    -- flatten to variable.progname
+                    local t = { }
+                    for k, v in next, data do -- v = progname
+                        if type(v) == "string" then
+                            t[k] = v
+                        else
+                            for kk, vv in next, v do -- vv = variable
+                                if type(vv) == "string" then
+                                    t[vv.."."..v] = kk
+                                end
+                            end
+                        end
+                    end
+                    instance['setup'][pathname] = t
+                else
+                    instance['setup'][pathname] = data
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","skipping configuration file %s",filename)
+                end
+                instance['setup'][pathname] = { }
+                instance.loaderror = true
+            end
+        elseif trace_verbose then
+            logs.report("fileio","skipping configuration file %s",filename)
+        end
+        instance.order[#instance.order+1] = instance.setup[pathname]
+        if instance.loaderror then break end
+    end
+end
+
+function resolvers.loadoldconfig()
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            resolvers.load_data(dname,'configuration')
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
+        end
+    end
+    resolvers.joinconfig()
+end
+
+function resolvers.expand_variables()
+    local expansions, environment, variables = { }, instance.environment, instance.variables
+    local env = resolvers.env
+    instance.expansions = expansions
+    if instance.engine   ~= "" then environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then environment['progname'] = instance.progname end
+    for k,v in next, environment do
+        local a, b = match(k,"^(%a+)%_(.*)%s*$")
+        if a and b then
+            expansions[a..'.'..b] = v
+        else
+            expansions[k] = v
+        end
+    end
+    for k,v in next, environment do -- move environment to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    for k,v in next, variables do -- move variables to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    local busy = false
+    local function resolve(a)
+        busy = true
+        return expansions[a] or env(a)
+    end
+    while true do
+        busy = false
+        for k,v in next, expansions do
+            local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve)
+            local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve)
+            if n > 0 or m > 0 then
+                expansions[k]= s
+            end
+        end
+        if not busy then break end
+    end
+    for k,v in next, expansions do
+        expansions[k] = gsub(v,"\\", '/')
+    end
+end
+
+function resolvers.variable(name)
+    return entry(instance.variables,name)
+end
+
+function resolvers.expansion(name)
+    return entry(instance.expansions,name)
+end
+
+function resolvers.is_variable(name)
+    return is_entry(instance.variables,name)
+end
+
+function resolvers.is_expansion(name)
+    return is_entry(instance.expansions,name)
+end
+
+function resolvers.unexpanded_path_list(str)
+    local pth = resolvers.variable(str)
+    local lst = resolvers.split_path(pth)
+    return expanded_path_from_list(lst)
+end
+
+function resolvers.unexpanded_path(str)
+    return file.join_path(resolvers.unexpanded_path_list(str))
+end
+
+do -- no longer needed
+
+    local done = { }
+
+    function resolvers.reset_extra_path()
+        local ep = instance.extra_paths
+        if not ep then
+            ep, done = { }, { }
+            instance.extra_paths = ep
+        elseif #ep > 0 then
+            instance.lists, done = { }, { }
+        end
+    end
+
+    function resolvers.register_extra_path(paths,subpaths)
+        local ep = instance.extra_paths or { }
+        local n = #ep
+        if paths and paths ~= "" then
+            if subpaths and subpaths ~= "" then
+                for p in gmatch(paths,"[^,]+") do
+                    -- we gmatch each step again, not that fast, but used seldom
+                    for s in gmatch(subpaths,"[^,]+") do
+                        local ps = p .. "/" .. s
+                        if not done[ps] then
+                            ep[#ep+1] = resolvers.clean_path(ps)
+                            done[ps] = true
+                        end
+                    end
+                end
+            else
+                for p in gmatch(paths,"[^,]+") do
+                    if not done[p] then
+                        ep[#ep+1] = resolvers.clean_path(p)
+                        done[p] = true
+                    end
+                end
+            end
+        elseif subpaths and subpaths ~= "" then
+            for i=1,n do
+                -- we gmatch each step again, not that fast, but used seldom
+                for s in gmatch(subpaths,"[^,]+") do
+                    local ps = ep[i] .. "/" .. s
+                    if not done[ps] then
+                        ep[#ep+1] = resolvers.clean_path(ps)
+                        done[ps] = true
+                    end
+                end
+            end
+        end
+        if #ep > 0 then
+            instance.extra_paths = ep -- register paths
+        end
+        if #ep > n then
+            instance.lists = { } -- erase the cache
+        end
+    end
+
+end
+
+local function made_list(instance,list)
+    local ep = instance.extra_paths
+    if not ep or #ep == 0 then
+        return list
+    else
+        local done, new = { }, { }
+        -- honour . .. ../.. but only when at the start
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                if find(v,"^[%.%/]$") then
+                    done[v] = true
+                    new[#new+1] = v
+                else
+                    break
+                end
+            end
+        end
+        -- first the extra paths
+        for k=1,#ep do
+            local v = ep[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        -- next the formal paths
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        return new
+    end
+end
+
+function resolvers.clean_path_list(str)
+    local t = resolvers.expanded_path_list(str)
+    if t then
+        for i=1,#t do
+            t[i] = file.collapse_path(resolvers.clean_path(t[i]))
+        end
+    end
+    return t
+end
+
+function resolvers.expand_path(str)
+    return file.join_path(resolvers.expanded_path_list(str))
+end
+
+function resolvers.expanded_path_list(str)
+    if not str then
+        return ep or { }
+    elseif instance.savelists then
+        -- engine+progname hash
+        str = gsub(str,"%$","")
+        if not instance.lists[str] then -- cached
+            local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str)))
+            instance.lists[str] = expanded_path_from_list(lst)
+        end
+        return instance.lists[str]
+    else
+        local lst = resolvers.split_path(resolvers.expansion(str))
+        return made_list(instance,expanded_path_from_list(lst))
+    end
+end
+
+function resolvers.expanded_path_list_from_var(str) -- brrr
+    local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$",""))
+    if tmp ~= "" then
+        return resolvers.expanded_path_list(str)
+    else
+        return resolvers.expanded_path_list(tmp)
+    end
+end
+
+function resolvers.expand_path_from_var(str)
+    return file.join_path(resolvers.expanded_path_list_from_var(str))
+end
+
+function resolvers.format_of_var(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+function resolvers.format_of_suffix(str)
+    return suffixmap[file.extname(str)] or 'tex'
+end
+
+function resolvers.variable_of_format(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+
+function resolvers.var_of_format_or_suffix(str)
+    local v = formats[str]
+    if v then
+        return v
+    end
+    v = formats[alternatives[str]]
+    if v then
+        return v
+    end
+    v = suffixmap[file.extname(str)]
+    if v then
+        return formats[isf]
+    end
+    return ''
+end
+
+function resolvers.expand_braces(str) -- output variable and brace expansion of STRING
+    local ori = resolvers.variable(str)
+    local pth = expanded_path_from_list(resolvers.split_path(ori))
+    return file.join_path(pth)
+end
+
+resolvers.isreadable = { }
+
+function resolvers.isreadable.file(name)
+    local readable = lfs.isfile(name) -- brrr
+    if trace_detail then
+        if readable then
+            logs.report("fileio","+ readable: %s",name)
+        else
+            logs.report("fileio","- readable: %s", name)
+        end
+    end
+    return readable
+end
+
+resolvers.isreadable.tex = resolvers.isreadable.file
+
+-- name
+-- name/name
+
+local function collect_files(names)
+    local filelist = { }
+    for k=1,#names do
+        local fname = names[k]
+        if trace_detail then
+            logs.report("fileio","? blobpath asked: %s",fname)
+        end
+        local bname = file.basename(fname)
+        local dname = file.dirname(fname)
+        if dname == "" or find(dname,"^%.") then
+            dname = false
+        else
+            dname = "/" .. dname .. "$"
+        end
+        local hashes = instance.hashes
+        for h=1,#hashes do
+            local hash = hashes[h]
+            local blobpath = hash.tag
+            local files = blobpath and instance.files[blobpath]
+            if files then
+                if trace_detail then
+                    logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname)
+                end
+                local blobfile = files[bname]
+                if not blobfile then
+                    local rname = "remap:"..bname
+                    blobfile = files[rname]
+                    if blobfile then
+                        bname = files[rname]
+                        blobfile = files[bname]
+                    end
+                end
+                if blobfile then
+                    if type(blobfile) == 'string' then
+                        if not dname or find(blobfile,dname) then
+                            filelist[#filelist+1] = {
+                                hash.type,
+                                file.join(blobpath,blobfile,bname), -- search
+                                resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result
+                            }
+                        end
+                    else
+                        for kk=1,#blobfile do
+                            local vv = blobfile[kk]
+                            if not dname or find(vv,dname) then
+                                filelist[#filelist+1] = {
+                                    hash.type,
+                                    file.join(blobpath,vv,bname), -- search
+                                    resolvers.concatinators[hash.type](blobpath,vv,bname) -- result
+                                }
+                            end
+                        end
+                    end
+                end
+            elseif trace_locating then
+                logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname)
+            end
+        end
+    end
+    if #filelist > 0 then
+        return filelist
+    else
+        return nil
+    end
+end
+
+function resolvers.suffix_of_format(str)
+    if suffixes[str] then
+        return suffixes[str][1]
+    else
+        return ""
+    end
+end
+
+function resolvers.suffixes_of_format(str)
+    if suffixes[str] then
+        return suffixes[str]
+    else
+        return {}
+    end
+end
+
+function resolvers.register_in_trees(name)
+    if not find(name,"^%.") then
+        instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one
+    end
+end
+
+-- split the next one up for readability (bu this module needs a cleanup anyway)
+
+local function can_be_dir(name) -- can become local
+    local fakepaths = instance.fakepaths
+    if not fakepaths[name] then
+        if lfs.isdir(name) then
+            fakepaths[name] = 1 -- directory
+        else
+            fakepaths[name] = 2 -- no directory
+        end
+    end
+    return (fakepaths[name] == 1)
+end
+
+local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc)
+    local result = collected or { }
+    local stamp  = nil
+    filename = file.collapse_path(filename)  -- elsewhere
+    filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere
+    -- speed up / beware: format problem
+    if instance.remember then
+        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format
+        if instance.found[stamp] then
+            if trace_locating then
+                logs.report("fileio",'! remembered: %s',filename)
+            end
+            return instance.found[stamp]
+        end
+    end
+    if not dangerous[instance.format or "?"] then
+        if resolvers.isreadable.file(filename) then
+            if trace_detail then
+                logs.report("fileio",'= found directly: %s',filename)
+            end
+            instance.found[stamp] = { filename }
+            return { filename }
+        end
+    end
+    if find(filename,'%*') then
+        if trace_locating then
+            logs.report("fileio",'! wildcard: %s', filename)
+        end
+        result = resolvers.find_wildcard_files(filename)
+    elseif file.is_qualified_path(filename) then
+        if resolvers.isreadable.file(filename) then
+            if trace_locating then
+                logs.report("fileio",'! qualified: %s', filename)
+            end
+            result = { filename }
+        else
+            local forcedname, ok, suffix = "", false, file.extname(filename)
+            if suffix == "" then -- why
+                if instance.format == "" then
+                    forcedname = filename .. ".tex"
+                    if resolvers.isreadable.file(forcedname) then
+                        if trace_locating then
+                            logs.report("fileio",'! no suffix, forcing standard filetype: tex')
+                        end
+                        result, ok = { forcedname }, true
+                    end
+                else
+                    local suffixes = resolvers.suffixes_of_format(instance.format)
+                    for _, s in next, suffixes do
+                        forcedname = filename .. "." .. s
+                        if resolvers.isreadable.file(forcedname) then
+                            if trace_locating then
+                                logs.report("fileio",'! no suffix, forcing format filetype: %s', s)
+                            end
+                            result, ok = { forcedname }, true
+                            break
+                        end
+                    end
+                end
+            end
+            if not ok and suffix ~= "" then
+                -- try to find in tree (no suffix manipulation), here we search for the
+                -- matching last part of the name
+                local basename = file.basename(filename)
+                local pattern = (filename .. "$"):gsub("([%.%-])","%%%1")
+                local savedformat = instance.format
+                local format = savedformat or ""
+                if format == "" then
+                    instance.format = resolvers.format_of_suffix(suffix)
+                end
+                if not format then
+                    instance.format = "othertextfiles" -- kind of everything, maybe texinput is better
+                end
+                --
+                local resolved = collect_instance_files(basename)
+                if #result == 0 then
+                    local lowered = lower(basename)
+                    if filename ~= lowered then
+                        resolved = collect_instance_files(lowered)
+                    end
+                end
+                resolvers.format = savedformat
+                --
+                for r=1,#resolved do
+                    local rr = resolved[r]
+                    if rr:find(pattern) then
+                        result[#result+1], ok = rr, true
+                    end
+                end
+                -- a real wildcard:
+                --
+                -- if not ok then
+                --     local filelist = collect_files({basename})
+                --     for f=1,#filelist do
+                --         local ff = filelist[f][3] or ""
+                --         if ff:find(pattern) then
+                --             result[#result+1], ok = ff, true
+                --         end
+                --     end
+                -- end
+            end
+            if not ok and trace_locating then
+                logs.report("fileio",'? qualified: %s', filename)
+            end
+        end
+    else
+        -- search spec
+        local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename)
+        if ext == "" then
+            if not instance.force_suffixes then
+                wantedfiles[#wantedfiles+1] = filename
+            end
+        else
+            wantedfiles[#wantedfiles+1] = filename
+        end
+        if instance.format == "" then
+            if ext == "" then
+                local forcedname = filename .. '.tex'
+                wantedfiles[#wantedfiles+1] = forcedname
+                filetype = resolvers.format_of_suffix(forcedname)
+                if trace_locating then
+                    logs.report("fileio",'! forcing filetype: %s',filetype)
+                end
+            else
+                filetype = resolvers.format_of_suffix(filename)
+                if trace_locating then
+                    logs.report("fileio",'! using suffix based filetype: %s',filetype)
+                end
+            end
+        else
+            if ext == "" then
+                local suffixes = resolvers.suffixes_of_format(instance.format)
+                for _, s in next, suffixes do
+                    wantedfiles[#wantedfiles+1] = filename .. "." .. s
+                end
+            end
+            filetype = instance.format
+            if trace_locating then
+                logs.report("fileio",'! using given filetype: %s',filetype)
+            end
+        end
+        local typespec = resolvers.variable_of_format(filetype)
+        local pathlist = resolvers.expanded_path_list(typespec)
+        if not pathlist or #pathlist == 0 then
+            -- no pathlist, access check only / todo == wildcard
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+                logs.report("fileio",'? filetype: %s',filetype or '?')
+                logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | "))
+            end
+            for k=1,#wantedfiles do
+                local fname = wantedfiles[k]
+                if fname and resolvers.isreadable.file(fname) then
+                    filename, done = fname, true
+                    result[#result+1] = file.join('.',fname)
+                    break
+                end
+            end
+            -- this is actually 'other text files' or 'any' or 'whatever'
+            local filelist = collect_files(wantedfiles)
+            local fl = filelist and filelist[1]
+            if fl then
+                filename = fl[3]
+                result[#result+1] = filename
+                done = true
+            end
+        else
+            -- list search
+            local filelist = collect_files(wantedfiles)
+            local doscan, recurse
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+            end
+            -- a bit messy ... esp the doscan setting here
+            for k=1,#pathlist do
+                local path = pathlist[k]
+                if find(path,"^!!") then doscan  = false else doscan  = true  end
+                if find(path,"//$") then recurse = true  else recurse = false end
+                local pathname = gsub(path,"^!+", '')
+                done = false
+                -- using file list
+                if filelist and not (done and not instance.allresults) and recurse then
+                    -- compare list entries with permitted pattern
+                    pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences
+                    pathname = gsub(pathname,"/+$", '/.*')      -- later usage of pathname
+                    pathname = gsub(pathname,"//", '/.-/')      -- not ok for /// but harmless
+                    local expr = "^" .. pathname
+                    for k=1,#filelist do
+                        local fl = filelist[k]
+                        local f = fl[2]
+                        if find(f,expr) then
+                            if trace_detail then
+                                logs.report("fileio",'= found in hash: %s',f)
+                            end
+                            --- todo, test for readable
+                            result[#result+1] = fl[3]
+                            resolvers.register_in_trees(f) -- for tracing used files
+                            done = true
+                            if not instance.allresults then break end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- check if on disk / unchecked / does not work at all / also zips
+                    if resolvers.splitmethod(pathname).scheme == 'file' then -- ?
+                        local pname = gsub(pathname,"%.%*$",'')
+                        if not find(pname,"%*") then
+                            local ppname = gsub(pname,"/+$","")
+                            if can_be_dir(ppname) then
+                                for k=1,#wantedfiles do
+                                    local w = wantedfiles[k]
+                                    local fname = file.join(ppname,w)
+                                    if resolvers.isreadable.file(fname) then
+                                        if trace_detail then
+                                            logs.report("fileio",'= found by scanning: %s',fname)
+                                        end
+                                        result[#result+1] = fname
+                                        done = true
+                                        if not instance.allresults then break end
+                                    end
+                                end
+                            else
+                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
+                            end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- todo: slow path scanning
+                end
+                if done and not instance.allresults then break end
+            end
+        end
+    end
+    for k=1,#result do
+        result[k] = file.collapse_path(result[k])
+    end
+    if instance.remember then
+        instance.found[stamp] = result
+    end
+    return result
+end
+
+if not resolvers.concatinators  then resolvers.concatinators = { } end
+
+resolvers.concatinators.tex  = file.join
+resolvers.concatinators.file = resolvers.concatinators.tex
+
+function resolvers.find_files(filename,filetype,mustexist)
+    if type(mustexist) == boolean then
+        -- all set
+    elseif type(filetype) == 'boolean' then
+        filetype, mustexist = nil, false
+    elseif type(filetype) ~= 'string' then
+        filetype, mustexist = nil, false
+    end
+    instance.format = filetype or ''
+    local result = collect_instance_files(filename)
+    if #result == 0 then
+        local lowered = lower(filename)
+        if filename ~= lowered then
+            return collect_instance_files(lowered)
+        end
+    end
+    instance.format = ''
+    return result
+end
+
+function resolvers.find_file(filename,filetype,mustexist)
+    return (resolvers.find_files(filename,filetype,mustexist)[1] or "")
+end
+
+function resolvers.find_given_files(filename)
+    local bname, result = file.basename(filename), { }
+    local hashes = instance.hashes
+    for k=1,#hashes do
+        local hash = hashes[k]
+        local files = instance.files[hash.tag]
+        local blist = files[bname]
+        if not blist then
+            local rname = "remap:"..bname
+            blist = files[rname]
+            if blist then
+                bname = files[rname]
+                blist = files[bname]
+            end
+        end
+        if blist then
+            if type(blist) == 'string' then
+                result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or ""
+                if not instance.allresults then break end
+            else
+                for kk=1,#blist do
+                    local vv = blist[kk]
+                    result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or ""
+                    if not instance.allresults then break end
+                end
+            end
+        end
+    end
+    return result
+end
+
+function resolvers.find_given_file(filename)
+    return (resolvers.find_given_files(filename)[1] or "")
+end
+
+local function doit(path,blist,bname,tag,kind,result,allresults)
+    local done = false
+    if blist and kind then
+        if type(blist) == 'string' then
+            -- make function and share code
+            if find(lower(blist),path) then
+                result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or ""
+                done = true
+            end
+        else
+            for kk=1,#blist do
+                local vv = blist[kk]
+                if find(lower(vv),path) then
+                    result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or ""
+                    done = true
+                    if not allresults then break end
+                end
+            end
+        end
+    end
+    return done
+end
+
+function resolvers.find_wildcard_files(filename) -- todo: remap:
+    local result = { }
+    local bname, dname = file.basename(filename), file.dirname(filename)
+    local path = gsub(dname,"^*/","")
+    path = gsub(path,"*",".*")
+    path = gsub(path,"-","%%-")
+    if dname == "" then
+        path = ".*"
+    end
+    local name = bname
+    name = gsub(name,"*",".*")
+    name = gsub(name,"-","%%-")
+    path = lower(path)
+    name = lower(name)
+    local files, allresults, done = instance.files, instance.allresults, false
+    if find(name,"%*") then
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            for kk, hh in next, files[hash.tag] do
+                if not find(kk,"^remap:") then
+                    if find(lower(kk),name) then
+                        if doit(path,hh,kk,tag,kind,result,allresults) then done = true end
+                        if done and not allresults then break end
+                    end
+                end
+            end
+        end
+    else
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end
+            if done and not allresults then break end
+        end
+    end
+    -- we can consider also searching the paths not in the database, but then
+    -- we end up with a messy search (all // in all path specs)
+    return result
+end
+
+function resolvers.find_wildcard_file(filename)
+    return (resolvers.find_wildcard_files(filename)[1] or "")
+end
+
+-- main user functions
+
+function resolvers.automount()
+    -- implemented later
+end
+
+function resolvers.load(option)
+    statistics.starttiming(instance)
+    resolvers.resetconfig()
+    resolvers.identify_cnf()
+    resolvers.load_lua()
+    resolvers.expand_variables()
+    resolvers.load_cnf()
+    resolvers.expand_variables()
+    if option ~= "nofiles" then
+        resolvers.load_hash()
+        resolvers.automount()
+    end
+    statistics.stoptiming(instance)
+end
+
+function resolvers.for_files(command, files, filetype, mustexist)
+    if files and #files > 0 then
+        local function report(str)
+            if trace_verbose then
+                logs.report("fileio",str) -- has already verbose
+            else
+                print(str)
+            end
+        end
+        if trace_verbose then
+            report('')
+        end
+        for _, file in ipairs(files) do
+            local result = command(file,filetype,mustexist)
+            if type(result) == 'string' then
+                report(result)
+            else
+                for _,v in ipairs(result) do
+                    report(v)
+                end
+            end
+        end
+    end
+end
+
+-- strtab
+
+resolvers.var_value  = resolvers.variable   -- output the value of variable $STRING.
+resolvers.expand_var = resolvers.expansion  -- output variable expansion of STRING.
+
+function resolvers.show_path(str)     -- output search path for file type NAME
+    return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str)))
+end
+
+-- resolvers.find_file(filename)
+-- resolvers.find_file(filename, filetype, mustexist)
+-- resolvers.find_file(filename, mustexist)
+-- resolvers.find_file(filename, filetype)
+
+function resolvers.register_file(files, name, path)
+    if files[name] then
+        if type(files[name]) == 'string' then
+            files[name] = { files[name], path }
+        else
+            files[name] = path
+        end
+    else
+        files[name] = path
+    end
+end
+
+function resolvers.splitmethod(filename)
+    if not filename then
+        return { } -- safeguard
+    elseif type(filename) == "table" then
+        return filename -- already split
+    elseif not find(filename,"://") then
+        return { scheme="file", path = filename, original=filename } -- quick hack
+    else
+        return url.hashed(filename)
+    end
+end
+
+function table.sequenced(t,sep) -- temp here
+    local s = { }
+    for k, v in pairs(t) do -- pairs?
+        s[#s+1] = k .. "=" .. v
+    end
+    return concat(s, sep or " | ")
+end
+
+function resolvers.methodhandler(what, filename, filetype) -- ...
+    local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb
+    local scheme = specification.scheme
+    if resolvers[what][scheme] then
+        if trace_locating then
+            logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification))
+        end
+        return resolvers[what][scheme](filename,filetype) -- todo: specification
+    else
+        return resolvers[what].tex(filename,filetype) -- todo: specification
+    end
+end
+
+function resolvers.clean_path(str)
+    if str then
+        str = gsub(str,"\\","/")
+        str = gsub(str,"^!+","")
+        str = gsub(str,"^~",resolvers.homedir)
+        return str
+    else
+        return nil
+    end
+end
+
+function resolvers.do_with_path(name,func)
+    for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs?
+        func("^"..resolvers.clean_path(v))
+    end
+end
+
+function resolvers.do_with_var(name,func)
+    func(expanded_var(name))
+end
+
+function resolvers.with_files(pattern,handle)
+    for _, hash in ipairs(instance.hashes) do
+        local blobpath = hash.tag
+        local blobtype = hash.type
+        if blobpath then
+            local files = instance.files[blobpath]
+            if files then
+                for k,v in next, files do
+                    if find(k,"^remap:") then
+                        k = files[k]
+                        v = files[k] -- chained
+                    end
+                    if find(k,pattern) then
+                        if type(v) == "string" then
+                            handle(blobtype,blobpath,v,k)
+                        else
+                            for _,vv in pairs(v) do -- ipairs?
+                                handle(blobtype,blobpath,vv,k)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+function resolvers.locate_format(name)
+    local barename, fmtname = name:gsub("%.%a+$",""), ""
+    if resolvers.usecache then
+        local path = file.join(caches.setpath("formats")) -- maybe platform
+        fmtname = file.join(path,barename..".fmt") or ""
+    end
+    if fmtname == "" then
+        fmtname = resolvers.find_files(barename..".fmt")[1] or ""
+    end
+    fmtname = resolvers.clean_path(fmtname)
+    if fmtname ~= "" then
+        local barename = file.removesuffix(fmtname)
+        local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui"
+        if lfs.isfile(luiname) then
+            return barename, luiname
+        elseif lfs.isfile(lucname) then
+            return barename, lucname
+        elseif lfs.isfile(luaname) then
+            return barename, luaname
+        end
+    end
+    return nil, nil
+end
+
+function resolvers.boolean_variable(str,default)
+    local b = resolvers.expansion(str)
+    if b == "" then
+        return default
+    else
+        b = toboolean(b)
+        return (b == nil and default) or b
+    end
+end
+
+texconfig.kpse_init = false
+
+kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } )
+
+-- for a while
+
+input = resolvers
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This module deals with caching data. It sets up the paths and
+implements loaders and savers for tables. Best is to set the
+following variable. When not set, the usual paths will be
+checked. Personally I prefer the (users) temporary path.

+
+
+TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
+
+
+
Currently we do no locking when we write files. This is no real
+problem because most caching involves fonts and the chance of them
+being written at the same time is small. We also need to extend
+luatools with a recache feature.

+--ldx]]--
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache = false  trackers.register("resolvers.cache", function(v) trace_cache = v end)
+
+caches = caches or { }
+
+caches.path     = caches.path or nil
+caches.base     = caches.base or "luatex-cache"
+caches.more     = caches.more or "context"
+caches.direct   = false -- true is faster but may need huge amounts of memory
+caches.tree     = false
+caches.paths    = caches.paths or nil
+caches.force    = false
+caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" }
+
+function caches.temp()
+    local cachepath = nil
+    local function check(list,isenv)
+        if not cachepath then
+            for k=1,#list do
+                local v = list[k]
+                cachepath = (isenv and (os.env[v] or "")) or v or ""
+                if cachepath == "" then
+                    -- next
+                else
+                    cachepath = resolvers.clean_path(cachepath)
+                    if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory"
+                        break
+                    elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
+                        dir.mkdirs(cachepath)
+                        if lfs.isdir(cachepath) and file.iswritable(cachepath) then
+                            break
+                        end
+                    end
+                end
+                cachepath = nil
+            end
+        end
+    end
+    check(resolvers.clean_path_list("TEXMFCACHE") or { })
+    check(caches.defaults,true)
+    if not cachepath then
+        print("\nfatal error: there is no valid (writable) cache path defined\n")
+        os.exit()
+    elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory"
+        print(format("\nfatal error: cache path %s is not a directory\n",cachepath))
+        os.exit()
+    end
+    cachepath = file.collapse_path(cachepath)
+    function caches.temp()
+        return cachepath
+    end
+    return cachepath
+end
+
+function caches.configpath()
+    return table.concat(resolvers.instance.cnffiles,";")
+end
+
+function caches.hashed(tree)
+    return md5.hex(gsub(lower(tree),"[\\\/]+","/"))
+end
+
+function caches.treehash()
+    local tree = caches.configpath()
+    if not tree or tree == "" then
+        return false
+    else
+        return caches.hashed(tree)
+    end
+end
+
+function caches.setpath(...)
+    if not caches.path then
+        if not caches.path then
+            caches.path = caches.temp()
+        end
+        caches.path = resolvers.clean_path(caches.path) -- to be sure
+        caches.tree = caches.tree or caches.treehash()
+        if caches.tree then
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
+        else
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
+        end
+    end
+    if not caches.path then
+        caches.path = '.'
+    end
+    caches.path = resolvers.clean_path(caches.path)
+    if not table.is_empty({...}) then
+        local pth = dir.mkdirs(caches.path,...)
+        return pth
+    end
+    caches.path = dir.expand_name(caches.path)
+    return caches.path
+end
+
+function caches.definepath(category,subcategory)
+    return function()
+        return caches.setpath(category,subcategory)
+    end
+end
+
+function caches.setluanames(path,name)
+    return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
+end
+
+function caches.loaddata(path,name)
+    local tmaname, tmcname = caches.setluanames(path,name)
+    local loader = loadfile(tmcname) or loadfile(tmaname)
+    if loader then
+        return loader()
+    else
+        return false
+    end
+end
+
+--~ function caches.loaddata(path,name)
+--~     local tmaname, tmcname = caches.setluanames(path,name)
+--~     return dofile(tmcname) or dofile(tmaname)
+--~ end
+
+function caches.iswritable(filepath,filename)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    return file.iswritable(tmaname)
+end
+
+function caches.savedata(filepath,filename,data,raw)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    local reduce, simplify = true, true
+    if raw then
+        reduce, simplify = false, false
+    end
+    if caches.direct then
+        file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex
+    else
+        table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true
+    end
+    local cleanup = resolvers.boolean_variable("PURGECACHE", false)
+    local strip = resolvers.boolean_variable("LUACSTRIP", true)
+    utils.lua.compile(tmaname, tmcname, cleanup, strip)
+end
+
+-- here we use the cache for format loading (texconfig.[formatname|jobname])
+
+--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then
+if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then
+    if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc
+    texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt")
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-res'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex"))
+
+local upper, lower, gsub = string.upper, string.lower, string.gsub
+
+local prefixes = { }
+
+prefixes.environment = function(str)
+    return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "")
+end
+
+prefixes.relative = function(str,n)
+    if io.exists(str) then
+        -- nothing
+    elseif io.exists("./" .. str) then
+        str = "./" .. str
+    else
+        local p = "../"
+        for i=1,n or 2 do
+            if io.exists(p .. str) then
+                str = p .. str
+                break
+            else
+                p = p .. "../"
+            end
+        end
+    end
+    return resolvers.clean_path(str)
+end
+
+prefixes.locate = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path((fullname ~= "" and fullname) or str)
+end
+
+prefixes.filename = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str))
+end
+
+prefixes.pathname = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str))
+end
+
+prefixes.env  = prefixes.environment
+prefixes.rel  = prefixes.relative
+prefixes.loc  = prefixes.locate
+prefixes.kpse = prefixes.locate
+prefixes.full = prefixes.locate
+prefixes.file = prefixes.filename
+prefixes.path = prefixes.pathname
+
+local function _resolve_(method,target)
+    if prefixes[method] then
+        return prefixes[method](target)
+    else
+        return method .. ":" .. target
+    end
+end
+
+local function resolve(str)
+    if type(str) == "table" then
+        for k, v in pairs(str) do -- ipairs
+            str[k] = resolve(v) or v
+        end
+    elseif str and str ~= "" then
+        str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_)
+    end
+    return str
+end
+
+resolvers.resolve = resolve
+
+if os.uname then
+
+    for k, v in pairs(os.uname()) do
+        if not prefixes[k] then
+            prefixes[k] = function() return v end
+        end
+    end
+
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+resolvers.finders = resolvers.finders or { }
+resolvers.openers = resolvers.openers or { }
+resolvers.loaders = resolvers.loaders or { }
+
+resolvers.finders.notfound  = { nil }
+resolvers.openers.notfound  = { nil }
+resolvers.loaders.notfound  = { false, nil, 0 }
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-out'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+outputs = outputs or { }
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-con'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache      = false  trackers.register("resolvers.cache",      function(v) trace_cache      = v end)
+local trace_containers = false  trackers.register("resolvers.containers", function(v) trace_containers = v end)
+local trace_storage    = false  trackers.register("resolvers.storage",    function(v) trace_storage    = v end)
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+--[[ldx--
+
Once we found ourselves defining similar cache constructs
+several times, containers were introduced. Containers are used
+to collect tables in memory and reuse them when possible based
+on (unique) hashes (to be provided by the calling function).

+
+
Caching to disk is disabled by default. Version numbers are
+stored in the saved table which makes it possible to change the
+table structures without bothering about the disk cache.

+
+
Examples of usage can be found in the font related code.

+--ldx]]--
+
+containers = containers or { }
+
+containers.usecache = true
+
+local function report(container,tag,name)
+    if trace_cache or trace_containers then
+        logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid')
+    end
+end
+
+local allocated = { }
+
+-- tracing
+
+function containers.define(category, subcategory, version, enabled)
+    return function()
+        if category and subcategory then
+            local c = allocated[category]
+            if not c then
+                c  = { }
+                allocated[category] = c
+            end
+            local s = c[subcategory]
+            if not s then
+                s = {
+                    category = category,
+                    subcategory = subcategory,
+                    storage = { },
+                    enabled = enabled,
+                    version = version or 1.000,
+                    trace = false,
+                    path = caches and caches.setpath and caches.setpath(category,subcategory),
+                }
+                c[subcategory] = s
+            end
+            return s
+        else
+            return nil
+        end
+    end
+end
+
+function containers.is_usable(container, name)
+    return container.enabled and caches and caches.iswritable(container.path, name)
+end
+
+function containers.is_valid(container, name)
+    if name and name ~= "" then
+        local storage = container.storage[name]
+        return storage and not table.is_empty(storage) and storage.cache_version == container.version
+    else
+        return false
+    end
+end
+
+function containers.read(container,name)
+    if container.enabled and caches and not container.storage[name] and containers.usecache then
+        container.storage[name] = caches.loaddata(container.path,name)
+        if containers.is_valid(container,name) then
+            report(container,"loaded",name)
+        else
+            container.storage[name] = nil
+        end
+    end
+    if container.storage[name] then
+        report(container,"reusing",name)
+    end
+    return container.storage[name]
+end
+
+function containers.write(container, name, data)
+    if data then
+        data.cache_version = container.version
+        if container.enabled and caches then
+            local unique, shared = data.unique, data.shared
+            data.unique, data.shared = nil, nil
+            caches.savedata(container.path, name, data)
+            report(container,"saved",name)
+            data.unique, data.shared = unique, shared
+        end
+        report(container,"stored",name)
+        container.storage[name] = data
+    end
+    return data
+end
+
+function containers.content(container,name)
+    return container.storage[name]
+end
+
+function containers.cleanname(name)
+    return (gsub(lower(name),"[^%w%d]+","-"))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-use'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+-- since we want to use the cache instead of the tree, we will now
+-- reimplement the saver.
+
+local save_data = resolvers.save_data
+local load_data = resolvers.load_data
+
+resolvers.cachepath = nil  -- public, for tracing
+resolvers.usecache  = true -- public, for tracing
+
+function resolvers.save_data(dataname)
+    save_data(dataname, function(cachename,dataname)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(cachename))
+        else
+            return file.join(cachename,dataname)
+        end
+    end)
+end
+
+function resolvers.load_data(pathname,dataname,filename)
+    load_data(pathname,dataname,filename,function(dataname,filename)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(pathname))
+        else
+            if not filename or (filename == "") then
+                filename = dataname
+            end
+            return file.join(pathname,filename)
+        end
+    end)
+end
+
+-- we will make a better format, maybe something xml or just text or lua
+
+resolvers.automounted = resolvers.automounted or { }
+
+function resolvers.automount(usecache)
+    local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT'))
+    if table.is_empty(mountpaths) and usecache then
+        mountpaths = { caches.setpath("mount") }
+    end
+    if not table.is_empty(mountpaths) then
+        statistics.starttiming(resolvers.instance)
+        for k, root in pairs(mountpaths) do
+            local f = io.open(root.."/url.tmi")
+            if f then
+                for line in f:lines() do
+                    if line then
+                        if line:find("^[%%#%-]") then -- or %W
+                            -- skip
+                        elseif line:find("^zip://") then
+                            if trace_locating then
+                                logs.report("fileio","mounting %s",line)
+                            end
+                            table.insert(resolvers.automounted,line)
+                            resolvers.usezipfile(line)
+                        end
+                    end
+                end
+                f:close()
+            end
+        end
+        statistics.stoptiming(resolvers.instance)
+    end
+end
+
+-- status info
+
+statistics.register("used config path", function() return caches.configpath()  end)
+statistics.register("used cache path",  function() return caches.temp() or "?" end)
+
+-- experiment (code will move)
+
+function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname
+    local enginebanner = status.list().banner
+    if formatbanner and enginebanner and sourcefile then
+        local luvname = file.replacesuffix(texname,"luv")
+        local luvdata = {
+            enginebanner = enginebanner,
+            formatbanner = formatbanner,
+            sourcehash   = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"),
+            sourcefile   = sourcefile,
+        }
+        io.savedata(luvname,table.serialize(luvdata,true))
+    end
+end
+
+function statistics.check_fmt_status(texname)
+    local enginebanner = status.list().banner
+    if enginebanner and texname then
+        local luvname = file.replacesuffix(texname,"luv")
+        if lfs.isfile(luvname) then
+            local luv = dofile(luvname)
+            if luv and luv.sourcefile then
+                local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown")
+                if luv.enginebanner and luv.enginebanner ~= enginebanner then
+                    return "engine mismatch"
+                end
+                if luv.sourcehash and luv.sourcehash ~= sourcehash then
+                    return "source mismatch"
+                end
+            else
+                return "invalid status file"
+            end
+        else
+            return "missing status file"
+        end
+    end
+    return true
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-zip'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, find = string.format, string.find
+
+local trace_locating, trace_verbose  = false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end)
+
+zip                 = zip or { }
+zip.archives        = zip.archives or { }
+zip.registeredfiles = zip.registeredfiles or { }
+
+local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
+local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators
+
+local archives = zip.archives
+
+-- zip:///oeps.zip?name=bla/bla.tex
+-- zip:///oeps.zip?tree=tex/texmf-local
+
+local function validzip(str) -- todo: use url splitter
+    if not find(str,"^zip://") then
+        return "zip:///" .. str
+    else
+        return str
+    end
+end
+
+function zip.openarchive(name)
+    if not name or name == "" then
+        return nil
+    else
+        local arch = archives[name]
+        if not arch then
+           local full = resolvers.find_file(name) or ""
+           arch = (full ~= "" and zip.open(full)) or false
+           archives[name] = arch
+        end
+       return arch
+    end
+end
+
+function zip.closearchive(name)
+    if not name or (name == "" and archives[name]) then
+        zip.close(archives[name])
+        archives[name] = nil
+    end
+end
+
+-- zip:///texmf.zip?tree=/tex/texmf
+-- zip:///texmf.zip?tree=/tex/texmf-local
+-- zip:///texmf-mine.zip?tree=/tex/texmf-projects
+
+function locators.zip(specification) -- where is this used? startup zips (untested)
+    specification = resolvers.splitmethod(specification)
+    local zipfile = specification.path
+    local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree
+    if trace_locating then
+        if zfile then
+            logs.report("fileio",'! zip locator, found: %s',specification.original)
+        else
+            logs.report("fileio",'? zip locator, not found: %s',specification.original)
+        end
+    end
+end
+
+function hashers.zip(tag,name)
+    if trace_verbose then
+        logs.report("fileio","loading zip file %s as %s",name,tag)
+    end
+    resolvers.usezipfile(format("%s?tree=%s",tag,name))
+end
+
+function concatinators.zip(tag,path,name)
+    if not path or path == "" then
+        return format('%s?name=%s',tag,name)
+    else
+        return format('%s?name=%s/%s',tag,path,name)
+    end
+end
+
+function resolvers.isreadable.zip(name)
+    return true
+end
+
+function finders.zip(specification,filetype)
+    specification = resolvers.splitmethod(specification)
+    if specification.path then
+        local q = url.query(specification.query)
+        if q.name then
+            local zfile = zip.openarchive(specification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'! zip finder, path: %s',specification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    dfile = zfile:close()
+                    if trace_locating then
+                        logs.report("fileio",'+ zip finder, name: %s',q.name)
+                    end
+                    return specification.original
+                end
+            elseif trace_locating then
+                logs.report("fileio",'? zip finder, path %s',specification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip finder, name: %s',filename)
+    end
+    return unpack(finders.notfound)
+end
+
+function openers.zip(specification)
+    local zipspecification = resolvers.splitmethod(specification)
+    if zipspecification.path then
+        local q = url.query(zipspecification.query)
+        if q.name then
+            local zfile = zip.openarchive(zipspecification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'+ zip starter, path: %s',zipspecification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    logs.show_open(specification)
+                    return openers.text_opener(specification,dfile,'zip')
+                end
+            elseif trace_locating then
+                logs.report("fileio",'- zip starter, path %s',zipspecification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip opener, name: %s',filename)
+    end
+    return unpack(openers.notfound)
+end
+
+function loaders.zip(specification)
+    specification = resolvers.splitmethod(specification)
+    if specification.path then
+        local q = url.query(specification.query)
+        if q.name then
+            local zfile = zip.openarchive(specification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'+ zip starter, path: %s',specification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    logs.show_load(filename)
+                    if trace_locating then
+                        logs.report("fileio",'+ zip loader, name: %s',filename)
+                    end
+                    local s = dfile:read("*all")
+                    dfile:close()
+                    return true, s, #s
+                end
+            elseif trace_locating then
+                logs.report("fileio",'- zip starter, path: %s',specification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip loader, name: %s',filename)
+    end
+    return unpack(openers.notfound)
+end
+
+-- zip:///somefile.zip
+-- zip:///somefile.zip?tree=texmf-local -> mount
+
+function resolvers.usezipfile(zipname)
+    zipname = validzip(zipname)
+    if trace_locating then
+        logs.report("fileio",'! zip use, file: %s',zipname)
+    end
+    local specification = resolvers.splitmethod(zipname)
+    local zipfile = specification.path
+    if zipfile and not zip.registeredfiles[zipname] then
+        local tree = url.query(specification.query).tree or ""
+        if trace_locating then
+            logs.report("fileio",'! zip register, file: %s',zipname)
+        end
+        local z = zip.openarchive(zipfile)
+        if z then
+            local instance = resolvers.instance
+            if trace_locating then
+                logs.report("fileio","= zipfile, registering: %s",zipname)
+            end
+            statistics.starttiming(instance)
+            resolvers.prepend_hash('zip',zipname,zipfile)
+            resolvers.extend_texmf_var(zipname) -- resets hashes too
+            zip.registeredfiles[zipname] = z
+            instance.files[zipname] = resolvers.register_zip_file(z,tree or "")
+            statistics.stoptiming(instance)
+        elseif trace_locating then
+            logs.report("fileio","? zipfile, unknown: %s",zipname)
+        end
+    elseif trace_locating then
+        logs.report("fileio",'! zip register, no file: %s',zipname)
+    end
+end
+
+function resolvers.register_zip_file(z,tree)
+    local files, filter = { }, ""
+    if tree == "" then
+        filter = "^(.+)/(.-)$"
+    else
+        filter = format("^%s/(.+)/(.-)$",tree)
+    end
+    if trace_locating then
+        logs.report("fileio",'= zip filter: %s',filter)
+    end
+    local register, n = resolvers.register_file, 0
+    for i in z:files() do
+        local path, name = i.filename:match(filter)
+        if path then
+            if name and name ~= '' then
+                register(files, name, path)
+                n = n + 1
+            else
+                -- directory
+            end
+        else
+            register(files, i.filename, '')
+            n = n + 1
+        end
+    end
+    logs.report("fileio",'= zip entries: %s',n)
+    return files
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-crl'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+curl = curl or { }
+
+curl.cached    = { }
+curl.cachepath = caches.definepath("curl")
+
+local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
+
+function curl.fetch(protocol, name)
+    local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-")
+--  cachename = cachename:gsub("[\\/]", io.fileseparator)
+    cachename = cachename:gsub("[\\]", "/") -- cleanup
+    if not curl.cached[name] then
+        if not io.exists(cachename) then
+            curl.cached[name] = cachename
+            local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://"
+            os.spawn(command)
+        end
+        if io.exists(cachename) then
+            curl.cached[name] = cachename
+        else
+            curl.cached[name] = ""
+        end
+    end
+    return curl.cached[name]
+end
+
+function finders.curl(protocol,filename)
+    local foundname = curl.fetch(protocol, filename)
+    return finders.generic(protocol,foundname,filetype)
+end
+
+function openers.curl(protocol,filename)
+    return openers.generic(protocol,filename)
+end
+
+function loaders.curl(protocol,filename)
+    return loaders.generic(protocol,filename)
+end
+
+-- todo: metamethod
+
+function curl.install(protocol)
+    finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end
+    openers[protocol] = function (filename)          return openers.curl(protocol,filename) end
+    loaders[protocol] = function (filename)          return loaders.curl(protocol,filename) end
+end
+
+curl.install('http')
+curl.install('https')
+curl.install('ftp')
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-kps'] = {
+    version   = 1.001,
+    comment   = "companion to luatools.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This file is used when we want the input handlers to behave like
+kpsewhich. What to do with the following:

+
+
+{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}
+$SELFAUTOLOC    : /usr/tex/bin/platform
+$SELFAUTODIR    : /usr/tex/bin
+$SELFAUTOPARENT : /usr/tex
+
+
+
How about just forgetting about them?

+--ldx]]--
+
+local suffixes = resolvers.suffixes
+local formats  = resolvers.formats
+
+suffixes['gf']                       = { 'gf' }
+suffixes['pk']                       = { 'pk' }
+suffixes['base']                     = { 'base' }
+suffixes['bib']                      = { 'bib' }
+suffixes['bst']                      = { 'bst' }
+suffixes['cnf']                      = { 'cnf' }
+suffixes['mem']                      = { 'mem' }
+suffixes['mf']                       = { 'mf' }
+suffixes['mfpool']                   = { 'pool' }
+suffixes['mft']                      = { 'mft' }
+suffixes['mppool']                   = { 'pool' }
+suffixes['graphic/figure']           = { 'eps', 'epsi' }
+suffixes['texpool']                  = { 'pool' }
+suffixes['PostScript header']        = { 'pro' }
+suffixes['ist']                      = { 'ist' }
+suffixes['web']                      = { 'web', 'ch' }
+suffixes['cweb']                     = { 'w', 'web', 'ch' }
+suffixes['cmap files']               = { 'cmap' }
+suffixes['lig files']                = { 'lig' }
+suffixes['bitmap font']              = { }
+suffixes['MetaPost support']         = { }
+suffixes['TeX system documentation'] = { }
+suffixes['TeX system sources']       = { }
+suffixes['dvips config']             = { }
+suffixes['type42 fonts']             = { }
+suffixes['web2c files']              = { }
+suffixes['other text files']         = { }
+suffixes['other binary files']       = { }
+suffixes['opentype fonts']           = { 'otf' }
+
+suffixes['fmt']                      = { 'fmt' }
+suffixes['texmfscripts']             = { 'rb','lua','py','pl' }
+
+suffixes['pdftex config']            = { }
+suffixes['Troff fonts']              = { }
+
+suffixes['ls-R']                     = { }
+
+--[[ldx--
+
If you wondered abou tsome of the previous mappings, how about
+the next bunch:

+--ldx]]--
+
+formats['bib']                      = ''
+formats['bst']                      = ''
+formats['mft']                      = ''
+formats['ist']                      = ''
+formats['web']                      = ''
+formats['cweb']                     = ''
+formats['MetaPost support']         = ''
+formats['TeX system documentation'] = ''
+formats['TeX system sources']       = ''
+formats['Troff fonts']              = ''
+formats['dvips config']             = ''
+formats['graphic/figure']           = ''
+formats['ls-R']                     = ''
+formats['other text files']         = ''
+formats['other binary files']       = ''
+
+formats['gf']                       = ''
+formats['pk']                       = ''
+formats['base']                     = 'MFBASES'
+formats['cnf']                      = ''
+formats['mem']                      = 'MPMEMS'
+formats['mf']                       = 'MFINPUTS'
+formats['mfpool']                   = 'MFPOOL'
+formats['mppool']                   = 'MPPOOL'
+formats['texpool']                  = 'TEXPOOL'
+formats['PostScript header']        = 'TEXPSHEADERS'
+formats['cmap files']               = 'CMAPFONTS'
+formats['type42 fonts']             = 'T42FONTS'
+formats['web2c files']              = 'WEB2C'
+formats['pdftex config']            = 'PDFTEXCONFIG'
+formats['texmfscripts']             = 'TEXMFSCRIPTS'
+formats['bitmap font']              = ''
+formats['lig files']                = 'LIGFONTS'
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-aux'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+local trace_verbose = false  trackers.register("resolvers.verbose", function(v) trace_verbose = v end)
+
+function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix
+    local scriptpath = "scripts/context/lua"
+    newname = file.addsuffix(newname,"lua")
+    local oldscript = resolvers.clean_path(oldname)
+    if trace_verbose then
+        logs.report("fileio","to be replaced old script %s", oldscript)
+    end
+    local newscripts = resolvers.find_files(newname) or { }
+    if #newscripts == 0 then
+        if trace_verbose then
+            logs.report("fileio","unable to locate new script")
+        end
+    else
+        for i=1,#newscripts do
+            local newscript = resolvers.clean_path(newscripts[i])
+            if trace_verbose then
+                logs.report("fileio","checking new script %s", newscript)
+            end
+            if oldscript == newscript then
+                if trace_verbose then
+                    logs.report("fileio","old and new script are the same")
+                end
+            elseif not find(newscript,scriptpath) then
+                if trace_verbose then
+                    logs.report("fileio","new script should come from %s",scriptpath)
+                end
+            elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then
+                if trace_verbose then
+                    logs.report("fileio","invalid new script name")
+                end
+            else
+                local newdata = io.loaddata(newscript)
+                if newdata then
+                    if trace_verbose then
+                        logs.report("fileio","old script content replaced by new content")
+                    end
+                    io.savedata(oldscript,newdata)
+                    break
+                elseif trace_verbose then
+                    logs.report("fileio","unable to load new script")
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- loads *.tmf files in minimal tree roots (to be optimized and documented)
+
+function resolvers.check_environment(tree)
+    logs.simpleline()
+    os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME'))
+    os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform()))
+    os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",''))
+    os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS'))
+    logs.simpleline()
+    logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH'))
+    logs.simple("preset : TEXOS   => %s", os.getenv('TEXOS'))
+    logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS'))
+    logs.simple("preset : TMP     => %s", os.getenv('TMP'))
+    logs.simple('')
+end
+
+function resolvers.load_environment(name) -- todo: key=value as well as lua
+    local f = io.open(name)
+    if f then
+        for line in f:lines() do
+            if line:find("^[%%%#]") then
+                -- skip comment
+            else
+                local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$")
+                if how then
+                    value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end)
+                        if how == "=" or how == "<<" then
+                            os.setenv(key,value)
+                    elseif how == "?" or how == "??" then
+                            os.setenv(key,os.getenv(key) or value)
+                    elseif how == "<" or how == "+=" then
+                        if os.getenv(key) then
+                            os.setenv(key,os.getenv(key) .. io.fileseparator .. value)
+                        else
+                            os.setenv(key,value)
+                        end
+                    elseif how == ">" or how == "=+" then
+                        if os.getenv(key) then
+                            os.setenv(key,value .. io.pathseparator .. os.getenv(key))
+                        else
+                            os.setenv(key,value)
+                        end
+                    end
+                end
+            end
+        end
+        f:close()
+    end
+end
+
+function resolvers.load_tree(tree)
+    if tree and tree ~= "" then
+        local setuptex = 'setuptex.tmf'
+        if lfs.attributes(tree, "mode") == "directory" then -- check if not nil
+            setuptex = tree .. "/" .. setuptex
+        else
+            setuptex = tree
+        end
+        if io.exists(setuptex) then
+            resolvers.check_environment(tree)
+            resolvers.load_environment(setuptex)
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-sta'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this code is used in the updater
+
+states          = states          or { }
+states.data     = states.data     or { }
+states.hash     = states.hash     or { }
+states.tag      = states.tag      or ""
+states.filename = states.filename or ""
+
+function states.save(filename,tag)
+    tag = tag or states.tag
+    filename = file.addsuffix(filename or states.filename,'lus')
+    io.savedata(filename,
+        "-- generator : luat-sta.lua\n" ..
+        "-- state tag : " .. tag .. "\n\n" ..
+        table.serialize(states.data[tag or states.tag] or {},true)
+    )
+end
+
+function states.load(filename,tag)
+    states.filename = filename
+    states.tag = tag or "whatever"
+    states.filename = file.addsuffix(states.filename,'lus')
+    states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { }
+end
+
+function states.set_by_tag(tag,key,value,default,persistent)
+    local d, h = states.data[tag], states.hash[tag]
+    if d then
+        if type(d) == "table" then
+            local dkey, hkey = key, key
+            local pre, post = key:match("(.+)%.([^%.]+)$")
+            if pre and post then
+                for k in pre:gmatch("[^%.]+") do
+                    local dk = d[k]
+                    if not dk then
+                        dk = { }
+                        d[k] = dk
+                    end
+                    d = dk
+                end
+                dkey, hkey = post, key
+            end
+            if type(value) == nil then
+                value = value or default
+            elseif persistent then
+                value = value or d[dkey] or default
+            else
+                value = value or default
+            end
+            d[dkey], h[hkey] = value, value
+        elseif type(d) == "string" then
+            -- weird
+            states.data[tag], states.hash[tag] = value, value
+        end
+    end
+end
+
+function states.get_by_tag(tag,key,default)
+    local h = states.hash[tag]
+    if h and h[key] then
+        return h[key]
+    else
+        local d = states.data[tag]
+        if d then
+            for k in key:gmatch("[^%.]+") do
+                local dk = d[k]
+                if dk then
+                    d = dk
+                else
+                    return default
+                end
+            end
+            return d or default
+        end
+    end
+end
+
+function states.set(key,value,default,persistent)
+    states.set_by_tag(states.tag,key,value,default,persistent)
+end
+
+function states.get(key,default)
+    return states.get_by_tag(states.tag,key,default)
+end
+
+--~ states.data.update = {
+--~ 	["version"] = {
+--~ 		["major"] = 0,
+--~ 		["minor"] = 1,
+--~ 	},
+--~ 	["rsync"] = {
+--~ 		["server"]     = "contextgarden.net",
+--~ 		["module"]     = "minimals",
+--~ 		["repository"] = "current",
+--~ 		["flags"]      = "-rpztlv --stats",
+--~ 	},
+--~ 	["tasks"] = {
+--~ 		["update"] = true,
+--~ 		["make"]   = true,
+--~         ["delete"] = false,
+--~ 	},
+--~ 	["platform"] = {
+--~ 		["host"]  = true,
+--~ 		["other"] = {
+--~ 			["mswin"]     = false,
+--~ 			["linux"]     = false,
+--~ 			["linux-64"]  = false,
+--~ 			["osx-intel"] = false,
+--~ 			["osx-ppc"]   = false,
+--~ 			["sun"]       = false,
+--~ 		},
+--~ 	},
+--~ 	["context"] = {
+--~ 		["available"] = {"current", "beta", "alpha", "experimental"},
+--~ 		["selected"]  = "current",
+--~ 	},
+--~ 	["formats"] = {
+--~ 		["cont-en"] = true,
+--~ 		["cont-nl"] = true,
+--~ 		["cont-de"] = false,
+--~ 		["cont-cz"] = false,
+--~ 		["cont-fr"] = false,
+--~ 		["cont-ro"] = false,
+--~ 	},
+--~ 	["engine"] = {
+--~ 		["pdftex"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 				["pdftex"] = true,
+--~ 			},
+--~ 		},
+--~ 		["luatex"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 			},
+--~ 		},
+--~ 		["xetex"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 				["xetex"] = false,
+--~ 			},
+--~ 		},
+--~ 		["metapost"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 				["mpost"] = true,
+--~ 				["metafun"] = true,
+--~ 			},
+--~ 		},
+--~ 	},
+--~ 	["fonts"] = {
+--~ 	},
+--~ 	["doc"] = {
+--~ 	},
+--~ 	["modules"] = {
+--~ 		["f-urwgaramond"] = false,
+--~ 		["f-urwgothic"] = false,
+--~ 		["t-bnf"] = false,
+--~ 		["t-chromato"] = false,
+--~ 		["t-cmscbf"] = false,
+--~ 		["t-cmttbf"] = false,
+--~ 		["t-construction-plan"] = false,
+--~ 		["t-degrade"] = false,
+--~ 		["t-french"] = false,
+--~ 		["t-lettrine"] = false,
+--~ 		["t-lilypond"] = false,
+--~ 		["t-mathsets"] = false,
+--~ 		["t-tikz"] = false,
+--~ 		["t-typearea"] = false,
+--~ 		["t-vim"] = false,
+--~ 	},
+--~ }
+
+--~ states.save("teststate", "update")
+--~ states.load("teststate", "update")
+
+--~ print(states.get_by_tag("update","rsync.server","unknown"))
+--~ states.set_by_tag("update","rsync.server","oeps")
+--~ print(states.get_by_tag("update","rsync.server","unknown"))
+--~ states.save("teststate", "update")
+--~ states.load("teststate", "update")
+--~ print(states.get_by_tag("update","rsync.server","unknown"))
+
+
+end -- of closure
+-- end library merge
+
+own = { } -- not local
+
+own.libs = { -- todo: check which ones are really needed
+    'l-string.lua',
+    'l-lpeg.lua',
+    'l-table.lua',
+    'l-io.lua',
+    'l-number.lua',
+    'l-set.lua',
+    'l-os.lua',
+    'l-file.lua',
+    'l-md5.lua',
+    'l-dir.lua',
+    'l-boolean.lua',
+    'l-math.lua',
+--  'l-unicode.lua',
+--  'l-tex.lua',
+    'l-utils.lua',
+--  'l-xml.lua',
+    'lxml-tab.lua',
+    'lxml-pth.lua',
+    'lxml-ent.lua',
+    'lxml-mis.lua',
+    'trac-tra.lua',
+    'luat-env.lua',
+    'trac-inf.lua',
+    'trac-log.lua',
+    'data-res.lua',
+    'data-tmp.lua',
+    'data-pre.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-tex.lua',
+--  'data-bin.lua',
+    'data-zip.lua',
+    'data-crl.lua',
+--  'data-lua.lua',
+    'data-kps.lua', -- so that we can replace kpsewhich
+    'data-aux.lua', -- updater
+    'data-tmf.lua', -- tree files
+    -- needed ?
+    'luat-sta.lua', -- states
+}
+
+-- We need this hack till luatex is fixed.
+--
+-- for k,v in pairs(arg) do print(k,v) end
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+-- End of hack.
+
+own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua'
+
+own.path = string.match(own.name,"^(.+)[\\/].-$") or "."
+own.list = { '.' }
+if own.path ~= '.' then
+    table.insert(own.list,own.path)
+end
+table.insert(own.list,own.path.."/../../../tex/context/base")
+table.insert(own.list,own.path.."/mtx")
+table.insert(own.list,own.path.."/../sources")
+
+local function locate_libs()
+    for _, lib in pairs(own.libs) do
+        for _, pth in pairs(own.list) do
+            local filename = string.gsub(pth .. "/" .. lib,"\\","/")
+            local codeblob = loadfile(filename)
+            if codeblob then
+                codeblob()
+                own.list = { pth } -- speed up te search
+                break
+            end
+        end
+    end
+end
+
+if not resolvers then
+    locate_libs()
+end
+
+if not resolvers then
+    print("")
+    print("Mtxrun is unable to start up due to lack of libraries. You may")
+    print("try to run 'lua mtxrun.lua --selfmerge' in the path where this")
+    print("script is located (normally under ..../scripts/context/lua) which")
+    print("will make this script library independent.")
+    os.exit()
+end
+
+logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false)
+
+local instance = resolvers.reset()
+
+runners  = runners  or { } -- global
+messages = messages or { }
+
+messages.help = [[
+--script              run an mtx script (--noquotes)
+--execute             run a script or program (--noquotes)
+--resolve             resolve prefixed arguments
+--ctxlua              run internally (using preloaded libs)
+--locate              locate given filename
+
+--autotree            use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree'
+--tree=pathtotree     use given texmf tree (default file: 'setuptex.tmf')
+--environment=name    use given (tmf) environment file
+--path=runpath        go to given path before execution
+--ifchanged=filename  only execute when given file has changed (md checksum)
+--iftouched=old,new   only execute when given file has changed (time stamp)
+
+--make                create stubs for (context related) scripts
+--remove              remove stubs (context related) scripts
+--stubpath=binpath    paths where stubs wil be written
+--windows             create windows (mswin) stubs
+--unix                create unix (linux) stubs
+
+--verbose             give a bit more info
+--engine=str          target engine
+--progname=str        format or backend
+
+--edit                launch editor with found file
+--launch (--all)      launch files like manuals, assumes os support
+
+--intern              run script using built in libraries
+
+--usekpse            use kpse as fallback (when no mkiv and cache installed, often slower)
+--forcekpse          force using kpse (handy when no mkiv and cache installed but less functionality)
+]]
+
+runners.applications = {
+    ["lua"] = "luatex --luaonly",
+    ["luc"] = "luatex --luaonly",
+    ["pl"] = "perl",
+    ["py"] = "python",
+    ["rb"] = "ruby",
+}
+
+runners.suffixes = {
+    'rb', 'lua', 'py', 'pl'
+}
+
+runners.registered = {
+    texexec      = { 'texexec.rb',      true  },  -- context mkii runner (only tool not to be luafied)
+    texutil      = { 'texutil.rb',      true  },  -- old perl based index sorter for mkii (old versions need it)
+    texfont      = { 'texfont.pl',      true  },  -- perl script that makes mkii font metric files
+    texfind      = { 'texfind.pl',      false },  -- perltk based tex searching tool, mostly used at pragma
+    texshow      = { 'texshow.pl',      false },  -- perltk based context help system, will be luafied
+ -- texwork      = { \texwork.pl',      false },  -- perltk based editing environment, only used at pragma
+
+    makempy      = { 'makempy.pl',      true  },
+    mptopdf      = { 'mptopdf.pl',      true  },
+    pstopdf      = { 'pstopdf.rb',      true  },  -- converts ps (and some more) images, does some cleaning (replaced)
+
+--  examplex     = { 'examplex.rb',     false },
+    concheck     = { 'concheck.rb',     false },
+
+    runtools     = { 'runtools.rb',     true  },
+    textools     = { 'textools.rb',     true  },
+    tmftools     = { 'tmftools.rb',     true  },
+    ctxtools     = { 'ctxtools.rb',     true  },
+    rlxtools     = { 'rlxtools.rb',     true  },
+    pdftools     = { 'pdftools.rb',     true  },
+    mpstools     = { 'mpstools.rb',     true  },
+--  exatools     = { 'exatools.rb',     true  },
+    xmltools     = { 'xmltools.rb',     true  },
+--  luatools     = { 'luatools.lua',    true  },
+    mtxtools     = { 'mtxtools.rb',     true  },
+
+    pdftrimwhite = { 'pdftrimwhite.pl', false }
+}
+
+runners.launchers = {
+    windows = { },
+    unix = { }
+}
+
+function runners.prepare()
+    local checkname = environment.argument("ifchanged")
+    if checkname and checkname ~= "" then
+        local oldchecksum = file.loadchecksum(checkname)
+        local newchecksum = file.checksum(checkname)
+        if oldchecksum == newchecksum then
+            logs.simple("file '%s' is unchanged",checkname)
+            return "skip"
+        else
+            logs.simple("file '%s' is changed, processing started",checkname)
+        end
+        file.savechecksum(checkname)
+    end
+    local oldname, newname = string.split(environment.argument("iftouched") or "", ",")
+    if oldname and newname and oldname ~= "" and newname ~= "" then
+        if not file.needs_updating(oldname,newname) then
+            logs.simple("file '%s' and '%s' have same age",oldname,newname)
+            return "skip"
+        else
+            logs.simple("file '%s' is older than '%s'",oldname,newname)
+        end
+    end
+    local tree = environment.argument('tree') or ""
+    if environment.argument('autotree') then
+        tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree
+    end
+    if tree and tree ~= "" then
+        resolvers.load_tree(tree)
+    end
+    local env = environment.argument('environment') or ""
+    if env and env ~= "" then
+        for _,e in pairs(string.split(env)) do
+            -- maybe force suffix when not given
+            resolvers.load_tree(e)
+        end
+    end
+    local runpath = environment.argument("path")
+    if runpath and not lfs.chdir(runpath) then
+        logs.simple("unable to change to path '%s'",runpath)
+        return "error"
+    end
+    return "run"
+end
+
+function runners.execute_script(fullname,internal)
+    local noquote = environment.argument("noquotes")
+    if fullname and fullname ~= "" then
+        local state = runners.prepare()
+        if state == 'error' then
+            return false
+        elseif state == 'skip' then
+            return true
+        elseif state == "run" then
+            instance.progname = environment.argument("progname") or instance.progname
+            instance.format   = environment.argument("format")   or instance.format
+            local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), ""
+            if path ~= "" then
+                result = fullname
+            elseif name then
+                name = name:gsub("^int[%a]*:",function()
+                    internal = true
+                    return ""
+                end )
+                name = name:gsub("^script:","")
+                if suffix == "" and runners.registered[name] and runners.registered[name][1] then
+                    name = runners.registered[name][1]
+                    suffix = file.extname(name)
+                end
+                if suffix == "" then
+                    -- loop over known suffixes
+                    for _,s in pairs(runners.suffixes) do
+                        result = resolvers.find_file(name .. "." .. s, 'texmfscripts')
+                        if result ~= "" then
+                            break
+                        end
+                    end
+                elseif runners.applications[suffix] then
+                    result = resolvers.find_file(name, 'texmfscripts')
+                else
+                    -- maybe look on path
+                    result = resolvers.find_file(name, 'other text files')
+                end
+            end
+            if result and result ~= "" then
+                local before, after = environment.split_arguments(fullname) -- already done
+                environment.arguments_before, environment.arguments_after = before, after
+                if internal then
+                    arg = { } for _,v in pairs(after) do arg[#arg+1] = v end
+                    dofile(result)
+                else
+                    local binary = runners.applications[file.extname(result)]
+                    if binary and binary ~= "" then
+                        result = binary .. " " .. result
+                    end
+                    local command = result .. " " .. environment.reconstruct_commandline(after,noquote)
+                    if logs.verbose then
+                        logs.simpleline()
+                        logs.simple("executing: %s",command)
+                        logs.simpleline()
+                        logs.simpleline()
+                        io.flush()
+                    end
+                    local code = os.exec(command) -- maybe spawn
+                    return code == 0
+                end
+            end
+        end
+    end
+    return false
+end
+
+function runners.execute_program(fullname)
+    local noquote = environment.argument("noquotes")
+    if fullname and fullname ~= "" then
+        local state = runners.prepare()
+        if state == 'error' then
+            return false
+        elseif state == 'skip' then
+            return true
+        elseif state == "run" then
+            local before, after = environment.split_arguments(fullname)
+            environment.initialize_arguments(after)
+            fullname = fullname:gsub("^bin:","")
+            local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "")
+            logs.simpleline()
+            logs.simple("executing: %s",command)
+            logs.simpleline()
+            logs.simpleline()
+            io.flush()
+            local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn
+            return code == 0
+        end
+    end
+    return false
+end
+
+-- the --usekpse flag will fallback on kpse
+
+local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010'
+local unix_stub    = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010'
+
+function runners.handle_stubs(create)
+    local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported
+    local windows  = environment.argument('windows') or environment.argument('mswin') or false
+    local unix     = environment.argument('unix') or environment.argument('linux') or false
+    if not windows and not unix then
+        if os.platform == "unix" then
+            unix = true
+        else
+            windows = true
+        end
+    end
+    for _,v in pairs(runners.registered) do
+        local name, doit = v[1], v[2]
+        if doit then
+            local base = string.gsub(file.basename(name), "%.(.-)$", "")
+            if create then
+                if windows then
+                    io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name))
+                    logs.simple("windows stub for '%s' created",base)
+                end
+                if unix then
+                    io.savedata(file.join(stubpath,base),string.format(unix_stub,name))
+                    logs.simple("unix stub for '%s' created",base)
+                end
+            else
+                if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then
+                    logs.simple("windows stub for '%s' removed", base)
+                end
+                if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then
+                    logs.simple("unix stub for '%s' removed",base)
+                end
+            end
+        end
+    end
+end
+
+function runners.resolve_string(filename)
+    if filename and filename ~= "" then
+        runners.report_location(resolvers.resolve(filename))
+    end
+end
+
+function runners.locate_file(filename)
+    -- differs from texmfstart where locate appends .com .exe .bat ... todo
+    if filename and filename ~= "" then
+        runners.report_location(resolvers.find_given_file(filename))
+    end
+end
+
+function runners.locate_platform()
+    runners.report_location(os.currentplatform())
+end
+
+function runners.report_location(result)
+    if logs.verbose then
+        logs.simpleline()
+        if result and result ~= "" then
+            logs.simple(result)
+        else
+            logs.simple("not found")
+        end
+    else
+        io.write(result)
+    end
+end
+
+function runners.edit_script(filename) -- we assume that vim is present on most systems
+    local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim'
+    local rest = resolvers.resolve(filename)
+    if rest ~= "" then
+        local command = editor .. " " .. rest
+        if logs.verbose then
+            logs.simpleline()
+            logs.simple("starting editor: %s",command)
+            logs.simple_line()
+            logs.simple_line()
+        end
+        os.launch(command)
+    end
+end
+
+function runners.save_script_session(filename, list)
+    local t = { }
+    for _, key in ipairs(list) do
+        t[key] = environment.arguments[key]
+    end
+    io.savedata(filename,table.serialize(t,true))
+end
+
+function runners.load_script_session(filename)
+    if lfs.isfile(filename) then
+        local t = io.loaddata(filename)
+        if t then
+            t = loadstring(t)
+            if t then t = t() end
+            for key, value in pairs(t) do
+                environment.arguments[key] = value
+            end
+        end
+    end
+end
+
+function resolvers.launch(str)
+    -- maybe we also need to test on mtxrun.launcher.suffix environment
+    -- variable or on windows consult the assoc and ftype vars and such
+    local launchers = runners.launchers[os.platform] if launchers then
+        local suffix = file.extname(str) if suffix then
+            local runner = launchers[suffix] if runner then
+                str = runner .. " " .. str
+            end
+        end
+    end
+    os.launch(str)
+end
+
+function runners.launch_file(filename)
+    instance.allresults = true
+    logs.setverbose(true)
+    local pattern = environment.arguments["pattern"]
+    if not pattern or pattern == "" then
+        pattern = filename
+    end
+    if not pattern or pattern == "" then
+        logs.simple("provide name or --pattern=")
+    else
+        local t = resolvers.find_files(pattern)
+        if not t or #t == 0 then
+            t = resolvers.find_files("*/" .. pattern)
+        end
+        if not t or #t == 0 then
+            t = resolvers.find_files("*/" .. pattern .. "*")
+        end
+        if t and #t > 0 then
+            if environment.arguments["all"] then
+                for _, v in pairs(t) do
+                    logs.simple("launching %s", v)
+                    resolvers.launch(v)
+                end
+            else
+                logs.simple("launching %s", t[1])
+                resolvers.launch(t[1])
+            end
+        else
+            logs.simple("no match for %s", pattern)
+        end
+    end
+end
+
+function runners.find_mtx_script(filename)
+    local function found(name)
+        local path = file.dirname(name)
+        if path and path ~= "" then
+            return false
+        else
+            local fullname = own and own.path and file.join(own.path,name)
+            return io.exists(fullname) and fullname
+        end
+    end
+    filename = file.addsuffix(filename,"lua")
+    local basename = file.removesuffix(file.basename(filename))
+    local suffix = file.extname(filename)
+    -- qualified path, raw name
+    local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- current path, raw name
+    fullname = "./" .. filename
+    fullname = io.exists(fullname) and fullname
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, mtx-
+    fullname = "mtx-" .. filename
+    fullname = found(fullname) or resolvers.find_file(fullname)
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, mtx-s
+    fullname = "mtx-" .. basename .. "s" .. "." .. suffix
+    fullname = found(fullname) or resolvers.find_file(fullname)
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, mtx-
+    fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix
+    fullname = found(fullname) or resolvers.find_file(fullname)
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, just 
+    fullname = resolvers.find_file(filename)
+    return fullname
+end
+
+function runners.execute_ctx_script(filename,arguments)
+    local fullname = runners.find_mtx_script(filename) or ""
+    -- retyr after generate but only if --autogenerate
+    if fullname == "" and environment.argument("autogenerate") then -- might become the default
+        instance.renewcache = true
+        logs.setverbose(true)
+        resolvers.load()
+        --
+        fullname = runners.find_mtx_script(filename) or ""
+    end
+    -- that should do it
+    if fullname ~= "" then
+        local state = runners.prepare()
+        if state == 'error' then
+            return false
+        elseif state == 'skip' then
+            return true
+        elseif state == "run" then
+            -- load and save ... kind of undocumented
+            arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end
+            environment.initialize_arguments(arg)
+            local loadname = environment.arguments['load']
+            if loadname then
+                if type(loadname) ~= "string" then loadname = file.basename(fullname) end
+                loadname = file.replacesuffix(loadname,"cfg")
+                runners.load_script_session(loadname)
+            end
+            filename = environment.files[1]
+            if logs.verbose then
+                logs.simple("using script: %s\n",fullname)
+            end
+            dofile(fullname)
+            local savename = environment.arguments['save']
+            if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then
+                if type(savename) ~= "string" then savename = file.basename(fullname) end
+                savename = file.replacesuffix(savename,"cfg")
+                runners.save_script_session(savename, runners.save_list)
+            end
+            return true
+        end
+    else
+        logs.setverbose(true)
+        filename = file.addsuffix(filename,"lua")
+        if filename == "" then
+            logs.simple("unknown script, no name given")
+        elseif file.is_qualified_path(filename) then
+            logs.simple("unknown script '%s'",filename)
+        else
+            logs.simple("unknown script '%s' or 'mtx-%s'",filename,filename)
+        end
+        return false
+    end
+end
+
+function runners.timed(action)
+    statistics.timed(action)
+end
+
+-- this is a bit dirty ... first we store the first filename and next we
+-- split the arguments so that we only see the ones meant for this script
+-- ... later we will use the second half
+
+local filename = environment.files[1] or ""
+local ok      = true
+
+local before, after = environment.split_arguments(filename)
+environment.arguments_before, environment.arguments_after = before, after
+environment.initialize_arguments(before)
+
+instance.engine   = environment.argument("engine")   or 'luatex'
+instance.progname = environment.argument("progname") or 'context'
+instance.lsrmode  = environment.argument("lsr")      or false
+
+-- maybe the unset has to go to this level
+
+if environment.argument("usekpse") or environment.argument("forcekpse") then
+
+    os.setenv("engine","")
+    os.setenv("progname","")
+
+    local remapper = {
+        otf = "opentype fonts",
+        ttf = "truetype fonts",
+        ttc = "truetype fonts",
+        pfb = "type1 fonts",
+        other = "other text files",
+    }
+
+    local function kpse_initialized()
+        texconfig.kpse_init = true
+        local t = os.clock()
+        local k = kpse.original.new("luatex",instance.progname)
+        local dummy = k:find_file("mtxrun.lua") -- so that we're initialized
+        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t)
+        kpse_initialized = function() return k end
+        return k
+    end
+
+    local find_file = resolvers.find_file
+    local show_path = resolvers.show_path
+
+    if environment.argument("forcekpse") then
+
+        function resolvers.find_file(name,kind)
+            return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or ""
+        end
+        function resolvers.show_path(name)
+            return (kpse_initialized():show_path(name)) or ""
+        end
+
+    elseif environment.argument("usekpse") then
+
+        resolvers.load()
+
+        function resolvers.find_file(name,kind)
+            local found = find_file(name,kind) or ""
+            if found ~= "" then
+                return found
+            else
+                return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or ""
+            end
+        end
+        function resolvers.show_path(name)
+            local found = show_path(name) or ""
+            if found ~= "" then
+                return found
+            else
+                return (kpse_initialized():show_path(name)) or ""
+            end
+        end
+
+    end
+
+else
+
+    resolvers.load()
+
+end
+
+
+if environment.argument("selfmerge") then
+    -- embed used libraries
+    utils.merger.selfmerge(own.name,own.libs,own.list)
+elseif environment.argument("selfclean") then
+    -- remove embedded libraries
+    utils.merger.selfclean(own.name)
+elseif environment.argument("selfupdate") then
+    logs.setverbose(true)
+    resolvers.update_script(own.name,"mtxrun")
+elseif environment.argument("ctxlua") or environment.argument("internal") then
+    -- run a script by loading it (using libs)
+    ok = runners.execute_script(filename,true)
+elseif environment.argument("script") or environment.argument("s") then
+    -- run a script by loading it (using libs), pass args
+    ok = runners.execute_ctx_script(filename,after)
+elseif environment.argument("execute") then
+    -- execute script
+    ok = runners.execute_script(filename)
+elseif environment.argument("direct") then
+    -- equals bin:
+    ok = runners.execute_program(filename)
+elseif environment.argument("edit") then
+    -- edit file
+    runners.edit_script(filename)
+elseif environment.argument("launch") then
+    runners.launch_file(filename)
+elseif environment.argument("make") then
+    -- make stubs
+    runners.handle_stubs(true)
+elseif environment.argument("remove") then
+    -- remove stub
+    runners.handle_stubs(false)
+elseif environment.argument("resolve") then
+    -- resolve string
+    runners.resolve_string(filename)
+elseif environment.argument("locate") then
+    -- locate file
+    runners.locate_file(filename)
+elseif environment.argument("platform")then
+    -- locate platform
+    runners.locate_platform()
+elseif environment.argument("help") or filename=='help' or filename == "" then
+    logs.help(messages.help)
+    -- execute script
+elseif filename:find("^bin:") then
+    ok = runners.execute_program(filename)
+else
+    ok = runners.execute_script(filename)
+end
+
+if os.platform == "unix" then
+    io.write("\n")
+end
+
+if ok == false then ok = 1 elseif ok == true then ok = 0 end
+
+os.exit(ok)
diff --git a/scripts/context/stubs/mswin/mtxtools.bat b/scripts/context/stubs/mswin/mtxtools.bat
new file mode 100755
index 000000000..9554220c4
--- /dev/null
+++ b/scripts/context/stubs/mswin/mtxtools.bat
@@ -0,0 +1,5 @@
+@echo off
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute mtxtools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/pdftools.bat b/scripts/context/stubs/mswin/pdftools.bat
index adc48eacf..5e893fb2a 100755
--- a/scripts/context/stubs/mswin/pdftools.bat
+++ b/scripts/context/stubs/mswin/pdftools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart pdftools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute pdftools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/pdftrimwhite.bat b/scripts/context/stubs/mswin/pdftrimwhite.bat
deleted file mode 100755
index a7034b400..000000000
--- a/scripts/context/stubs/mswin/pdftrimwhite.bat
+++ /dev/null
@@ -1,2 +0,0 @@
-@echo off
-texmfstart pdftrimwhite.pl %*
diff --git a/scripts/context/stubs/mswin/pstopdf.bat b/scripts/context/stubs/mswin/pstopdf.bat
index 248e34caf..f8d4325f4 100755
--- a/scripts/context/stubs/mswin/pstopdf.bat
+++ b/scripts/context/stubs/mswin/pstopdf.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart pstopdf.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute pstopdf.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/rlxtools.bat b/scripts/context/stubs/mswin/rlxtools.bat
index b78dec13b..82f09665a 100755
--- a/scripts/context/stubs/mswin/rlxtools.bat
+++ b/scripts/context/stubs/mswin/rlxtools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart rlxtools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute rlxtools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/runtools.bat b/scripts/context/stubs/mswin/runtools.bat
index 68a7b4f97..f471e747d 100755
--- a/scripts/context/stubs/mswin/runtools.bat
+++ b/scripts/context/stubs/mswin/runtools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart runtools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute runtools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/texexec.bat b/scripts/context/stubs/mswin/texexec.bat
index 02b297b9c..acbe41fd8 100755
--- a/scripts/context/stubs/mswin/texexec.bat
+++ b/scripts/context/stubs/mswin/texexec.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart texexec.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute texexec.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/texexec.cmd b/scripts/context/stubs/mswin/texexec.cmd
new file mode 100644
index 000000000..acbe41fd8
--- /dev/null
+++ b/scripts/context/stubs/mswin/texexec.cmd
@@ -0,0 +1,5 @@
+@echo off
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute texexec.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/texfind.bat b/scripts/context/stubs/mswin/texfind.bat
deleted file mode 100755
index b7c11cbca..000000000
--- a/scripts/context/stubs/mswin/texfind.bat
+++ /dev/null
@@ -1,2 +0,0 @@
-@echo off
-texmfstart texfind %*
diff --git a/scripts/context/stubs/mswin/texfont.bat b/scripts/context/stubs/mswin/texfont.bat
index 3134bf14c..98e9f7c76 100755
--- a/scripts/context/stubs/mswin/texfont.bat
+++ b/scripts/context/stubs/mswin/texfont.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart texfont.pl %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute texfont.pl %*
+endlocal
diff --git a/scripts/context/stubs/mswin/texmfstart.cmd b/scripts/context/stubs/mswin/texmfstart.cmd
new file mode 100644
index 000000000..47a10cc54
--- /dev/null
+++ b/scripts/context/stubs/mswin/texmfstart.cmd
@@ -0,0 +1,5 @@
+@echo off
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse %*
+endlocal
diff --git a/scripts/context/stubs/mswin/texshow.bat b/scripts/context/stubs/mswin/texshow.bat
deleted file mode 100755
index 2060846ad..000000000
--- a/scripts/context/stubs/mswin/texshow.bat
+++ /dev/null
@@ -1,2 +0,0 @@
-@echo off
-texmfstart texshow.pl %*
diff --git a/scripts/context/stubs/mswin/textools.bat b/scripts/context/stubs/mswin/textools.bat
index 727b4a36d..3b047ba7d 100755
--- a/scripts/context/stubs/mswin/textools.bat
+++ b/scripts/context/stubs/mswin/textools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart textools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute textools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/texutil.bat b/scripts/context/stubs/mswin/texutil.bat
index 1e63639bb..f01ced1a3 100755
--- a/scripts/context/stubs/mswin/texutil.bat
+++ b/scripts/context/stubs/mswin/texutil.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart texutil.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute texutil.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/tmftools.bat b/scripts/context/stubs/mswin/tmftools.bat
index c9c0c08bd..689a93c65 100755
--- a/scripts/context/stubs/mswin/tmftools.bat
+++ b/scripts/context/stubs/mswin/tmftools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart tmftools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute tmftools.rb %*
+endlocal
diff --git a/scripts/context/stubs/mswin/xmltools.bat b/scripts/context/stubs/mswin/xmltools.bat
index 2de0e4457..572cc9b8b 100755
--- a/scripts/context/stubs/mswin/xmltools.bat
+++ b/scripts/context/stubs/mswin/xmltools.bat
@@ -1,2 +1,5 @@
 @echo off
-texmfstart xmltools.rb %*
+setlocal
+set ownpath=%~dp0%
+texlua "%ownpath%mtxrun.lua" --usekpse --execute xmltools.rb %*
+endlocal
diff --git a/scripts/context/stubs/unix/ctxtools b/scripts/context/stubs/unix/ctxtools
index 84e47bbee..4658a345a 100755
--- a/scripts/context/stubs/unix/ctxtools
+++ b/scripts/context/stubs/unix/ctxtools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart ctxtools.rb "$@"
+mtxrun --usekpse --execute ctxtools.rb "$@"
diff --git a/scripts/context/stubs/unix/exatools b/scripts/context/stubs/unix/exatools
deleted file mode 100755
index 50ff0f07e..000000000
--- a/scripts/context/stubs/unix/exatools
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-texmfstart exatools.rb "$@"
diff --git a/scripts/context/stubs/unix/luatools b/scripts/context/stubs/unix/luatools
new file mode 100755
index 000000000..aacdbd16d
--- /dev/null
+++ b/scripts/context/stubs/unix/luatools
@@ -0,0 +1,6977 @@
+#!/usr/bin/env texlua
+
+if not modules then modules = { } end modules ['luatools'] = {
+    version   = 1.001,
+    comment   = "companion to context.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+-- one can make a stub:
+--
+-- #!/bin/sh
+-- env LUATEXDIR=/....../texmf/scripts/context/lua texlua luatools.lua "$@"
+
+-- Although this script is part of the ConTeXt distribution it is
+-- relatively indepent of ConTeXt.  The same is true for some of
+-- the luat files. We may may make them even less dependent in
+-- the future. As long as Luatex is under development the
+-- interfaces and names of functions may change.
+
+-- For the sake of independence we optionally can merge the library
+-- code here. It's too much code, but that does not harm. Much of the
+-- library code is used elsewhere. We don't want dependencies on
+-- Lua library paths simply because these scripts are located in the
+-- texmf tree and not in some Lua path. Normally this merge is not
+-- needed when texmfstart is used, or when the proper stub is used or
+-- when (windows) suffix binding is active.
+
+texlua = true
+
+-- begin library merge
+
+
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-string'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep
+
+if not string.split then
+
+    -- this will be overloaded by a faster lpeg variant
+
+    function string:split(pattern)
+        if #self > 0 then
+            local t = { }
+            for s in gmatch(self..pattern,"(.-)"..pattern) do
+                t[#t+1] = s
+            end
+            return t
+        else
+            return { }
+        end
+    end
+
+end
+
+local chr_to_esc = {
+    ["%"] = "%%",
+    ["."] = "%.",
+    ["+"] = "%+", ["-"] = "%-", ["*"] = "%*",
+    ["^"] = "%^", ["$"] = "%$",
+    ["["] = "%[", ["]"] = "%]",
+    ["("] = "%(", [")"] = "%)",
+    ["{"] = "%{", ["}"] = "%}"
+}
+
+string.chr_to_esc = chr_to_esc
+
+function string:esc() -- variant 2
+    return (gsub(self,"(.)",chr_to_esc))
+end
+
+function string:unquote()
+    return (gsub(self,"^([\"\'])(.*)%1$","%2"))
+end
+
+function string:quote() -- we could use format("%q")
+    return '"' .. self:unquote() .. '"'
+end
+
+function string:count(pattern) -- variant 3
+    local n = 0
+    for _ in gmatch(self,pattern) do
+        n = n + 1
+    end
+    return n
+end
+
+function string:limit(n,sentinel)
+    if #self > n then
+        sentinel = sentinel or " ..."
+        return sub(self,1,(n-#sentinel)) .. sentinel
+    else
+        return self
+    end
+end
+
+function string:strip()
+    return (gsub(self,"^%s*(.-)%s*$", "%1"))
+end
+
+function string:is_empty()
+    return not find(find,"%S")
+end
+
+function string:enhance(pattern,action)
+    local ok, n = true, 0
+    while ok do
+        ok = false
+        self = gsub(self,pattern, function(...)
+            ok, n = true, n + 1
+            return action(...)
+        end)
+    end
+    return self, n
+end
+
+local chr_to_hex, hex_to_chr = { }, { }
+
+for i=0,255 do
+    local c, h = char(i), format("%02X",i)
+    chr_to_hex[c], hex_to_chr[h] = h, c
+end
+
+function string:to_hex()
+    return (gsub(self or "","(.)",chr_to_hex))
+end
+
+function string:from_hex()
+    return (gsub(self or "","(..)",hex_to_chr))
+end
+
+if not string.characters then
+
+    local function nextchar(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, str:sub(index,index)
+    end
+    function string:characters()
+        return nextchar, self, 0
+    end
+    local function nextbyte(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, byte(str:sub(index,index))
+    end
+    function string:bytes()
+        return nextbyte, self, 0
+    end
+
+end
+
+-- we can use format for this (neg n)
+
+function string:rpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self .. self.rep(chr or " ",m)
+    else
+        return self
+    end
+end
+
+function string:lpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self.rep(chr or " ",m) .. self
+    else
+        return self
+    end
+end
+
+string.padd = string.rpadd
+
+function is_number(str) -- tonumber
+    return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1
+end
+
+--~ print(is_number("1"))
+--~ print(is_number("1.1"))
+--~ print(is_number(".1"))
+--~ print(is_number("-0.1"))
+--~ print(is_number("+0.1"))
+--~ print(is_number("-.1"))
+--~ print(is_number("+.1"))
+
+function string:split_settings() -- no {} handling, see l-aux for lpeg variant
+    if find(self,"=") then
+        local t = { }
+        for k,v in gmatch(self,"(%a+)=([^%,]*)") do
+            t[k] = v
+        end
+        return t
+    else
+        return nil
+    end
+end
+
+local patterns_escapes = {
+    ["-"] = "%-",
+    ["."] = "%.",
+    ["+"] = "%+",
+    ["*"] = "%*",
+    ["%"] = "%%",
+    ["("] = "%)",
+    [")"] = "%)",
+    ["["] = "%[",
+    ["]"] = "%]",
+}
+
+function string:pattesc()
+    return (gsub(self,".",patterns_escapes))
+end
+
+function string:tohash()
+    local t = { }
+    for s in gmatch(self,"([^, ]+)") do -- lpeg
+        t[s] = true
+    end
+    return t
+end
+
+local pattern = lpeg.Ct(lpeg.C(1)^0)
+
+function string:totable()
+    return pattern:match(self)
+end
+
+--~ for _, str in ipairs {
+--~     "1234567123456712345671234567",
+--~     "a\tb\tc",
+--~     "aa\tbb\tcc",
+--~     "aaa\tbbb\tccc",
+--~     "aaaa\tbbbb\tcccc",
+--~     "aaaaa\tbbbbb\tccccc",
+--~     "aaaaaa\tbbbbbb\tcccccc",
+--~ } do print(string.tabtospace(str)) end
+
+function string.tabtospace(str,tab)
+    -- we don't handle embedded newlines
+    while true do
+        local s = find(str,"\t")
+        if s then
+            if not tab then tab = 7 end -- only when found
+            local d = tab-(s-1)%tab
+            if d > 0 then
+                str = gsub(str,"\t",rep(" ",d),1)
+            else
+                str = gsub(str,"\t","",1)
+            end
+        else
+            break
+        end
+    end
+    return str
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-lpeg'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
+
+--~ l-lpeg.lua :
+
+--~ lpeg.digit         = lpeg.R('09')^1
+--~ lpeg.sign          = lpeg.S('+-')^1
+--~ lpeg.cardinal      = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.integer       = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.float         = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1)
+--~ lpeg.number        = lpeg.float + lpeg.integer
+--~ lpeg.oct           = lpeg.P("0") * lpeg.R('07')^1
+--~ lpeg.hex           = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1
+--~ lpeg.uppercase     = lpeg.P("AZ")
+--~ lpeg.lowercase     = lpeg.P("az")
+
+--~ lpeg.eol           = lpeg.S('\r\n\f')^1 -- includes formfeed
+--~ lpeg.space         = lpeg.S(' ')^1
+--~ lpeg.nonspace      = lpeg.P(1-lpeg.space)^1
+--~ lpeg.whitespace    = lpeg.S(' \r\n\f\t')^1
+--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1
+
+local hash = { }
+
+function lpeg.anywhere(pattern) --slightly adapted from website
+    return P { P(pattern) + 1 * lpeg.V(1) }
+end
+
+function lpeg.startswith(pattern) --slightly adapted
+    return P(pattern)
+end
+
+function lpeg.splitter(pattern, action)
+    return (((1-P(pattern))^1)/action+1)^0
+end
+
+-- variant:
+
+--~ local parser = lpeg.Ct(lpeg.splitat(newline))
+
+local crlf     = P("\r\n")
+local cr       = P("\r")
+local lf       = P("\n")
+local space    = S(" \t\f\v")  -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local newline  = crlf + cr + lf
+local spacing  = space^0 * newline
+
+local empty    = spacing * Cc("")
+local nonempty = Cs((1-spacing)^1) * spacing^-1
+local content  = (empty + nonempty)^1
+
+local capture = Ct(content^0)
+
+function string:splitlines()
+    return capture:match(self)
+end
+
+lpeg.linebyline = content -- better make a sublibrary
+
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps->what->more"))  -- oeps what more
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps->what->more"))  -- oeps what->more
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps"))              -- oeps
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps"))              -- oeps
+
+local splitters_s, splitters_m = { }, { }
+
+local function splitat(separator,single)
+    local splitter = (single and splitters_s[separator]) or splitters_m[separator]
+    if not splitter then
+        separator = P(separator)
+        if single then
+            local other, any = C((1 - separator)^0), P(1)
+            splitter = other * (separator * C(any^0) + "")
+            splitters_s[separator] = splitter
+        else
+            local other = C((1 - separator)^0)
+            splitter = other * (separator * other)^0
+            splitters_m[separator] = splitter
+        end
+    end
+    return splitter
+end
+
+lpeg.splitat = splitat
+
+local cache = { }
+
+function string:split(separator)
+    local c = cache[separator]
+    if not c then
+        c = Ct(splitat(separator))
+        cache[separator] = c
+    end
+    return c:match(self)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-table'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+table.join = table.concat
+
+local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove
+local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump
+local getmetatable, setmetatable = getmetatable, setmetatable
+local type, next, tostring, ipairs = type, next, tostring, ipairs
+
+function table.strip(tab)
+    local lst = { }
+    for i=1,#tab do
+        local s = gsub(tab[i],"^%s*(.-)%s*$","%1")
+        if s == "" then
+            -- skip this one
+        else
+            lst[#lst+1] = s
+        end
+    end
+    return lst
+end
+
+local function sortedkeys(tab)
+    local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+        if kind == 3 then
+            -- no further check
+        else
+            local tkey = type(key)
+            if tkey == "string" then
+            --  if kind == 2 then kind = 3 else kind = 1 end
+                kind = (kind == 2 and 3) or 1
+            elseif tkey == "number" then
+            --  if kind == 1 then kind = 3 else kind = 2 end
+                kind = (kind == 1 and 3) or 2
+            else
+                kind = 3
+            end
+        end
+    end
+    if kind == 0 or kind == 3 then
+        sort(srt,function(a,b) return (tostring(a) < tostring(b)) end)
+    else
+        sort(srt)
+    end
+    return srt
+end
+
+local function sortedhashkeys(tab) -- fast one
+    local srt = { }
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+    end
+    sort(srt)
+    return srt
+end
+
+table.sortedkeys     = sortedkeys
+table.sortedhashkeys = sortedhashkeys
+
+function table.sortedpairs(t)
+    local s = sortedhashkeys(t) -- maybe just sortedkeys
+    local n = 0
+    local function kv(s)
+        n = n + 1
+        local k = s[n]
+        return k, t[k]
+    end
+    return kv, s
+end
+
+function table.append(t, list)
+    for _,v in next, list do
+        insert(t,v)
+    end
+end
+
+function table.prepend(t, list)
+    for k,v in next, list do
+        insert(t,k,v)
+    end
+end
+
+function table.merge(t, ...) -- first one is target
+    t = t or {}
+    local lst = {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            t[k] = v
+        end
+    end
+    return t
+end
+
+function table.merged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            tmp[k] = v
+        end
+    end
+    return tmp
+end
+
+function table.imerge(t, ...)
+    local lst = {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            t[#t+1] = nst[j]
+        end
+    end
+    return t
+end
+
+function table.imerged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            tmp[#tmp+1] = nst[j]
+        end
+    end
+    return tmp
+end
+
+local function fastcopy(old) -- fast one
+    if old then
+        local new = { }
+        for k,v in next, old do
+            if type(v) == "table" then
+                new[k] = fastcopy(v) -- was just table.copy
+            else
+                new[k] = v
+            end
+        end
+        -- optional second arg
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
+        end
+        return new
+    else
+        return { }
+    end
+end
+
+local function copy(t, tables) -- taken from lua wiki, slightly adapted
+    tables = tables or { }
+    local tcopy = {}
+    if not tables[t] then
+        tables[t] = tcopy
+    end
+    for i,v in next, t do -- brrr, what happens with sparse indexed
+        if type(i) == "table" then
+            if tables[i] then
+                i = tables[i]
+            else
+                i = copy(i, tables)
+            end
+        end
+        if type(v) ~= "table" then
+            tcopy[i] = v
+        elseif tables[v] then
+            tcopy[i] = tables[v]
+        else
+            tcopy[i] = copy(v, tables)
+        end
+    end
+    local mt = getmetatable(t)
+    if mt then
+        setmetatable(tcopy,mt)
+    end
+    return tcopy
+end
+
+table.fastcopy = fastcopy
+table.copy     = copy
+
+-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack)
+
+function table.sub(t,i,j)
+    return { unpack(t,i,j) }
+end
+
+function table.replace(a,b)
+    for k,v in next, b do
+        a[k] = v
+    end
+end
+
+-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice)
+
+function table.is_empty(t)
+    return not t or not next(t)
+end
+
+function table.one_entry(t)
+    local n = next(t)
+    return n and not next(t,n)
+end
+
+function table.starts_at(t)
+    return ipairs(t,1)(t,0)
+end
+
+function table.tohash(t,value)
+    local h = { }
+    if t then
+        if value == nil then value = true end
+        for _, v in next, t do -- no ipairs here
+            h[v] = value
+        end
+    end
+    return h
+end
+
+function table.fromhash(t)
+    local h = { }
+    for k, v in next, t do -- no ipairs here
+        if v then h[#h+1] = k end
+    end
+    return h
+end
+
+--~ print(table.serialize(t), "\n")
+--~ print(table.serialize(t,"name"), "\n")
+--~ print(table.serialize(t,false), "\n")
+--~ print(table.serialize(t,true), "\n")
+--~ print(table.serialize(t,"name",true), "\n")
+--~ print(table.serialize(t,"name",true,true), "\n")
+
+table.serialize_functions = true
+table.serialize_compact   = true
+table.serialize_inline    = true
+
+local noquotes, hexify, handle, reduce, compact, inline, functions
+
+local reserved = table.tohash { -- intercept a language flaw, no reserved words as key
+    'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
+    'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
+}
+
+local function simple_table(t)
+    if #t > 0 then
+        local n = 0
+        for _,v in next, t do
+            n = n + 1
+        end
+        if n == #t then
+            local tt = { }
+            for i=1,#t do
+                local v = t[i]
+                local tv = type(v)
+                if tv == "number" then
+                    if hexify then
+                        tt[#tt+1] = format("0x%04X",v)
+                    else
+                        tt[#tt+1] = tostring(v) -- tostring not needed
+                    end
+                elseif tv == "boolean" then
+                    tt[#tt+1] = tostring(v)
+                elseif tv == "string" then
+                    tt[#tt+1] = format("%q",v)
+                else
+                    tt = nil
+                    break
+                end
+            end
+            return tt
+        end
+    end
+    return nil
+end
+
+-- Because this is a core function of mkiv I moved some function calls
+-- inline.
+--
+-- twice as fast in a test:
+--
+-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) )
+
+local function do_serialize(root,name,depth,level,indexed)
+    if level > 0 then
+        depth = depth .. " "
+        if indexed then
+            handle(format("%s{",depth))
+        elseif name then
+        --~ handle(format("%s%s={",depth,key(name)))
+            if type(name) == "number" then -- or find(k,"^%d+$") then
+                if hexify then
+                    handle(format("%s[0x%04X]={",depth,name))
+                else
+                    handle(format("%s[%s]={",depth,name))
+                end
+            elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then
+                handle(format("%s%s={",depth,name))
+            else
+                handle(format("%s[%q]={",depth,name))
+            end
+        else
+            handle(format("%s{",depth))
+        end
+    end
+    if root and next(root) then
+        local first, last = nil, 0 -- #root cannot be trusted here
+        if compact then
+            -- NOT: for k=1,#root do (we need to quit at nil)
+            for k,v in ipairs(root) do -- can we use next?
+                if not first then first = k end
+                last = last + 1
+            end
+        end
+        local sk = sortedkeys(root)
+        for i=1,#sk do
+            local k = sk[i]
+            local v = root[k]
+            --~ if v == root then
+                -- circular
+            --~ else
+            local t = type(v)
+            if compact and first and type(k) == "number" and k >= first and k <= last then
+                if t == "number" then
+                    if hexify then
+                        handle(format("%s 0x%04X,",depth,v))
+                    else
+                        handle(format("%s %s,",depth,v))
+                    end
+                elseif t == "string" then
+                    if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                        handle(format("%s %s,",depth,v))
+                    else
+                        handle(format("%s %q,",depth,v))
+                    end
+                elseif t == "table" then
+                    if not next(v) then
+                        handle(format("%s {},",depth))
+                    elseif inline then -- and #t > 0
+                        local st = simple_table(v)
+                        if st then
+                            handle(format("%s { %s },",depth,concat(st,", ")))
+                        else
+                            do_serialize(v,k,depth,level+1,true)
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1,true)
+                    end
+                elseif t == "boolean" then
+                    handle(format("%s %s,",depth,tostring(v)))
+                elseif t == "function" then
+                    if functions then
+                        handle(format('%s loadstring(%q),',depth,dump(v)))
+                    else
+                        handle(format('%s "function",',depth))
+                    end
+                else
+                    handle(format("%s %q,",depth,tostring(v)))
+                end
+            elseif k == "__p__" then -- parent
+                if false then
+                    handle(format("%s __p__=nil,",depth))
+                end
+            elseif t == "number" then
+            --~ if hexify then
+            --~     handle(format("%s %s=0x%04X,",depth,key(k),v))
+            --~ else
+            --~     handle(format("%s %s=%s,",depth,key(k),v))
+            --~ end
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,v))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    if hexify then
+                        handle(format("%s %s=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s %s=%s,",depth,k,v))
+                    end
+                else
+                    if hexify then
+                        handle(format("%s [%q]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                end
+            elseif t == "string" then
+                if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                --~ handle(format("%s %s=%s,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%s,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%s,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%s,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                else
+                --~ handle(format("%s %s=%q,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%q,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%q,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%q,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%q,",depth,k,v))
+                    end
+                end
+            elseif t == "table" then
+                if not next(v) then
+                    --~ handle(format("%s %s={},",depth,key(k)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]={},",depth,k))
+                        else
+                            handle(format("%s [%s]={},",depth,k))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s={},",depth,k))
+                    else
+                        handle(format("%s [%q]={},",depth,k))
+                    end
+                elseif inline then
+                    local st = simple_table(v)
+                    if st then
+                    --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", ")))
+                        if type(k) == "number" then -- or find(k,"^%d+$") then
+                            if hexify then
+                                handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", ")))
+                            else
+                                handle(format("%s [%s]={ %s },",depth,k,concat(st,", ")))
+                            end
+                        elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                            handle(format("%s %s={ %s },",depth,k,concat(st,", ")))
+                        else
+                            handle(format("%s [%q]={ %s },",depth,k,concat(st,", ")))
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1)
+                    end
+                else
+                    do_serialize(v,k,depth,level+1)
+                end
+            elseif t == "boolean" then
+            --~ handle(format("%s %s=%s,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%s,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%s,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%s,",depth,k,tostring(v)))
+                end
+            elseif t == "function" then
+                if functions then
+                    --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v)))
+                        else
+                            handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v)))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=loadstring(%q),",depth,k,dump(v)))
+                    else
+                        handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v)))
+                    end
+                end
+            else
+                --~ handle(format("%s %s=%q,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%q,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%q,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%q,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%q,",depth,k,tostring(v)))
+                end
+            end
+            --~ end
+        end
+    end
+   if level > 0 then
+        handle(format("%s},",depth))
+    end
+end
+
+-- replacing handle by a direct t[#t+1] = ... (plus test) is not much
+-- faster (0.03 on 1.00 for zapfino.tma)
+
+local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
+    noquotes = _noquotes
+    hexify = _hexify
+    handle = _handle or print
+    reduce = _reduce or false
+    compact = table.serialize_compact
+    inline  = compact and table.serialize_inline
+    functions = table.serialize_functions
+    local tname = type(name)
+    if tname == "string" then
+        if name == "return" then
+            handle("return {")
+        else
+            handle(name .. "={")
+        end
+    elseif tname == "number" then
+        if hexify then
+            handle(format("[0x%04X]={",name))
+        else
+            handle("[" .. name .. "]={")
+        end
+    elseif tname == "boolean" then
+        if name then
+            handle("return {")
+        else
+            handle("{")
+        end
+    else
+        handle("t={")
+    end
+    if root and next(root) then
+        do_serialize(root,name,"",0,indexed)
+    end
+    handle("}")
+end
+
+--~ name:
+--~
+--~ true     : return     { }
+--~ false    :            { }
+--~ nil      : t        = { }
+--~ string   : string   = { }
+--~ 'return' : return     { }
+--~ number   : [number] = { }
+
+function table.serialize(root,name,reduce,noquotes,hexify)
+    local t = { }
+    local function flush(s)
+        t[#t+1] = s
+    end
+    serialize(root,name,flush,reduce,noquotes,hexify)
+    return concat(t,"\n")
+end
+
+function table.tohandle(handle,root,name,reduce,noquotes,hexify)
+    serialize(root,name,handle,reduce,noquotes,hexify)
+end
+
+-- sometimes tables are real use (zapfino extra pro is some 85M) in which
+-- case a stepwise serialization is nice; actually, we could consider:
+--
+-- for line in table.serializer(root,name,reduce,noquotes) do
+--    ...(line)
+-- end
+--
+-- so this is on the todo list
+
+table.tofile_maxtab = 2*1024
+
+function table.tofile(filename,root,name,reduce,noquotes,hexify)
+    local f = io.open(filename,'w')
+    if f then
+        local maxtab = table.tofile_maxtab
+        if maxtab > 1 then
+            local t = { }
+            local function flush(s)
+                t[#t+1] = s
+                if #t > maxtab then
+                    f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice
+                    t = { }
+                end
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+            f:write(concat(t,"\n"),"\n")
+        else
+            local function flush(s)
+                f:write(s,"\n")
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+        end
+        f:close()
+    end
+end
+
+local function flatten(t,f,complete)
+    for i=1,#t do
+        local v = t[i]
+        if type(v) == "table" then
+            if complete or type(v[1]) == "table" then
+                flatten(v,f,complete)
+            else
+                f[#f+1] = v
+            end
+        else
+            f[#f+1] = v
+        end
+    end
+end
+
+function table.flatten(t)
+    local f = { }
+    flatten(t,f,true)
+    return f
+end
+
+function table.unnest(t) -- bad name
+    local f = { }
+    flatten(t,f,false)
+    return f
+end
+
+table.flatten_one_level = table.unnest
+
+-- the next three may disappear
+
+function table.remove_value(t,value) -- todo: n
+    if value then
+        for i=1,#t do
+            if t[i] == value then
+                remove(t,i)
+                -- remove all, so no: return
+            end
+        end
+    end
+end
+
+function table.insert_before_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i,str)
+                    return
+                end
+            end
+        end
+        insert(t,1,str)
+    elseif value then
+        insert(t,1,value)
+    end
+end
+
+function table.insert_after_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i+1,str)
+                    return
+                end
+            end
+        end
+        t[#t+1] = str
+    elseif value then
+        t[#t+1] = value
+    end
+end
+
+local function are_equal(a,b,n,m) -- indexed
+    if #a == #b then
+        n = n or 1
+        m = m or #a
+        for i=n,m do
+            local ai, bi = a[i], b[i]
+            if ai==bi then
+                -- same
+            elseif type(ai)=="table" and type(bi)=="table" then
+                if not are_equal(ai,bi) then
+                    return false
+                end
+            else
+                return false
+            end
+        end
+        return true
+    else
+        return false
+    end
+end
+
+local function identical(a,b) -- assumes same structure
+    for ka, va in next, a do
+        local vb = b[k]
+        if va == vb then
+            -- same
+        elseif type(va) == "table" and  type(vb) == "table" then
+            if not identical(va,vb) then
+                return false
+            end
+        else
+            return false
+        end
+    end
+    return true
+end
+
+table.are_equal = are_equal
+table.identical = identical
+
+-- maybe also make a combined one
+
+function table.compact(t)
+    if t then
+        for k,v in next, t do
+            if not next(v) then
+                t[k] = nil
+            end
+        end
+    end
+end
+
+function table.contains(t, v)
+    if t then
+        for i=1, #t do
+            if t[i] == v then
+                return i
+            end
+        end
+    end
+    return false
+end
+
+function table.count(t)
+    local n, e = 0, next(t)
+    while e do
+        n, e = n + 1, next(t,e)
+    end
+    return n
+end
+
+function table.swapped(t)
+    local s = { }
+    for k, v in next, t do
+        s[v] = k
+    end
+    return s
+end
+
+--~ function table.are_equal(a,b)
+--~     return table.serialize(a) == table.serialize(b)
+--~ end
+
+function table.clone(t,p) -- t is optional or nil or table
+    if not p then
+        t, p = { }, t or { }
+    elseif not t then
+        t = { }
+    end
+    setmetatable(t, { __index = function(_,key) return p[key] end })
+    return t
+end
+
+function table.hexed(t,seperator)
+    local tt = { }
+    for i=1,#t do tt[i] = format("0x%04X",t[i]) end
+    return concat(tt,seperator or " ")
+end
+
+function table.reverse_hash(h)
+    local r = { }
+    for k,v in next, h do
+        r[v] = lower(gsub(k," ",""))
+    end
+    return r
+end
+
+function table.reverse(t)
+    local tt = { }
+    if #t > 0 then
+        for i=#t,1,-1 do
+            tt[#tt+1] = t[i]
+        end
+    end
+    return tt
+end
+
+--~ function table.keys(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return k
+--~ end
+
+--~ function table.keys_as_string(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return concat(k,"")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-io'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local byte = string.byte
+
+if string.find(os.getenv("PATH"),";") then
+    io.fileseparator, io.pathseparator = "\\", ";"
+else
+    io.fileseparator, io.pathseparator = "/" , ":"
+end
+
+function io.loaddata(filename,textmode)
+    local f = io.open(filename,(textmode and 'r') or 'rb')
+    if f then
+        local data = f:read('*all')
+    --  garbagecollector.check(data)
+        f:close()
+        return data
+    else
+        return nil
+    end
+end
+
+function io.savedata(filename,data,joiner)
+    local f = io.open(filename,"wb")
+    if f then
+        if type(data) == "table" then
+            f:write(table.join(data,joiner or ""))
+        elseif type(data) == "function" then
+            data(f)
+        else
+            f:write(data)
+        end
+        f:close()
+        return true
+    else
+        return false
+    end
+end
+
+function io.exists(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return false
+    else
+        assert(f:close())
+        return true
+    end
+end
+
+function io.size(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return 0
+    else
+        local s = f:seek("end")
+        assert(f:close())
+        return s
+    end
+end
+
+function io.noflines(f)
+    local n = 0
+    for _ in f:lines() do
+        n = n + 1
+    end
+    f:seek('set',0)
+    return n
+end
+
+local nextchar = {
+    [ 4] = function(f)
+        return f:read(1,1,1,1)
+    end,
+    [ 2] = function(f)
+        return f:read(1,1)
+    end,
+    [ 1] = function(f)
+        return f:read(1)
+    end,
+    [-2] = function(f)
+        local a, b = f:read(1,1)
+        return b, a
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        return d, c, b, a
+    end
+}
+
+function io.characters(f,n)
+    if f then
+        return nextchar[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+local nextbyte = {
+    [4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(a), byte(b), byte(c), byte(d)
+        else
+            return nil, nil, nil, nil
+        end
+    end,
+    [2] = function(f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(a), byte(b)
+        else
+            return nil, nil
+        end
+    end,
+    [1] = function (f)
+        local a = f:read(1)
+        if a then
+            return byte(a)
+        else
+            return nil
+        end
+    end,
+    [-2] = function (f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(b), byte(a)
+        else
+            return nil, nil
+        end
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(d), byte(c), byte(b), byte(a)
+        else
+            return nil, nil, nil, nil
+        end
+    end
+}
+
+function io.bytes(f,n)
+    if f then
+        return nextbyte[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+function io.ask(question,default,options)
+    while true do
+        io.write(question)
+        if options then
+            io.write(string.format(" [%s]",table.concat(options,"|")))
+        end
+        if default then
+            io.write(string.format(" [%s]",default))
+        end
+        io.write(string.format(" "))
+        local answer = io.read()
+        answer = answer:gsub("^%s*(.*)%s*$","%1")
+        if answer == "" and default then
+            return default
+        elseif not options then
+            return answer
+        else
+            for _,v in pairs(options) do
+                if v == answer then
+                    return answer
+                end
+            end
+            local pattern = "^" .. answer
+            for _,v in pairs(options) do
+                if v:find(pattern) then
+                    return v
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-number'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+number = number or { }
+
+-- a,b,c,d,e,f = number.toset(100101)
+
+function number.toset(n)
+    return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)")
+end
+
+function number.toevenhex(n)
+    local s = format("%X",n)
+    if #s % 2 == 0 then
+        return s
+    else
+        return "0" .. s
+    end
+end
+
+-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5%
+-- on
+--
+-- for i=1,1000000 do
+--     local a,b,c,d,e,f,g,h = number.toset(12345678)
+--     local a,b,c,d         = number.toset(1234)
+--     local a,b,c           = number.toset(123)
+-- end
+--
+-- of course dedicated "(.)(.)(.)(.)" matches are even faster
+
+local one = lpeg.C(1-lpeg.S(''))^1
+
+function number.toset(n)
+    return one:match(tostring(n))
+end
+
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-set'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+set = set or { }
+
+local nums   = { }
+local tabs   = { }
+local concat = table.concat
+
+set.create = table.tohash
+
+function set.tonumber(t)
+    if next(t) then
+        local s = ""
+    --  we could save mem by sorting, but it slows down
+        for k, v in pairs(t) do
+            if v then
+            --  why bother about the leading space
+                s = s .. " " .. k
+            end
+        end
+        if not nums[s] then
+            tabs[#tabs+1] = t
+            nums[s] = #tabs
+        end
+        return nums[s]
+    else
+        return 0
+    end
+end
+
+function set.totable(n)
+    if n == 0 then
+        return { }
+    else
+        return tabs[n] or { }
+    end
+end
+
+function set.contains(n,s)
+    if type(n) == "table" then
+        return n[s]
+    elseif n == 0 then
+        return false
+    else
+        local t = tabs[n]
+        return t and t[s]
+    end
+end
+
+--~ local c = set.create{'aap','noot','mies'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ local c = set.create{'zus','wim','jet'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ print(t['jet'])
+--~ print(set.contains(t,'jet'))
+--~ print(set.contains(t,'aap'))
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-os'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lub.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+function os.resultof(command)
+    return io.popen(command,"r"):read("*all")
+end
+
+if not os.exec  then os.exec  = os.execute end
+if not os.spawn then os.spawn = os.execute end
+
+--~ os.type : windows | unix (new, we already guessed os.platform)
+--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new)
+
+if not io.fileseparator then
+    if find(os.getenv("PATH"),";") then
+        io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows"
+    else
+        io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix"
+    end
+end
+
+os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix"
+
+function os.launch(str)
+    if os.platform == "windows" then
+        os.execute("start " .. str) -- os.spawn ?
+    else
+        os.execute(str .. " &")     -- os.spawn ?
+    end
+end
+
+if not os.setenv then
+    function os.setenv() return false end
+end
+
+if not os.times then
+    -- utime  = user time
+    -- stime  = system time
+    -- cutime = children user time
+    -- cstime = children system time
+    function os.times()
+        return {
+            utime  = os.gettimeofday(), -- user
+            stime  = 0,                 -- system
+            cutime = 0,                 -- children user
+            cstime = 0,                 -- children system
+        }
+    end
+end
+
+os.gettimeofday = os.gettimeofday or os.clock
+
+local startuptime = os.gettimeofday()
+
+function os.runtime()
+    return os.gettimeofday() - startuptime
+end
+
+--~ print(os.gettimeofday()-os.time())
+--~ os.sleep(1.234)
+--~ print (">>",os.runtime())
+--~ print(os.date("%H:%M:%S",os.gettimeofday()))
+--~ print(os.date("%H:%M:%S",os.time()))
+
+os.arch = os.arch or function()
+    local a = os.resultof("uname -m") or "linux"
+    os.arch = function()
+        return a
+    end
+    return a
+end
+
+local platform
+
+function os.currentplatform(name,default)
+    if not platform then
+        local name = os.name or os.platform or name -- os.name is built in, os.platform is mine
+        if not name then
+            platform = default or "linux"
+        elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then
+            if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then
+                platform = "mswin-64"
+            else
+                platform = "mswin"
+            end
+        else
+            local architecture = os.arch()
+            if name == "linux" then
+                if find(architecture,"x86_64") then
+                    platform = "linux-64"
+                elseif find(architecture,"ppc") then
+                    platform = "linux-ppc"
+                else
+                    platform = "linux"
+                end
+            elseif name == "macosx" then
+                if find(architecture,"i386") then
+                    platform = "osx-intel"
+                else
+                    platform = "osx-ppc"
+                end
+            elseif name == "sunos" then
+                if find(architecture,"sparc") then
+                    platform = "solaris-sparc"
+                else -- if architecture == 'i86pc'
+                    platform = "solaris-intel"
+                end
+            elseif name == "freebsd" then
+                if find(architecture,"amd64") then
+                    platform = "freebsd-amd64"
+                else
+                    platform = "freebsd"
+                end
+            else
+                platform = default or name
+            end
+        end
+        function os.currentplatform()
+            return platform
+        end
+    end
+    return platform
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-file'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- needs a cleanup
+
+file = file or { }
+
+local concat = table.concat
+local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub
+
+function file.removesuffix(filename)
+    return (gsub(filename,"%.[%a%d]+$",""))
+end
+
+function file.addsuffix(filename, suffix)
+    if not find(filename,"%.[%a%d]+$") then
+        return filename .. "." .. suffix
+    else
+        return filename
+    end
+end
+
+function file.replacesuffix(filename, suffix)
+    return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix
+end
+
+function file.dirname(name)
+    return match(name,"^(.+)[/\\].-$") or ""
+end
+
+function file.basename(name)
+    return match(name,"^.+[/\\](.-)$") or name
+end
+
+function file.nameonly(name)
+    return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$",""))
+end
+
+function file.extname(name)
+    return match(name,"^.+%.([^/\\]-)$") or  ""
+end
+
+file.suffix = file.extname
+
+--~ print(file.join("x/","/y"))
+--~ print(file.join("http://","/y"))
+--~ print(file.join("http://a","/y"))
+--~ print(file.join("http:///a","/y"))
+--~ print(file.join("//nas-1","/y"))
+
+function file.join(...)
+    local pth = concat({...},"/")
+    pth = gsub(pth,"\\","/")
+    local a, b = match(pth,"^(.*://)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    a, b = match(pth,"^(//)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    return (gsub(pth,"//+","/"))
+end
+
+function file.iswritable(name)
+    local a = lfs.attributes(name)
+    if a and a.permissions:sub(2,2) == "w" then
+        return true
+    else
+        name = file.dirname(name) or "."
+        if name == "" then name = "." end
+        a = lfs.attributes(name)
+        return a and a.permissions:sub(2,2) == "w"
+    end
+end
+
+function file.isreadable(name)
+    local a = lfs.attributes(name)
+    return a and a.permissions:sub(1,1) == "r"
+end
+
+file.is_readable = file.isreadable
+file.is_writable = file.iswritable
+
+-- todo: lpeg
+
+function file.split_path(str)
+    local t = { }
+    str = gsub(str,"\\", "/")
+    str = gsub(str,"(%a):([;/])", "%1\001%2")
+    for name in gmatch(str,"([^;:]+)") do
+        if name ~= "" then
+            t[#t+1] = gsub(name,"\001",":")
+        end
+    end
+    return t
+end
+
+function file.join_path(tab)
+    return concat(tab,io.pathseparator) -- can have trailing //
+end
+
+function file.collapse_path(str)
+    str = gsub(str,"/%./","/")
+    local n, m = 1, 1
+    while n > 0 or m > 0 do
+        str, n = gsub(str,"[^/%.]+/%.%.$","")
+        str, m = gsub(str,"[^/%.]+/%.%./","")
+    end
+    str = gsub(str,"([^/])/$","%1")
+    str = gsub(str,"^%./","")
+    str = gsub(str,"/%.$","")
+    if str == "" then str = "." end
+    return str
+end
+
+--~ print(file.collapse_path("a/./b/.."))
+--~ print(file.collapse_path("a/aa/../b/bb"))
+--~ print(file.collapse_path("a/../.."))
+--~ print(file.collapse_path("a/.././././b/.."))
+--~ print(file.collapse_path("a/./././b/.."))
+--~ print(file.collapse_path("a/b/c/../.."))
+
+function file.robustname(str)
+    return (gsub(str,"[^%a%d%/%-%.\\]+","-"))
+end
+
+file.readdata = io.loaddata
+file.savedata = io.savedata
+
+function file.copy(oldname,newname)
+    file.savedata(newname,io.loaddata(oldname))
+end
+
+-- lpeg variants, slightly faster, not always
+
+--~ local period    = lpeg.P(".")
+--~ local slashes   = lpeg.S("\\/")
+--~ local noperiod  = 1-period
+--~ local noslashes = 1-slashes
+--~ local name      = noperiod^1
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1
+
+--~ function file.extname(name)
+--~     return pattern:match(name) or ""
+--~ end
+
+--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1)
+
+--~ function file.removesuffix(name)
+--~     return pattern:match(name)
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1
+
+--~ function file.basename(name)
+--~     return pattern:match(name) or name
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1
+
+--~ function file.dirname(name)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2)
+--~     else
+--~         return ""
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.addsuffix(name, suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.replacesuffix(name,suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2) .. "." .. suffix
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1
+
+--~ function file.nameonly(name)
+--~     local a, b = pattern:match(name)
+--~     if b then
+--~         return name:sub(a,b-2)
+--~     elseif a then
+--~         return name:sub(a)
+--~     else
+--~         return name
+--~     end
+--~ end
+
+--~ local test = file.extname
+--~ local test = file.basename
+--~ local test = file.dirname
+--~ local test = file.addsuffix
+--~ local test = file.replacesuffix
+--~ local test = file.nameonly
+
+--~ print(1,test("./a/b/c/abd.def.xxx","!!!"))
+--~ print(2,test("./../b/c/abd.def.xxx","!!!"))
+--~ print(3,test("a/b/c/abd.def.xxx","!!!"))
+--~ print(4,test("a/b/c/def.xxx","!!!"))
+--~ print(5,test("a/b/c/def","!!!"))
+--~ print(6,test("def","!!!"))
+--~ print(7,test("def.xxx","!!!"))
+
+--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim)
+
+-- also rewrite previous
+
+local letter    = lpeg.R("az","AZ") + lpeg.S("_-+")
+local separator = lpeg.P("://")
+
+local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/")
+local rootbased = lpeg.P("/") + letter*lpeg.P(":")
+
+-- ./name ../name  /name c: :// name/name
+
+function file.is_qualified_path(filename)
+    return qualified:match(filename)
+end
+
+function file.is_rootbased_path(filename)
+    return rootbased:match(filename)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-md5'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This also provides file checksums and checkers.
+
+local gsub, format, byte = string.gsub, string.format, string.byte
+
+local function convert(str,fmt)
+    return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end))
+end
+
+if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end
+if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end
+if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end
+
+--~ if not md5.HEX then
+--~     local function remap(chr) return format("%02X",byte(chr)) end
+--~     function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.hex then
+--~     local function remap(chr) return format("%02x",byte(chr)) end
+--~     function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.dec then
+--~     local function remap(chr) return format("%03i",byte(chr)) end
+--~     function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+
+file.needs_updating_threshold = 1
+
+function file.needs_updating(oldname,newname) -- size modification access change
+    local oldtime = lfs.attributes(oldname, modification)
+    local newtime = lfs.attributes(newname, modification)
+    if newtime >= oldtime then
+        return false
+    elseif oldtime - newtime < file.needs_updating_threshold then
+        return false
+    else
+        return true
+    end
+end
+
+function file.checksum(name)
+    if md5 then
+        local data = io.loaddata(name)
+        if data then
+            return md5.HEXsum(data)
+        end
+    end
+    return nil
+end
+
+function file.loadchecksum(name)
+    if md5 then
+        local data = io.loaddata(name .. ".md5")
+        return data and data:gsub("%s","")
+    end
+    return nil
+end
+
+function file.savechecksum(name, checksum)
+    if not checksum then checksum = file.checksum(name) end
+    if checksum then
+        io.savedata(name .. ".md5",checksum)
+        return checksum
+    end
+    return nil
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-url'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local char, gmatch = string.char, string.gmatch
+local tonumber, type = tonumber, type
+
+-- from the spec (on the web):
+--
+--     foo://example.com:8042/over/there?name=ferret#nose
+--     \_/   \______________/\_________/ \_________/ \__/
+--      |           |            |            |        |
+--   scheme     authority       path        query   fragment
+--      |   _____________________|__
+--     / \ /                        \
+--     urn:example:animal:ferret:nose
+
+url = url or { }
+
+local function tochar(s)
+    return char(tonumber(s,16))
+end
+
+local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1)
+
+local hexdigit  = lpeg.R("09","AF","af")
+local plus      = lpeg.P("+")
+local escaped   = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar)
+
+local scheme    =                 lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("")
+local authority = slash * slash * lpeg.Cs((escaped+(1-      slash-qmark-hash))^0)         + lpeg.Cc("")
+local path      = slash *         lpeg.Cs((escaped+(1-            qmark-hash))^0)         + lpeg.Cc("")
+local query     = qmark         * lpeg.Cs((escaped+(1-                  hash))^0)         + lpeg.Cc("")
+local fragment  = hash          * lpeg.Cs((escaped+(1-           endofstring))^0)         + lpeg.Cc("")
+
+local parser = lpeg.Ct(scheme * authority * path * query * fragment)
+
+function url.split(str)
+    return (type(str) == "string" and parser:match(str)) or str
+end
+
+function url.hashed(str)
+    local s = url.split(str)
+    return {
+        scheme = (s[1] ~= "" and s[1]) or "file",
+        authority = s[2],
+        path = s[3],
+        query = s[4],
+        fragment = s[5],
+        original = str
+    }
+end
+
+function url.filename(filename)
+    local t = url.hashed(filename)
+    return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename
+end
+
+function url.query(str)
+    if type(str) == "string" then
+        local t = { }
+        for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do
+            t[k] = v
+        end
+        return t
+    else
+        return str
+    end
+end
+
+--~ print(url.filename("file:///c:/oeps.txt"))
+--~ print(url.filename("c:/oeps.txt"))
+--~ print(url.filename("file:///oeps.txt"))
+--~ print(url.filename("file:///etc/test.txt"))
+--~ print(url.filename("/oeps.txt"))
+
+--~ from the spec on the web (sort of):
+--~
+--~ function test(str)
+--~     print(table.serialize(url.hashed(str)))
+--~ end
+--~
+--~ test("%56pass%20words")
+--~ test("file:///c:/oeps.txt")
+--~ test("file:///c|/oeps.txt")
+--~ test("file:///etc/oeps.txt")
+--~ test("file://./etc/oeps.txt")
+--~ test("file:////etc/oeps.txt")
+--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt")
+--~ test("http://www.ietf.org/rfc/rfc2396.txt")
+--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what")
+--~ test("mailto:John.Doe@example.com")
+--~ test("news:comp.infosystems.www.servers.unix")
+--~ test("tel:+1-816-555-1212")
+--~ test("telnet://192.0.2.16:80/")
+--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2")
+--~ test("/etc/passwords")
+--~ test("http://www.pragma-ade.com/spaced%20name")
+
+--~ test("zip:///oeps/oeps.zip#bla/bla.tex")
+--~ test("zip:///oeps/oeps.zip?bla/bla.tex")
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-dir'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type = type
+local find, gmatch = string.find, string.gmatch
+
+dir = dir or { }
+
+-- optimizing for no string.find (*) does not save time
+
+local attributes = lfs.attributes
+local walkdir    = lfs.dir
+
+local function glob_pattern(path,patt,recurse,action)
+    local ok, scanner
+    if path == "/" then
+        ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
+    else
+        ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
+    end
+    if ok and type(scanner) == "function" then
+        if not find(path,"/$") then path = path .. '/' end
+        for name in scanner do
+            local full = path .. name
+            local mode = attributes(full,'mode')
+            if mode == 'file' then
+                if find(full,patt) then
+                    action(full)
+                end
+            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
+                glob_pattern(full,patt,recurse,action)
+            end
+        end
+    end
+end
+
+dir.glob_pattern = glob_pattern
+
+local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V
+
+local pattern = Ct {
+    [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3),
+    [2] = C(((1-S("*?/"))^0 * P("/"))^0),
+    [3] = C(P(1)^0)
+}
+
+local filter = Cs ( (
+    P("**") / ".*" +
+    P("*")  / "[^/]*" +
+    P("?")  / "[^/]" +
+    P(".")  / "%%." +
+    P("+")  / "%%+" +
+    P("-")  / "%%-" +
+    P(1)
+)^0 )
+
+local function glob(str,t)
+    if type(str) == "table" then
+        local t = t or { }
+        for s=1,#str do
+            glob(str[s],t)
+        end
+        return t
+    elseif lfs.isfile(str) then
+        local t = t or { }
+        t[#t+1] = str
+        return t
+    else
+        local split = pattern:match(str)
+        if split then
+            local t = t or { }
+            local action = action or function(name) t[#t+1] = name end
+            local root, path, base = split[1], split[2], split[3]
+            local recurse = find(base,"%*%*")
+            local start = root .. path
+            local result = filter:match(start .. base)
+            glob_pattern(start,result,recurse,action)
+            return t
+        else
+            return { }
+        end
+    end
+end
+
+dir.glob = glob
+
+--~ list = dir.glob("**/*.tif")
+--~ list = dir.glob("/**/*.tif")
+--~ list = dir.glob("./**/*.tif")
+--~ list = dir.glob("oeps/**/*.tif")
+--~ list = dir.glob("/oeps/**/*.tif")
+
+local function globfiles(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func -- alas, we need this indirect way
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if recurse then
+                    globfiles(path .. "/" .. name,recurse,func,files)
+                end
+            elseif mode == "file" then
+                if func then
+                    if func(name) then
+                        files[#files+1] = path .. "/" .. name
+                    end
+                else
+                    files[#files+1] = path .. "/" .. name
+                end
+            end
+        end
+    end
+    return files
+end
+
+dir.globfiles = globfiles
+
+-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
+-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
+-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
+-- t = dir.glob("f:/minimal/tex/**/*")
+-- print(dir.ls("f:/minimal/tex/**/*"))
+-- print(dir.ls("*.tex"))
+
+function dir.ls(pattern)
+    return table.concat(glob(pattern),"\n")
+end
+
+--~ mkdirs("temp")
+--~ mkdirs("a/b/c")
+--~ mkdirs(".","/a/b/c")
+--~ mkdirs("a","b","c")
+
+local make_indeed = true -- false
+
+if string.find(os.getenv("PATH"),";") then
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        local first, middle, last
+        local drive = false
+        first, middle, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            -- empty network path == local path
+        else
+            first, last = str:match("^(//)/*(.-)$")
+            if first then
+                middle, last = str:match("([^/]+)/+(.-)$")
+                if middle then
+                    pth = "//" .. middle
+                else
+                    pth = "//" .. last
+                    last = ""
+                end
+            else
+                first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$")
+                if first then
+                    pth, drive = first .. middle, true
+                else
+                    middle, last = str:match("^(/*)(.-)$")
+                    if not middle then
+                        last = str
+                    end
+                end
+            end
+        end
+        for s in gmatch(last,"[^/]+") do
+            if pth == "" then
+                pth = s
+            elseif drive then
+                pth, drive = pth .. s, false
+            else
+                pth = pth .. "/" .. s
+            end
+            if make_indeed and not lfs.isdir(pth) then
+                lfs.mkdir(pth)
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("a:"))
+--~         print(dir.mkdirs("a:/b/c"))
+--~         print(dir.mkdirs("a:b/c"))
+--~         print(dir.mkdirs("a:/bbb/c"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        local first, nothing, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            first = lfs.currentdir() .. "/"
+            first = first:gsub("\\","/")
+        end
+        if not first then
+            first, last = str:match("^(//)/*(.*)$")
+        end
+        if not first then
+            first, last = str:match("^([a-zA-Z]:)(.*)$")
+            if first and not find(last,"^/") then
+                local d = lfs.currentdir()
+                if lfs.chdir(first) then
+                    first = lfs.currentdir()
+                    first = first:gsub("\\","/")
+                end
+                lfs.chdir(d)
+            end
+        end
+        if not first then
+            first, last = lfs.currentdir(), str
+            first = first:gsub("\\","/")
+        end
+        last = last:gsub("//","/")
+        last = last:gsub("/%./","/")
+        last = last:gsub("^/*","")
+        first = first:gsub("/*$","")
+        if last == "" then
+            return first
+        else
+            return first .. "/" .. last
+        end
+    end
+
+else
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        str = str:gsub("/+","/")
+        if find(str,"^/") then
+            pth = "/"
+            for s in gmatch(str,"[^/]+") do
+                local first = (pth == "/")
+                if first then
+                    pth = pth .. s
+                else
+                    pth = pth .. "/" .. s
+                end
+                if make_indeed and not first and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        else
+            pth = "."
+            for s in gmatch(str,"[^/]+") do
+                pth = pth .. "/" .. s
+                if make_indeed and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        if not find(str,"^/") then
+            str = lfs.currentdir() .. "/" .. str
+        end
+        str = str:gsub("//","/")
+        str = str:gsub("/%./","/")
+        return str
+    end
+
+end
+
+dir.makedirs = dir.mkdirs
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-boolean'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+boolean = boolean or { }
+
+local type, tonumber = type, tonumber
+
+function boolean.tonumber(b)
+    if b then return 1 else return 0 end
+end
+
+function toboolean(str,tolerant)
+    if tolerant then
+        local tstr = type(str)
+        if tstr == "string" then
+            return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t"
+        elseif tstr == "number" then
+            return tonumber(str) ~= 0
+        elseif tstr == "nil" then
+            return false
+        else
+            return str
+        end
+    elseif str == "true" then
+        return true
+    elseif str == "false" then
+        return false
+    else
+        return str
+    end
+end
+
+function string.is_boolean(str)
+    if type(str) == "string" then
+        if str == "true" or str == "yes" or str == "on" or str == "t" then
+            return true
+        elseif str == "false" or str == "no" or str == "off" or str == "f" then
+            return false
+        end
+    end
+    return nil
+end
+
+function boolean.alwaystrue()
+    return true
+end
+
+function boolean.falsetrue()
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-unicode'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+utf = utf or unicode.utf8
+
+local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs
+
+unicode = unicode or { }
+
+-- 0  EF BB BF      UTF-8
+-- 1  FF FE         UTF-16-little-endian
+-- 2  FE FF         UTF-16-big-endian
+-- 3  FF FE 00 00   UTF-32-little-endian
+-- 4  00 00 FE FF   UTF-32-big-endian
+
+unicode.utfname = {
+    [0] = 'utf-8',
+    [1] = 'utf-16-le',
+    [2] = 'utf-16-be',
+    [3] = 'utf-32-le',
+    [4] = 'utf-32-be'
+}
+
+function unicode.utftype(f) -- \000 fails !
+    local str = f:read(4)
+    if not str then
+        f:seek('set')
+        return 0
+    elseif find(str,"^%z%z\254\255") then
+        return 4
+    elseif find(str,"^\255\254%z%z") then
+        return 3
+    elseif find(str,"^\254\255") then
+        f:seek('set',2)
+        return 2
+    elseif find(str,"^\255\254") then
+        f:seek('set',2)
+        return 1
+    elseif find(str,"^\239\187\191") then
+        f:seek('set',3)
+        return 0
+    else
+        f:seek('set')
+        return 0
+    end
+end
+
+function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg
+    local result, tmp, n, m, p = { }, { }, 0, 0, 0
+    -- lf | cr | crlf / (cr:13, lf:10)
+    local function doit()
+        if n == 10 then
+            if p ~= 13 then
+                result[#result+1] = concat(tmp)
+                tmp = { }
+                p = 0
+            end
+        elseif n == 13 then
+            result[#result+1] = concat(tmp)
+            tmp = { }
+            p = n
+        else
+            tmp[#tmp+1] = utfchar(n)
+            p = 0
+        end
+    end
+    for l,r in bytepairs(str) do
+        if r then
+            if endian then
+                n = l*256 + r
+            else
+                n = r*256 + l
+            end
+            if m > 0 then
+                n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000
+                m = 0
+                doit()
+            elseif n >= 0xD800 and n <= 0xDBFF then
+                m = n
+            else
+                doit()
+            end
+        end
+    end
+    if #tmp > 0 then
+        result[#result+1] = concat(tmp)
+    end
+    return result
+end
+
+function unicode.utf32_to_utf8(str, endian)
+    local result = { }
+    local tmp, n, m, p = { }, 0, -1, 0
+    -- lf | cr | crlf / (cr:13, lf:10)
+    local function doit()
+        if n == 10 then
+            if p ~= 13 then
+                result[#result+1] = concat(tmp)
+                tmp = { }
+                p = 0
+            end
+        elseif n == 13 then
+            result[#result+1] = concat(tmp)
+            tmp = { }
+            p = n
+        else
+            tmp[#tmp+1] = utfchar(n)
+            p = 0
+        end
+    end
+    for a,b in bytepairs(str) do
+        if a and b then
+            if m < 0 then
+                if endian then
+                    m = a*256*256*256 + b*256*256
+                else
+                    m = b*256 + a
+                end
+            else
+                if endian then
+                    n = m + a*256 + b
+                else
+                    n = m + b*256*256*256 + a*256*256
+                end
+                m = -1
+                doit()
+            end
+        else
+            break
+        end
+    end
+    if #tmp > 0 then
+        result[#result+1] = concat(tmp)
+    end
+    return result
+end
+
+local function little(c)
+    local b = byte(c) -- b = c:byte()
+    if b < 0x10000 then
+        return char(b%256,b/256)
+    else
+        b = b - 0x10000
+        local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00
+        return char(b1%256,b1/256,b2%256,b2/256)
+    end
+end
+
+local function big(c)
+    local b = byte(c)
+    if b < 0x10000 then
+        return char(b/256,b%256)
+    else
+        b = b - 0x10000
+        local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00
+        return char(b1/256,b1%256,b2/256,b2%256)
+    end
+end
+
+function unicode.utf8_to_utf16(str,littleendian)
+    if littleendian then
+        return char(255,254) .. utfgsub(str,".",little)
+    else
+        return char(254,255) .. utfgsub(str,".",big)
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-math'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan
+
+if not math.round then
+    function math.round(x)
+        return floor(x + 0.5)
+    end
+end
+
+if not math.div then
+    function math.div(n,m)
+        return floor(n/m)
+    end
+end
+
+if not math.mod then
+    function math.mod(n,m)
+        return n % m
+    end
+end
+
+local pipi = 2*math.pi/360
+
+function math.sind(d)
+    return sin(d*pipi)
+end
+
+function math.cosd(d)
+    return cos(d*pipi)
+end
+
+function math.tand(d)
+    return tan(d*pipi)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-utils'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- hm, quite unreadable
+
+if not utils        then utils        = { } end
+if not utils.merger then utils.merger = { } end
+if not utils.lua    then utils.lua    = { } end
+
+utils.merger.m_begin = "begin library merge"
+utils.merger.m_end   = "end library merge"
+utils.merger.pattern =
+    "%c+" ..
+    "%-%-%s+" .. utils.merger.m_begin ..
+    "%c+(.-)%c+" ..
+    "%-%-%s+" .. utils.merger.m_end ..
+    "%c+"
+
+function utils.merger._self_fake_()
+    return
+        "-- " .. "created merged file" .. "\n\n" ..
+        "-- " .. utils.merger.m_begin  .. "\n\n" ..
+        "-- " .. utils.merger.m_end    .. "\n\n"
+end
+
+function utils.report(...)
+    print(...)
+end
+
+utils.merger.strip_comment = true
+
+function utils.merger._self_load_(name)
+    local f, data = io.open(name), ""
+    if f then
+        utils.report("reading merge from %s",name)
+        data = f:read("*all")
+        f:close()
+    else
+        utils.report("unknown file to merge %s",name)
+    end
+    if data and utils.merger.strip_comment then
+        -- saves some 20K
+        data = data:gsub("%-%-~[^\n\r]*[\r\n]", "")
+    end
+    return data or ""
+end
+
+function utils.merger._self_save_(name, data)
+    if data ~= "" then
+        local f = io.open(name,'w')
+        if f then
+            utils.report("saving merge from %s",name)
+            f:write(data)
+            f:close()
+        end
+    end
+end
+
+function utils.merger._self_swap_(data,code)
+    if data ~= "" then
+        return (data:gsub(utils.merger.pattern, function(s)
+            return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n"
+        end, 1))
+    else
+        return ""
+    end
+end
+
+--~ stripper:
+--~
+--~ data = string.gsub(data,"%-%-~[^\n]*\n","")
+--~ data = string.gsub(data,"\n\n+","\n")
+
+function utils.merger._self_libs_(libs,list)
+    local result, f, frozen = { }, nil, false
+    result[#result+1] = "\n"
+    if type(libs) == 'string' then libs = { libs } end
+    if type(list) == 'string' then list = { list } end
+    local foundpath = nil
+    for _, lib in ipairs(libs) do
+        for _, pth in ipairs(list) do
+            pth = string.gsub(pth,"\\","/") -- file.clean_path
+            utils.report("checking library path %s",pth)
+            local name = pth .. "/" .. lib
+            if lfs.isfile(name) then
+                foundpath = pth
+            end
+        end
+        if foundpath then break end
+    end
+    if foundpath then
+        utils.report("using library path %s",foundpath)
+        local right, wrong = { }, { }
+        for _, lib in ipairs(libs) do
+            local fullname = foundpath .. "/" .. lib
+            if lfs.isfile(fullname) then
+            --  right[#right+1] = lib
+                utils.report("merging library %s",fullname)
+                result[#result+1] = "do -- create closure to overcome 200 locals limit"
+                result[#result+1] = io.loaddata(fullname,true)
+                result[#result+1] = "end -- of closure"
+            else
+            --  wrong[#wrong+1] = lib
+                utils.report("no library %s",fullname)
+            end
+        end
+        if #right > 0 then
+            utils.report("merged libraries: %s",table.concat(right," "))
+        end
+        if #wrong > 0 then
+            utils.report("skipped libraries: %s",table.concat(wrong," "))
+        end
+    else
+        utils.report("no valid library path found")
+    end
+    return table.concat(result, "\n\n")
+end
+
+function utils.merger.selfcreate(libs,list,target)
+    if target then
+        utils.merger._self_save_(
+            target,
+            utils.merger._self_swap_(
+                utils.merger._self_fake_(),
+                utils.merger._self_libs_(libs,list)
+            )
+        )
+    end
+end
+
+function utils.merger.selfmerge(name,libs,list,target)
+    utils.merger._self_save_(
+        target or name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            utils.merger._self_libs_(libs,list)
+        )
+    )
+end
+
+function utils.merger.selfclean(name)
+    utils.merger._self_save_(
+        name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            ""
+        )
+    )
+end
+
+function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true
+ -- utils.report("compiling",luafile,"into",lucfile)
+    os.remove(lucfile)
+    local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile)
+    if strip ~= false then
+        command = "-s " .. command
+    end
+    local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0)
+    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+     -- utils.report("removing",luafile)
+        os.remove(luafile)
+    end
+    return done
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-tra'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- the  tag is kind of generic and used for functions that are not
+-- bound to a variable, like node.new, node.copy etc (contrary to for instance
+-- node.has_attribute which is bound to a has_attribute local variable in mkiv)
+
+debugger = debugger or { }
+
+local counters = { }
+local names = { }
+local getinfo = debug.getinfo
+local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch
+
+-- one
+
+local function hook()
+    local f = getinfo(2,"f").func
+    local n = getinfo(2,"Sn")
+--  if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end
+    if f then
+        local cf = counters[f]
+        if cf == nil then
+            counters[f] = 1
+            names[f] = n
+        else
+            counters[f] = cf + 1
+        end
+    end
+end
+local function getname(func)
+    local n = names[func]
+    if n then
+        if n.what == "C" then
+            return n.name or ''
+        else
+            -- source short_src linedefined what name namewhat nups func
+            local name = n.name or n.namewhat or n.what
+            if not name or name == "" then name = "?" end
+            return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name)
+        end
+    else
+        return "unknown"
+    end
+end
+function debugger.showstats(printer,threshold)
+    printer   = printer or texio.write or print
+    threshold = threshold or 0
+    local total, grandtotal, functions = 0, 0, 0
+    printer("\n") -- ugly but ok
+ -- table.sort(counters)
+    for func, count in pairs(counters) do
+        if count > threshold then
+            local name = getname(func)
+            if not name:find("for generator") then
+                printer(format("%8i  %s", count, name))
+                total = total + count
+            end
+        end
+        grandtotal = grandtotal + count
+        functions = functions + 1
+    end
+    printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+end
+
+-- two
+
+--~ local function hook()
+--~     local n = getinfo(2)
+--~     if n.what=="C" and not n.name then
+--~         local f = tostring(debug.traceback())
+--~         local cf = counters[f]
+--~         if cf == nil then
+--~             counters[f] = 1
+--~             names[f] = n
+--~         else
+--~             counters[f] = cf + 1
+--~         end
+--~     end
+--~ end
+--~ function debugger.showstats(printer,threshold)
+--~     printer   = printer or texio.write or print
+--~     threshold = threshold or 0
+--~     local total, grandtotal, functions = 0, 0, 0
+--~     printer("\n") -- ugly but ok
+--~  -- table.sort(counters)
+--~     for func, count in pairs(counters) do
+--~         if count > threshold then
+--~             printer(format("%8i  %s", count, func))
+--~             total = total + count
+--~         end
+--~         grandtotal = grandtotal + count
+--~         functions = functions + 1
+--~     end
+--~     printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+--~ end
+
+-- rest
+
+function debugger.savestats(filename,threshold)
+    local f = io.open(filename,'w')
+    if f then
+        debugger.showstats(function(str) f:write(str) end,threshold)
+        f:close()
+    end
+end
+
+function debugger.enable()
+    debug.sethook(hook,"c")
+end
+
+function debugger.disable()
+    debug.sethook()
+--~ counters[debug.getinfo(2,"f").func] = nil
+end
+
+function debugger.tracing()
+    local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0
+    if n > 0 then
+        function debugger.tracing() return true  end ; return true
+    else
+        function debugger.tracing() return false end ; return false
+    end
+end
+
+--~ debugger.enable()
+
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+
+--~ debugger.disable()
+
+--~ print("")
+--~ debugger.showstats()
+--~ print("")
+--~ debugger.showstats(print,3)
+
+trackers = trackers or { }
+
+local data, done = { }, { }
+
+local function set(what,value)
+    for w in gmatch(lower(what),"[^, ]+") do
+        for d, f in next, data do
+            if done[d] then
+                -- prevent recursion due to wildcards
+            elseif find(d,w) then
+                done[d] = true
+                for i=1,#f do
+                    f[i](value)
+                end
+            end
+        end
+    end
+end
+
+local function reset()
+    for d, f in next, data do
+        for i=1,#f do
+            f[i](false)
+        end
+    end
+end
+
+function trackers.register(what,...)
+    what = lower(what)
+    local w = data[what]
+    if not w then
+        w = { }
+        data[what] = w
+    end
+    for _, fnc in next, { ... } do
+        local typ = type(fnc)
+        if typ == "function" then
+            w[#w+1] = fnc
+        elseif typ == "string" then
+            w[#w+1] = function(value) set(fnc,value,nesting) end
+        end
+    end
+end
+
+function trackers.enable(what)
+    done = { }
+    set(what,true)
+end
+
+function trackers.disable(what)
+    done = { }
+    if not what or what == "" then
+        trackers.reset(what)
+    else
+        set(what,false)
+    end
+end
+
+function trackers.reset(what)
+    done = { }
+    reset()
+end
+
+function trackers.list() -- pattern
+    local list = table.sortedkeys(data)
+    local user, system = { }, { }
+    for l=1,#list do
+        local what = list[l]
+        if find(what,"^%*") then
+            system[#system+1] = what
+        else
+            user[#user+1] = what
+        end
+    end
+    return user, system
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-env'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- A former version provided functionality for non embeded core
+-- scripts i.e. runtime library loading. Given the amount of
+-- Lua code we use now, this no longer makes sense. Much of this
+-- evolved before bytecode arrays were available and so a lot of
+-- code has disappeared already.
+
+local trace_verbose  = false  trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+
+local format = string.format
+
+-- precautions
+
+os.setlocale(nil,nil) -- useless feature and even dangerous in luatex
+
+function os.setlocale()
+    -- no way you can mess with it
+end
+
+-- dirty tricks
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then
+    profiler.start("luatex-profile.log")
+end
+
+-- environment
+
+environment             = environment or { }
+environment.arguments   = { }
+environment.files       = { }
+environment.sortedflags = nil
+
+if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end
+if not environment.version or environment.version == "" then             environment.version = "unknown"   end
+if not environment.jobname                              then             environment.jobname = "unknown"   end
+
+function environment.initialize_arguments(arg)
+    local arguments, files = { }, { }
+    environment.arguments, environment.files, environment.sortedflags = arguments, files, nil
+    for index, argument in pairs(arg) do
+        if index > 0 then
+            local flag, value = argument:match("^%-+(.+)=(.-)$")
+            if flag then
+                arguments[flag] = string.unquote(value or "")
+            else
+                flag = argument:match("^%-+(.+)")
+                if flag then
+                    arguments[flag] = true
+                else
+                    files[#files+1] = argument
+                end
+            end
+        end
+    end
+    environment.ownname = environment.ownname or arg[0] or 'unknown.lua'
+end
+
+function environment.setargument(name,value)
+    environment.arguments[name] = value
+end
+
+-- todo: defaults, better checks e.g on type (boolean versus string)
+--
+-- tricky: too many hits when we support partials unless we add
+-- a registration of arguments so from now on we have 'partial'
+
+function environment.argument(name,partial)
+    local arguments, sortedflags = environment.arguments, environment.sortedflags
+    if arguments[name] then
+        return arguments[name]
+    elseif partial then
+        if not sortedflags then
+            sortedflags = { }
+            for _,v in pairs(table.sortedkeys(arguments)) do
+                sortedflags[#sortedflags+1] = "^" .. v
+            end
+            environment.sortedflags = sortedflags
+        end
+        -- example of potential clash: ^mode ^modefile
+        for _,v in ipairs(sortedflags) do
+            if name:find(v) then
+                return arguments[v:sub(2,#v)]
+            end
+        end
+    end
+    return nil
+end
+
+function environment.split_arguments(separator) -- rather special, cut-off before separator
+    local done, before, after = false, { }, { }
+    for _,v in ipairs(environment.original_arguments) do
+        if not done and v == separator then
+            done = true
+        elseif done then
+            after[#after+1] = v
+        else
+            before[#before+1] = v
+        end
+    end
+    return before, after
+end
+
+function environment.reconstruct_commandline(arg,noquote)
+    arg = arg or environment.original_arguments
+    if noquote and #arg == 1 then
+        local a = arg[1]
+        a = resolvers.resolve(a)
+        a = a:unquote()
+        return a
+    elseif next(arg) then
+        local result = { }
+        for _,a in ipairs(arg) do -- ipairs 1 .. #n
+            a = resolvers.resolve(a)
+            a = a:unquote()
+            a = a:gsub('"','\\"') -- tricky
+            if a:find(" ") then
+                result[#result+1] = a:quote()
+            else
+                result[#result+1] = a
+            end
+        end
+        return table.join(result," ")
+    else
+        return ""
+    end
+end
+
+if arg then
+
+    -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later)
+    local newarg, instring = { }, false
+
+    for index, argument in ipairs(arg) do
+        if argument:find("^\"") then
+            newarg[#newarg+1] = argument:gsub("^\"","")
+            if not argument:find("\"$") then
+                instring = true
+            end
+        elseif argument:find("\"$") then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","")
+            instring = false
+        elseif instring then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument
+        else
+            newarg[#newarg+1] = argument
+        end
+    end
+    for i=1,-5,-1 do
+        newarg[i] = arg[i]
+    end
+
+    environment.initialize_arguments(newarg)
+    environment.original_arguments = newarg
+    environment.raw_arguments = arg
+
+    arg = { } -- prevent duplicate handling
+
+end
+
+-- weird place ... depends on a not yet loaded module
+
+function environment.texfile(filename)
+    return resolvers.find_file(filename,'tex')
+end
+
+function environment.luafile(filename)
+    local resolved = resolvers.find_file(filename,'tex') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    resolved = resolvers.find_file(filename,'texmfscripts') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    return resolvers.find_file(filename,'luatexlibs') or ""
+end
+
+environment.loadedluacode = loadfile -- can be overloaded
+
+--~ function environment.loadedluacode(name)
+--~     if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then
+--~         local chunk = loadstring(io.loaddata("texluac.luc"))
+--~         os.remove("texluac.luc")
+--~         return chunk
+--~     else
+--~         environment.loadedluacode = loadfile -- can be overloaded
+--~         return loadfile(name)
+--~     end
+--~ end
+
+function environment.luafilechunk(filename) -- used for loading lua bytecode in the format
+    filename = file.replacesuffix(filename, "lua")
+    local fullname = environment.luafile(filename)
+    if fullname and fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading file %s", fullname)
+        end
+        return environment.loadedluacode(fullname)
+    else
+        if trace_verbose then
+            logs.report("fileio","unknown file %s", filename)
+        end
+        return nil
+    end
+end
+
+-- the next ones can use the previous ones / combine
+
+function environment.loadluafile(filename, version)
+    local lucname, luaname, chunk
+    local basename = file.removesuffix(filename)
+    if basename == filename then
+        lucname, luaname = basename .. ".luc",  basename .. ".lua"
+    else
+        lucname, luaname = nil, basename -- forced suffix
+    end
+    -- when not overloaded by explicit suffix we look for a luc file first
+    local fullname = (lucname and environment.luafile(lucname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+    end
+    if chunk then
+        assert(chunk)()
+        if version then
+            -- we check of the version number of this chunk matches
+            local v = version -- can be nil
+            if modules and modules[filename] then
+                v = modules[filename].version -- new method
+            elseif versions and versions[filename] then
+                v = versions[filename]        -- old method
+            end
+            if v == version then
+                return true
+            else
+                if trace_verbose then
+                    logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version)
+                end
+                environment.loadluafile(filename)
+            end
+        else
+            return true
+        end
+    end
+    fullname = (luaname and environment.luafile(luaname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+        if not chunk then
+            if verbose then
+                logs.report("fileio","unknown file %s", filename)
+            end
+        else
+            assert(chunk)()
+            return true
+        end
+    end
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-inf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+local statusinfo, n, registered = { }, 0, { }
+
+statistics = statistics or { }
+
+statistics.enable    = true
+statistics.threshold = 0.05
+
+-- timing functions
+
+local clock = os.gettimeofday or os.clock
+
+function statistics.hastimer(instance)
+    return instance and instance.starttime
+end
+
+function statistics.starttiming(instance)
+    if instance then
+        local it = instance.timing
+        if not it then
+            it = 0
+        end
+        if it == 0 then
+            instance.starttime = clock()
+            if not instance.loadtime then
+                instance.loadtime = 0
+            end
+        end
+        instance.timing = it + 1
+    end
+end
+
+function statistics.stoptiming(instance, report)
+    if instance then
+        local it = instance.timing
+        if it > 1 then
+            instance.timing = it - 1
+        else
+            local starttime = instance.starttime
+            if starttime then
+                local stoptime = clock()
+                local loadtime = stoptime - starttime
+                instance.stoptime = stoptime
+                instance.loadtime = instance.loadtime + loadtime
+                if report then
+                    statistics.report("load time %0.3f",loadtime)
+                end
+                instance.timing = 0
+                return loadtime
+            end
+        end
+    end
+    return 0
+end
+
+function statistics.elapsedtime(instance)
+    return format("%0.3f",(instance and instance.loadtime) or 0)
+end
+
+function statistics.elapsedindeed(instance)
+    local t = (instance and instance.loadtime) or 0
+    return t > statistics.threshold
+end
+
+-- general function
+
+function statistics.register(tag,fnc)
+    if statistics.enable and type(fnc) == "function" then
+        local rt = registered[tag] or (#statusinfo + 1)
+        statusinfo[rt] = { tag, fnc }
+        registered[tag] = rt
+        if #tag > n then n = #tag end
+    end
+end
+
+function statistics.show(reporter)
+    if statistics.enable then
+        if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end
+        -- this code will move
+        local register = statistics.register
+        register("luatex banner", function()
+            return string.lower(status.banner)
+        end)
+        register("control sequences", function()
+            return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra)
+        end)
+        register("callbacks", function()
+            local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
+            return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
+        end)
+        register("current memory usage", statistics.memused)
+        register("runtime",statistics.runtime)
+--         --
+        for i=1,#statusinfo do
+            local s = statusinfo[i]
+            local r = s[2]()
+            if r then
+                reporter(s[1],r,n)
+            end
+        end
+        statistics.enable = false
+    end
+end
+
+function statistics.show_job_stat(tag,data,n)
+    texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data))
+end
+
+function statistics.memused() -- no math.round yet -)
+    local round = math.round or math.floor
+    return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000))
+end
+
+if statistics.runtime then
+    -- already loaded and set
+elseif luatex and luatex.starttime then
+    statistics.starttime = luatex.starttime
+    statistics.loadtime = 0
+    statistics.timing = 0
+else
+    statistics.starttiming(statistics)
+end
+
+function statistics.runtime()
+    statistics.stoptiming(statistics)
+    return statistics.formatruntime(statistics.elapsedtime(statistics))
+end
+
+function statistics.formatruntime(runtime)
+    return format("%s seconds", statistics.elapsedtime(statistics))
+end
+
+function statistics.timed(action,report)
+    local timer = { }
+    report = report or logs.simple
+    statistics.starttiming(timer)
+    action()
+    statistics.stoptiming(timer)
+    report("total runtime: %s",statistics.elapsedtime(timer))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-log'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this is old code that needs an overhaul
+
+local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+
+if texlua then
+    write_nl = print
+    write    = io.write
+end
+
+--[[ldx--
+
This is a prelude to a more extensive logging module. For the sake
+of parsing log files, in addition to the standard logging we will
+provide an  structured file. Actually, any logging that
+is hooked into callbacks will be \XML\ by default.

+--ldx]]--
+
+logs     = logs     or { }
+logs.xml = logs.xml or { }
+logs.tex = logs.tex or { }
+
+--[[ldx--
+
This looks pretty ugly but we need to speed things up a bit.

+--ldx]]--
+
+logs.moreinfo = [[
+more information about ConTeXt and the tools that come with it can be found at:
+
+maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
+webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
+wiki     : http://contextgarden.net
+]]
+
+logs.levels = {
+    ['error']   = 1,
+    ['warning'] = 2,
+    ['info']    = 3,
+    ['debug']   = 4,
+}
+
+logs.functions = {
+    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct',
+    'start_run', 'stop_run',
+    'start_page_number', 'stop_page_number',
+    'report_output_pages', 'report_output_log',
+    'report_tex_stat', 'report_job_stat',
+    'show_open', 'show_close', 'show_load',
+}
+
+logs.tracers = {
+}
+
+logs.level = 0
+logs.mode  = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"))
+
+function logs.set_level(level)
+    logs.level = logs.levels[level] or level
+end
+
+function logs.set_method(method)
+    for _, v in next, logs.functions do
+        logs[v] = logs[method][v] or function() end
+    end
+end
+
+-- tex logging
+
+function logs.tex.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(category .. " | " .. format(fmt,...))
+    else
+        write_nl(category .. " |")
+    end
+end
+
+function logs.tex.line(fmt,...) -- new
+    if fmt then
+        write_nl(format(fmt,...))
+    else
+        write_nl("")
+    end
+end
+
+local texcount = tex and tex.count
+
+function logs.tex.start_page_number()
+    local real, user, sub = texcount[0], texcount[1], texcount[2]
+    if real > 0 then
+        if user > 0 then
+            if sub > 0 then
+                write(format("[%s.%s.%s",real,user,sub))
+            else
+                write(format("[%s.%s",real,user))
+            end
+        else
+            write(format("[%s",real))
+        end
+    else
+        write("[-")
+    end
+end
+
+function logs.tex.stop_page_number()
+    write("]")
+end
+
+logs.tex.report_job_stat = statistics.show_job_stat
+
+-- xml logging
+
+function logs.xml.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",category,format(fmt,...)))
+    else
+        write_nl(format("",category))
+    end
+end
+function logs.xml.line(fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",format(fmt,...)))
+    else
+        write_nl("")
+    end
+end
+
+function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
+function logs.xml.stop () if logs.level > 0 then tw("") end end
+function logs.xml.push () if logs.level > 0 then tw("" ) end end
+
+function logs.xml.start_run()
+    write_nl("")
+    write_nl("") --  xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng'
+    write_nl("")
+end
+
+function logs.xml.stop_run()
+    write_nl("")
+end
+
+function logs.xml.start_page_number()
+    write_nl(format("
")
+    write_nl("")
+end
+
+function logs.xml.report_output_pages(p,b)
+    write_nl(format("", p))
+    write_nl(format("", b))
+    write_nl("")
+end
+
+function logs.xml.report_output_log()
+end
+
+function logs.xml.report_tex_stat(k,v)
+    texiowrite_nl("log",""..tostring(v).."")
+end
+
+local level = 0
+
+function logs.xml.show_open(name)
+    level = level + 1
+    texiowrite_nl(format("",level,name))
+end
+
+function logs.xml.show_close(name)
+    texiowrite(" ")
+    level = level - 1
+end
+
+function logs.xml.show_load(name)
+    texiowrite_nl(format("",level+1,name))
+end
+
+--
+
+local name, banner = 'report', 'context'
+
+local function report(category,fmt,...)
+    if fmt then
+        write_nl(format("%s | %s: %s",name,category,format(fmt,...)))
+    elseif category then
+        write_nl(format("%s | %s",name,category))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+local function simple(fmt,...)
+    if fmt then
+        write_nl(format("%s | %s",name,format(fmt,...)))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+function logs.setprogram(_name_,_banner_,_verbose_)
+    name, banner = _name_, _banner_
+    if _verbose_ then
+        trackers.enable("resolvers.verbose")
+    end
+    logs.set_method("tex")
+    logs.report = report -- also used in libraries
+    logs.simple = simple -- only used in scripts !
+    if utils then
+        utils.report = simple
+    end
+    logs.verbose = _verbose_
+end
+
+function logs.setverbose(what)
+    if what then
+        trackers.enable("resolvers.verbose")
+    else
+        trackers.disable("resolvers.verbose")
+    end
+    logs.verbose = what or false
+end
+
+function logs.extendbanner(_banner_,_verbose_)
+    banner = banner .. " | ".. _banner_
+    if _verbose_ ~= nil then
+        logs.setverbose(what)
+    end
+end
+
+logs.verbose = false
+logs.report  = logs.tex.report
+logs.simple  = logs.tex.report
+
+function logs.reportlines(str) -- todo: 
+    for line in str:gmatch("(.-)[\n\r]") do
+        logs.report(line)
+    end
+end
+
+function logs.reportline() -- for scripts too
+    logs.report()
+end
+
+logs.simpleline = logs.reportline
+
+function logs.help(message,option)
+    logs.report(banner)
+    logs.reportline()
+    logs.reportlines(message)
+    local moreinfo = logs.moreinfo or ""
+    if moreinfo ~= "" and option ~= "nomoreinfo" then
+        logs.reportline()
+        logs.reportlines(moreinfo)
+    end
+end
+
+logs.set_level('error')
+logs.set_method('tex')
+
+function logs.system(whereto,process,jobname,category,...)
+    for i=1,10 do
+        local f = io.open(whereto,"a")
+        if f then
+            f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)))
+            f:close()
+            break
+        else
+            sleep(0.1)
+        end
+    end
+end
+
+--~ local syslogname = "oeps.xxx"
+--~
+--~ for i=1,10 do
+--~     logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+    comment   = "companion to luat-lib.tex",
+}
+
+-- After a few years using the code the large luat-inp.lua file
+-- has been split up a bit. In the process some functionality was
+-- dropped:
+--
+-- * support for reading lsr files
+-- * selective scanning (subtrees)
+-- * some public auxiliary functions were made private
+--
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
+-- This lib is multi-purpose and can be loaded again later on so that
+-- additional functionality becomes available. We will split thislogs.report("fileio",
+-- module in components once we're done with prototyping. This is the
+-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing
+-- something in this module one can best check with Taco or Hans first; there
+-- is some nasty trickery going on that relates to traditional kpse support.
+
+-- To be considered: hash key lowercase, first entry in table filename
+-- (any case), rest paths (so no need for optimization). Or maybe a
+-- separate table that matches lowercase names to mixed case when
+-- present. In that case the lower() cases can go away. I will do that
+-- only when we run into problems with names ... well ... Iwona-Regular.
+
+-- Beware, loading and saving is overloaded in luat-tmp!
+
+local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
+local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
+local next, type = next, type
+
+local trace_locating, trace_detail, trace_verbose  = false, false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+trackers.register("resolvers.detail",   function(v) trace_detail   = v trackers.enable("resolvers.verbose,resolvers.detail") end)
+
+if not resolvers then
+    resolvers = {
+        suffixes     = { },
+        formats      = { },
+        dangerous    = { },
+        suffixmap    = { },
+        alternatives = { },
+        locators     = { },  -- locate databases
+        hashers      = { },  -- load databases
+        generators   = { },  -- generate databases
+    }
+end
+
+local resolvers = resolvers
+
+resolvers.locators  .notfound = { nil }
+resolvers.hashers   .notfound = { nil }
+resolvers.generators.notfound = { nil }
+
+resolvers.cacheversion = '1.0.1'
+resolvers.cnfname      = 'texmf.cnf'
+resolvers.luaname      = 'texmfcnf.lua'
+resolvers.homedir      = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~'
+resolvers.cnfdefault   = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
+
+local dummy_path_expr = "^!*unset/*$"
+
+local formats      = resolvers.formats
+local suffixes     = resolvers.suffixes
+local dangerous    = resolvers.dangerous
+local suffixmap    = resolvers.suffixmap
+local alternatives = resolvers.alternatives
+
+formats['afm'] = 'AFMFONTS'       suffixes['afm'] = { 'afm' }
+formats['enc'] = 'ENCFONTS'       suffixes['enc'] = { 'enc' }
+formats['fmt'] = 'TEXFORMATS'     suffixes['fmt'] = { 'fmt' }
+formats['map'] = 'TEXFONTMAPS'    suffixes['map'] = { 'map' }
+formats['mp']  = 'MPINPUTS'       suffixes['mp']  = { 'mp' }
+formats['ocp'] = 'OCPINPUTS'      suffixes['ocp'] = { 'ocp' }
+formats['ofm'] = 'OFMFONTS'       suffixes['ofm'] = { 'ofm', 'tfm' }
+formats['otf'] = 'OPENTYPEFONTS'  suffixes['otf'] = { 'otf' } -- 'ttf'
+formats['opl'] = 'OPLFONTS'       suffixes['opl'] = { 'opl' }
+formats['otp'] = 'OTPINPUTS'      suffixes['otp'] = { 'otp' }
+formats['ovf'] = 'OVFFONTS'       suffixes['ovf'] = { 'ovf', 'vf' }
+formats['ovp'] = 'OVPFONTS'       suffixes['ovp'] = { 'ovp' }
+formats['tex'] = 'TEXINPUTS'      suffixes['tex'] = { 'tex' }
+formats['tfm'] = 'TFMFONTS'       suffixes['tfm'] = { 'tfm' }
+formats['ttf'] = 'TTFONTS'        suffixes['ttf'] = { 'ttf', 'ttc' }
+formats['pfb'] = 'T1FONTS'        suffixes['pfb'] = { 'pfb', 'pfa' }
+formats['vf']  = 'VFFONTS'        suffixes['vf']  = { 'vf' }
+
+formats['fea'] = 'FONTFEATURES'   suffixes['fea'] = { 'fea' }
+formats['cid'] = 'FONTCIDMAPS'    suffixes['cid'] = { 'cid', 'cidmap' }
+
+formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new
+suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua'
+
+formats ['lua'] = 'LUAINPUTS' -- new
+suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' }
+
+-- backward compatible ones
+
+alternatives['map files']            = 'map'
+alternatives['enc files']            = 'enc'
+alternatives['cid files']            = 'cid'
+alternatives['fea files']            = 'fea'
+alternatives['opentype fonts']       = 'otf'
+alternatives['truetype fonts']       = 'ttf'
+alternatives['truetype collections'] = 'ttc'
+alternatives['type1 fonts']          = 'pfb'
+
+-- obscure ones
+
+formats ['misc fonts'] = ''
+suffixes['misc fonts'] = { }
+
+formats     ['sfd']                      = 'SFDFONTS'
+suffixes    ['sfd']                      = { 'sfd' }
+alternatives['subfont definition files'] = 'sfd'
+
+-- In practice we will work within one tds tree, but i want to keep
+-- the option open to build tools that look at multiple trees, which is
+-- why we keep the tree specific data in a table. We used to pass the
+-- instance but for practical pusposes we now avoid this and use a
+-- instance variable.
+
+-- here we catch a few new thingies (todo: add these paths to context.tmf)
+--
+-- FONTFEATURES  = .;$TEXMF/fonts/fea//
+-- FONTCIDMAPS   = .;$TEXMF/fonts/cid//
+
+-- we always have one instance active
+
+resolvers.instance = resolvers.instance or nil -- the current one (slow access)
+local instance = resolvers.instance or nil -- the current one (fast access)
+
+function resolvers.newinstance()
+
+    -- store once, freeze and faster (once reset we can best use
+    -- instance.environment) maybe better have a register suffix
+    -- function
+
+    for k, v in next, suffixes do
+        for i=1,#v do
+            local vi = v[i]
+            if vi then
+                suffixmap[vi] = k
+            end
+        end
+    end
+
+    -- because vf searching is somewhat dangerous, we want to prevent
+    -- too liberal searching esp because we do a lookup on the current
+    -- path anyway; only tex (or any) is safe
+
+    for k, v in next, formats do
+        dangerous[k] = true
+    end
+    dangerous.tex = nil
+
+    -- the instance
+
+    local newinstance = {
+        rootpath        = '',
+        treepath        = '',
+        progname        = 'context',
+        engine          = 'luatex',
+        format          = '',
+        environment     = { },
+        variables       = { },
+        expansions      = { },
+        files           = { },
+        remap           = { },
+        configuration   = { },
+        setup           = { },
+        order           = { },
+        found           = { },
+        foundintrees    = { },
+        kpsevars        = { },
+        hashes          = { },
+        cnffiles        = { },
+        luafiles        = { },
+        lists           = { },
+        remember        = true,
+        diskcache       = true,
+        renewcache      = false,
+        scandisk        = true,
+        cachepath       = nil,
+        loaderror       = false,
+        sortdata        = false,
+        savelists       = true,
+        cleanuppaths    = true,
+        allresults      = false,
+        pattern         = nil, -- lists
+        data            = { }, -- only for loading
+        force_suffixes  = true,
+        fakepaths       = { },
+    }
+
+    local ne = newinstance.environment
+
+    for k,v in next, os.env do
+        ne[k] = resolvers.bare_variable(v)
+    end
+
+    return newinstance
+
+end
+
+function resolvers.setinstance(someinstance)
+    instance = someinstance
+    resolvers.instance = someinstance
+    return someinstance
+end
+
+function resolvers.reset()
+    return resolvers.setinstance(resolvers.newinstance())
+end
+
+local function reset_hashes()
+    instance.lists = { }
+    instance.found = { }
+end
+
+local function check_configuration() -- not yet ok, no time for debugging now
+    local ie = instance.environment
+    local function fix(varname,default)
+        local proname = varname .. "." .. instance.progname or "crap"
+        local p, v = ie[proname], ie[varname]
+        if not ((p and p ~= "") or (v and v ~= "")) then
+            instance.variables[varname] = default -- or environment?
+        end
+    end
+    local name = os.name
+    if name == "windows" then
+        fix("OSFONTDIR", "c:/windows/fonts//")
+    elseif name == "macosx" then
+        fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//")
+    else
+        -- bad luck
+    end
+    fix("LUAINPUTS"   , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm
+    fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("LUATEXLIBS"  , ".;$TEXMF/luatex/lua//")
+end
+
+function resolvers.bare_variable(str) -- assumes str is a string
+    return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2"))
+end
+
+function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail'
+    if n then
+        trackers.disable("resolvers.*")
+        trackers.enable("resolvers."..n)
+    end
+end
+
+resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE"))
+
+function resolvers.osenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
+        if e == nil then
+         -- value = "" -- false
+        else
+            value = resolvers.bare_variable(e)
+        end
+        ie[key] = value
+    end
+    return value or ""
+end
+
+function resolvers.env(key)
+    return instance.environment[key] or resolvers.osenv(key)
+end
+
+--
+
+local function expand_vars(lst) -- simple vars
+    local variables, env = instance.variables, resolvers.env
+    local function resolve(a)
+        return variables[a] or env(a)
+    end
+    for k=1,#lst do
+        lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve)
+    end
+end
+
+local function expanded_var(var) -- simple vars
+    local function resolve(a)
+        return instance.variables[a] or resolvers.env(a)
+    end
+    return (gsub(var,"%$([%a%d%_%-]+)",resolve))
+end
+
+local function entry(entries,name)
+    if name and (name ~= "") then
+        name = gsub(name,'%$','')
+        local result = entries[name..'.'..instance.progname] or entries[name]
+        if result then
+            return result
+        else
+            result = resolvers.env(name)
+            if result then
+                instance.variables[name] = result
+                resolvers.expand_variables()
+                return instance.expansions[name] or ""
+            end
+        end
+    end
+    return ""
+end
+
+local function is_entry(entries,name)
+    if name and name ~= "" then
+        name = gsub(name,'%$','')
+        return (entries[name..'.'..instance.progname] or entries[name]) ~= nil
+    else
+        return false
+    end
+end
+
+-- {a,b,c,d}
+-- a,b,c/{p,q,r},d
+-- a,b,c/{p,q,r}/d/{x,y,z}//
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a{b,c}{d,e}f
+-- {a,b,c,d}
+-- {a,b,c/{p,q,r},d}
+-- {a,b,c/{p,q,r}/d/{x,y,z}//}
+-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
+-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
+-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
+
+-- this one is better and faster, but it took me a while to realize
+-- that this kind of replacement is cleaner than messy parsing and
+-- fuzzy concatenating we can probably gain a bit with selectively
+-- applying lpeg, but experiments with lpeg parsing this proved not to
+-- work that well; the parsing is ok, but dealing with the resulting
+-- table is a pain because we need to work inside-out recursively
+
+local function splitpathexpr(str, t, validate)
+    -- no need for further optimization as it is only called a
+    -- few times, we can use lpeg for the sub; we could move
+    -- the local functions outside the body
+    t = t or { }
+    str = gsub(str,",}",",@}")
+    str = gsub(str,"{,","{@,")
+ -- str = "@" .. str .. "@"
+    local ok, done
+    local function do_first(a,b)
+        local t = { }
+        for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_second(a,b)
+        local t = { }
+        for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_both(a,b)
+        local t = { }
+        for sa in gmatch(a,"[^,]+") do
+            for sb in gmatch(b,"[^,]+") do
+                t[#t+1] = sa .. sb
+            end
+        end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_three(a,b,c)
+        return a .. b.. c
+    end
+    while true do
+        done = false
+        while true do
+            str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both)
+            if ok > 0 then done = true else break end
+        end
+        str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three)
+        if ok > 0 then done = true end
+        if not done then break end
+    end
+    str = gsub(str,"[{}]", "")
+    str = gsub(str,"@","")
+    if validate then
+        for s in gmatch(str,"[^,]+") do
+            s = validate(s)
+            if s then t[#t+1] = s end
+        end
+    else
+        for s in gmatch(str,"[^,]+") do
+            t[#t+1] = s
+        end
+    end
+    return t
+end
+
+local function expanded_path_from_list(pathlist) -- maybe not a list, just a path
+    -- a previous version fed back into pathlist
+    local newlist, ok = { }, false
+    for k=1,#pathlist do
+        if find(pathlist[k],"[{}]") then
+            ok = true
+            break
+        end
+    end
+    if ok then
+        local function validate(s)
+            s = file.collapse_path(s)
+            return s ~= "" and not find(s,dummy_path_expr) and s
+        end
+        for k=1,#pathlist do
+            splitpathexpr(pathlist[k],newlist,validate)
+        end
+    else
+        for k=1,#pathlist do
+            for p in gmatch(pathlist[k],"([^,]+)") do
+                p = file.collapse_path(p)
+                if p ~= "" then newlist[#newlist+1] = p end
+            end
+        end
+    end
+    return newlist
+end
+
+-- we follow a rather traditional approach:
+--
+-- (1) texmf.cnf given in TEXMFCNF
+-- (2) texmf.cnf searched in default variable
+--
+-- also we now follow the stupid route: if not set then just assume *one*
+-- cnf file under texmf (i.e. distribution)
+
+resolvers.ownpath     = resolvers.ownpath or nil
+resolvers.ownbin      = resolvers.ownbin  or arg[-2] or arg[-1] or arg[0] or "luatex"
+resolvers.autoselfdir = true -- false may be handy for debugging
+
+function resolvers.getownpath()
+    if not resolvers.ownpath then
+        if resolvers.autoselfdir and os.selfdir then
+            resolvers.ownpath = os.selfdir
+        else
+            local binary = resolvers.ownbin
+            if os.platform == "windows" then
+                binary = file.replacesuffix(binary,"exe")
+            end
+            for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+                local b = file.join(p,binary)
+                if lfs.isfile(b) then
+                    -- we assume that after changing to the path the currentdir function
+                    -- resolves to the real location and use this side effect here; this
+                    -- trick is needed because on the mac installations use symlinks in the
+                    -- path instead of real locations
+                    local olddir = lfs.currentdir()
+                    if lfs.chdir(p) then
+                        local pp = lfs.currentdir()
+                        if trace_verbose and p ~= pp then
+                            logs.report("fileio","following symlink %s to %s",p,pp)
+                        end
+                        resolvers.ownpath = pp
+                        lfs.chdir(olddir)
+                    else
+                        if trace_verbose then
+                            logs.report("fileio","unable to check path %s",p)
+                        end
+                        resolvers.ownpath =  p
+                    end
+                    break
+                end
+            end
+        end
+        if not resolvers.ownpath then resolvers.ownpath = '.' end
+    end
+    return resolvers.ownpath
+end
+
+local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" }
+
+local function identify_own()
+    local ownpath = resolvers.getownpath() or lfs.currentdir()
+    local ie = instance.environment
+    if ownpath then
+        if resolvers.env('SELFAUTOLOC')    == "" then os.env['SELFAUTOLOC']    = file.collapse_path(ownpath) end
+        if resolvers.env('SELFAUTODIR')    == "" then os.env['SELFAUTODIR']    = file.collapse_path(ownpath .. "/..") end
+        if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end
+    else
+        logs.report("fileio","error: unable to locate ownpath")
+        os.exit()
+    end
+    if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end
+    if resolvers.env('TEXOS')    == "" then os.env['TEXOS']    = resolvers.env('SELFAUTODIR') end
+    if resolvers.env('TEXROOT')  == "" then os.env['TEXROOT']  = resolvers.env('SELFAUTOPARENT') end
+    if trace_verbose then
+        for i=1,#own_places do
+            local v = own_places[i]
+            logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown")
+        end
+    end
+    identify_own = function() end
+end
+
+function resolvers.identify_cnf()
+    if #instance.cnffiles == 0 then
+        -- fallback
+        identify_own()
+        -- the real search
+        resolvers.expand_variables()
+        local t = resolvers.split_path(resolvers.env('TEXMFCNF'))
+        t = expanded_path_from_list(t)
+        expand_vars(t) -- redundant
+        local function locate(filename,list)
+            for i=1,#t do
+                local ti = t[i]
+                local texmfcnf = file.collapse_path(file.join(ti,filename))
+                if lfs.isfile(texmfcnf) then
+                    list[#list+1] = texmfcnf
+                end
+            end
+        end
+        locate(resolvers.luaname,instance.luafiles)
+        locate(resolvers.cnfname,instance.cnffiles)
+    end
+end
+
+local function load_cnf_file(fname)
+    fname = resolvers.clean_path(fname)
+    local lname = file.replacesuffix(fname,'lua')
+    local f = io.open(lname)
+    if f then -- this will go
+        f:close()
+        local dname = file.dirname(fname)
+        if not instance.configuration[dname] then
+            resolvers.load_data(dname,'configuration',lname and file.basename(lname))
+            instance.order[#instance.order+1] = instance.configuration[dname]
+        end
+    else
+        f = io.open(fname)
+        if f then
+            if trace_verbose then
+                logs.report("fileio","loading %s", fname)
+            end
+            local line, data, n, k, v
+            local dname = file.dirname(fname)
+            if not instance.configuration[dname] then
+                instance.configuration[dname] = { }
+                instance.order[#instance.order+1] = instance.configuration[dname]
+            end
+            local data = instance.configuration[dname]
+            while true do
+                local line, n = f:read(), 0
+                if line then
+                    while true do -- join lines
+                        line, n = gsub(line,"\\%s*$", "")
+                        if n > 0 then
+                            line = line .. f:read()
+                        else
+                            break
+                        end
+                    end
+                    if not find(line,"^[%%#]") then
+                        local l = gsub(line,"%s*%%.*$","")
+                        local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$")
+                        if k and v and not data[k] then
+                            v = gsub(v,"[%%#].*",'')
+                            data[k] = gsub(v,"~","$HOME")
+                            instance.kpsevars[k] = true
+                        end
+                    end
+                else
+                    break
+                end
+            end
+            f:close()
+        elseif trace_verbose then
+            logs.report("fileio","skipping %s", fname)
+        end
+    end
+end
+
+local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in next, c do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = resolvers.bare_variable(v)
+                end
+            end
+        end
+    end
+end
+
+function resolvers.load_cnf()
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            load_cnf_file(fname)
+        end
+    end
+    -- instance.cnffiles contain complete names now !
+    if #instance.cnffiles == 0 then
+        if trace_verbose then
+            logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)")
+        end
+    else
+        instance.rootpath = instance.cnffiles[1]
+        for k,fname in ipairs(instance.cnffiles) do
+            instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        if instance.diskcache and not instance.renewcache then
+            resolvers.loadoldconfig(instance.cnffiles)
+            if instance.loaderror then
+                loadoldconfigdata()
+                resolvers.saveoldconfig()
+            end
+        else
+            loadoldconfigdata()
+            if instance.renewcache then
+                resolvers.saveoldconfig()
+            end
+        end
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+function resolvers.load_lua()
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        resolvers.loadnewconfig()
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+-- database loading
+
+function resolvers.load_hash()
+    resolvers.locatelists()
+    if instance.diskcache and not instance.renewcache then
+        resolvers.loadfiles()
+        if instance.loaderror then
+            resolvers.loadlists()
+            resolvers.savefiles()
+        end
+    else
+        resolvers.loadlists()
+        if instance.renewcache then
+            resolvers.savefiles()
+        end
+    end
+end
+
+function resolvers.append_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash append: %s",tag)
+    end
+    insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.prepend_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash prepend: %s",tag)
+    end
+    insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash
+--  local t = resolvers.expanded_path_list('TEXMF') -- full expansion
+    local t = resolvers.split_path(resolvers.env('TEXMF'))
+    insert(t,1,specification)
+    local newspec = concat(t,";")
+    if instance.environment["TEXMF"] then
+        instance.environment["TEXMF"] = newspec
+    elseif instance.variables["TEXMF"] then
+        instance.variables["TEXMF"] = newspec
+    else
+        -- weird
+    end
+    resolvers.expand_variables()
+    reset_hashes()
+end
+
+-- locators
+
+function resolvers.locatelists()
+    for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do
+        if trace_verbose then
+            logs.report("fileio","locating list of %s",path)
+        end
+        resolvers.locatedatabase(file.collapse_path(path))
+    end
+end
+
+function resolvers.locatedatabase(specification)
+    return resolvers.methodhandler('locators', specification)
+end
+
+function resolvers.locators.tex(specification)
+    if specification and specification ~= '' and lfs.isdir(specification) then
+        if trace_locating then
+            logs.report("fileio",'! tex locator found: %s',specification)
+        end
+        resolvers.append_hash('file',specification,filename)
+    elseif trace_locating then
+        logs.report("fileio",'? tex locator not found: %s',specification)
+    end
+end
+
+-- hashers
+
+function resolvers.hashdatabase(tag,name)
+    return resolvers.methodhandler('hashers',tag,name)
+end
+
+function resolvers.loadfiles()
+    instance.loaderror = false
+    instance.files = { }
+    if not instance.renewcache then
+        for _, hash in ipairs(instance.hashes) do
+            resolvers.hashdatabase(hash.tag,hash.name)
+            if instance.loaderror then break end
+        end
+    end
+end
+
+function resolvers.hashers.tex(tag,name)
+    resolvers.load_data(tag,'files')
+end
+
+-- generators:
+
+function resolvers.loadlists()
+    for _, hash in ipairs(instance.hashes) do
+        resolvers.generatedatabase(hash.tag)
+    end
+end
+
+function resolvers.generatedatabase(specification)
+    return resolvers.methodhandler('generators', specification)
+end
+
+-- starting with . or .. etc or funny char
+
+local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t"))
+
+function resolvers.generators.tex(specification)
+    local tag = specification
+    if trace_verbose then
+        logs.report("fileio","scanning path %s",specification)
+    end
+    instance.files[tag] = { }
+    local files = instance.files[tag]
+    local n, m, r = 0, 0, 0
+    local spec = specification .. '/'
+    local attributes = lfs.attributes
+    local directory = lfs.dir
+    local function action(path)
+        local full
+        if path then
+            full = spec .. path .. '/'
+        else
+            full = spec
+        end
+        for name in directory(full) do
+            if not weird:match(name) then
+                local mode = attributes(full..name,'mode')
+                if mode == 'file' then
+                    if path then
+                        n = n + 1
+                        local f = files[name]
+                        if f then
+                            if type(f) == 'string' then
+                                files[name] = { f, path }
+                            else
+                                f[#f+1] = path
+                            end
+                        else -- probably unique anyway
+                            files[name] = path
+                            local lower = lower(name)
+                            if name ~= lower then
+                                files["remap:"..lower] = name
+                                r = r + 1
+                            end
+                        end
+                    end
+                elseif mode == 'directory' then
+                    m = m + 1
+                    if path then
+                        action(path..'/'..name)
+                    else
+                        action(name)
+                    end
+                end
+            end
+        end
+    end
+    action()
+    if trace_verbose then
+        logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r)
+    end
+end
+
+-- savers, todo
+
+function resolvers.savefiles()
+    resolvers.save_data('files')
+end
+
+-- A config (optionally) has the paths split in tables. Internally
+-- we join them and split them after the expansion has taken place. This
+-- is more convenient.
+
+function resolvers.splitconfig()
+    for i,c in ipairs(instance) do
+        for k,v in pairs(c) do
+            if type(v) == 'string' then
+                local t = file.split_path(v)
+                if #t > 1 then
+                    c[k] = t
+                end
+            end
+        end
+    end
+end
+
+function resolvers.joinconfig()
+    for i,c in ipairs(instance.order) do
+        for k,v in pairs(c) do -- ipairs?
+            if type(v) == 'table' then
+                c[k] = file.join_path(v)
+            end
+        end
+    end
+end
+function resolvers.split_path(str)
+    if type(str) == 'table' then
+        return str
+    else
+        return file.split_path(str)
+    end
+end
+function resolvers.join_path(str)
+    if type(str) == 'table' then
+        return file.join_path(str)
+    else
+        return str
+    end
+end
+
+function resolvers.splitexpansions()
+    local ie = instance.expansions
+    for k,v in next, ie do
+        local t, h = { }, { }
+        for _,vv in ipairs(file.split_path(v)) do
+            if vv ~= "" and not h[vv] then
+                t[#t+1] = vv
+                h[vv] = true
+            end
+        end
+        if #t > 1 then
+            ie[k] = t
+        else
+            ie[k] = t[1]
+        end
+    end
+end
+
+-- end of split/join code
+
+function resolvers.saveoldconfig()
+    resolvers.splitconfig()
+    resolvers.save_data('configuration')
+    resolvers.joinconfig()
+end
+
+resolvers.configbanner = [[
+-- This is a Luatex configuration file created by 'luatools.lua' or
+-- 'luatex.exe' directly. For comment, suggestions and questions you can
+-- contact the ConTeXt Development Team. This configuration file is
+-- not copyrighted. [HH & TH]
+]]
+
+function resolvers.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local function dump(k,v,m) -- could be moved inline
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+        end
+    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sortedkeys(files)) do -- ipairs
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sortedkeys(fk)) do -- ipairs
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
+            end
+        end
+    else
+        for k, v in next, files do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in next, v do
+                    t[#t+1] = dump(kk,vv,"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,v,"\t")
+            end
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+function resolvers.save_data(dataname, makename) -- untested without cache overload
+    for cachename, files in next, instance[dataname] do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. ".lua", name .. ".luc"
+        if trace_verbose then
+            logs.report("fileio","preparing %s for %s",dataname,cachename)
+        end
+        for k, v in next, files do
+            if type(v) == "table" and #v == 1 then
+                files[k] = v[1]
+            end
+        end
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = resolvers.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local ok = io.savedata(luaname,resolvers.serialize(data))
+        if ok then
+            if trace_verbose then
+                logs.report("fileio","%s saved in %s",dataname,luaname)
+            end
+            if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip
+                if trace_verbose then
+                    logs.report("fileio","%s compiled to %s",dataname,lucname)
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname)
+                end
+                os.remove(lucname)
+            end
+        elseif trace_verbose then
+            logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname)
+        end
+    end
+end
+
+function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua")
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then
+            if trace_verbose then
+                logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = data.content
+        else
+            if trace_verbose then
+                logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
+        end
+    elseif trace_verbose then
+        logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+    end
+end
+
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
+
+function resolvers.resetconfig()
+    identify_own()
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function resolvers.loadnewconfig()
+    for _, cnf in ipairs(instance.luafiles) do
+        local pathname = file.dirname(cnf)
+        local filename = file.join(pathname,resolvers.luaname)
+        local blob = loadfile(filename)
+        if blob then
+            local data = blob()
+            if data then
+                if trace_verbose then
+                    logs.report("fileio","loading configuration file %s",filename)
+                end
+                if true then
+                    -- flatten to variable.progname
+                    local t = { }
+                    for k, v in next, data do -- v = progname
+                        if type(v) == "string" then
+                            t[k] = v
+                        else
+                            for kk, vv in next, v do -- vv = variable
+                                if type(vv) == "string" then
+                                    t[vv.."."..v] = kk
+                                end
+                            end
+                        end
+                    end
+                    instance['setup'][pathname] = t
+                else
+                    instance['setup'][pathname] = data
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","skipping configuration file %s",filename)
+                end
+                instance['setup'][pathname] = { }
+                instance.loaderror = true
+            end
+        elseif trace_verbose then
+            logs.report("fileio","skipping configuration file %s",filename)
+        end
+        instance.order[#instance.order+1] = instance.setup[pathname]
+        if instance.loaderror then break end
+    end
+end
+
+function resolvers.loadoldconfig()
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            resolvers.load_data(dname,'configuration')
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
+        end
+    end
+    resolvers.joinconfig()
+end
+
+function resolvers.expand_variables()
+    local expansions, environment, variables = { }, instance.environment, instance.variables
+    local env = resolvers.env
+    instance.expansions = expansions
+    if instance.engine   ~= "" then environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then environment['progname'] = instance.progname end
+    for k,v in next, environment do
+        local a, b = match(k,"^(%a+)%_(.*)%s*$")
+        if a and b then
+            expansions[a..'.'..b] = v
+        else
+            expansions[k] = v
+        end
+    end
+    for k,v in next, environment do -- move environment to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    for k,v in next, variables do -- move variables to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    local busy = false
+    local function resolve(a)
+        busy = true
+        return expansions[a] or env(a)
+    end
+    while true do
+        busy = false
+        for k,v in next, expansions do
+            local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve)
+            local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve)
+            if n > 0 or m > 0 then
+                expansions[k]= s
+            end
+        end
+        if not busy then break end
+    end
+    for k,v in next, expansions do
+        expansions[k] = gsub(v,"\\", '/')
+    end
+end
+
+function resolvers.variable(name)
+    return entry(instance.variables,name)
+end
+
+function resolvers.expansion(name)
+    return entry(instance.expansions,name)
+end
+
+function resolvers.is_variable(name)
+    return is_entry(instance.variables,name)
+end
+
+function resolvers.is_expansion(name)
+    return is_entry(instance.expansions,name)
+end
+
+function resolvers.unexpanded_path_list(str)
+    local pth = resolvers.variable(str)
+    local lst = resolvers.split_path(pth)
+    return expanded_path_from_list(lst)
+end
+
+function resolvers.unexpanded_path(str)
+    return file.join_path(resolvers.unexpanded_path_list(str))
+end
+
+do -- no longer needed
+
+    local done = { }
+
+    function resolvers.reset_extra_path()
+        local ep = instance.extra_paths
+        if not ep then
+            ep, done = { }, { }
+            instance.extra_paths = ep
+        elseif #ep > 0 then
+            instance.lists, done = { }, { }
+        end
+    end
+
+    function resolvers.register_extra_path(paths,subpaths)
+        local ep = instance.extra_paths or { }
+        local n = #ep
+        if paths and paths ~= "" then
+            if subpaths and subpaths ~= "" then
+                for p in gmatch(paths,"[^,]+") do
+                    -- we gmatch each step again, not that fast, but used seldom
+                    for s in gmatch(subpaths,"[^,]+") do
+                        local ps = p .. "/" .. s
+                        if not done[ps] then
+                            ep[#ep+1] = resolvers.clean_path(ps)
+                            done[ps] = true
+                        end
+                    end
+                end
+            else
+                for p in gmatch(paths,"[^,]+") do
+                    if not done[p] then
+                        ep[#ep+1] = resolvers.clean_path(p)
+                        done[p] = true
+                    end
+                end
+            end
+        elseif subpaths and subpaths ~= "" then
+            for i=1,n do
+                -- we gmatch each step again, not that fast, but used seldom
+                for s in gmatch(subpaths,"[^,]+") do
+                    local ps = ep[i] .. "/" .. s
+                    if not done[ps] then
+                        ep[#ep+1] = resolvers.clean_path(ps)
+                        done[ps] = true
+                    end
+                end
+            end
+        end
+        if #ep > 0 then
+            instance.extra_paths = ep -- register paths
+        end
+        if #ep > n then
+            instance.lists = { } -- erase the cache
+        end
+    end
+
+end
+
+local function made_list(instance,list)
+    local ep = instance.extra_paths
+    if not ep or #ep == 0 then
+        return list
+    else
+        local done, new = { }, { }
+        -- honour . .. ../.. but only when at the start
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                if find(v,"^[%.%/]$") then
+                    done[v] = true
+                    new[#new+1] = v
+                else
+                    break
+                end
+            end
+        end
+        -- first the extra paths
+        for k=1,#ep do
+            local v = ep[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        -- next the formal paths
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        return new
+    end
+end
+
+function resolvers.clean_path_list(str)
+    local t = resolvers.expanded_path_list(str)
+    if t then
+        for i=1,#t do
+            t[i] = file.collapse_path(resolvers.clean_path(t[i]))
+        end
+    end
+    return t
+end
+
+function resolvers.expand_path(str)
+    return file.join_path(resolvers.expanded_path_list(str))
+end
+
+function resolvers.expanded_path_list(str)
+    if not str then
+        return ep or { }
+    elseif instance.savelists then
+        -- engine+progname hash
+        str = gsub(str,"%$","")
+        if not instance.lists[str] then -- cached
+            local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str)))
+            instance.lists[str] = expanded_path_from_list(lst)
+        end
+        return instance.lists[str]
+    else
+        local lst = resolvers.split_path(resolvers.expansion(str))
+        return made_list(instance,expanded_path_from_list(lst))
+    end
+end
+
+function resolvers.expanded_path_list_from_var(str) -- brrr
+    local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$",""))
+    if tmp ~= "" then
+        return resolvers.expanded_path_list(str)
+    else
+        return resolvers.expanded_path_list(tmp)
+    end
+end
+
+function resolvers.expand_path_from_var(str)
+    return file.join_path(resolvers.expanded_path_list_from_var(str))
+end
+
+function resolvers.format_of_var(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+function resolvers.format_of_suffix(str)
+    return suffixmap[file.extname(str)] or 'tex'
+end
+
+function resolvers.variable_of_format(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+
+function resolvers.var_of_format_or_suffix(str)
+    local v = formats[str]
+    if v then
+        return v
+    end
+    v = formats[alternatives[str]]
+    if v then
+        return v
+    end
+    v = suffixmap[file.extname(str)]
+    if v then
+        return formats[isf]
+    end
+    return ''
+end
+
+function resolvers.expand_braces(str) -- output variable and brace expansion of STRING
+    local ori = resolvers.variable(str)
+    local pth = expanded_path_from_list(resolvers.split_path(ori))
+    return file.join_path(pth)
+end
+
+resolvers.isreadable = { }
+
+function resolvers.isreadable.file(name)
+    local readable = lfs.isfile(name) -- brrr
+    if trace_detail then
+        if readable then
+            logs.report("fileio","+ readable: %s",name)
+        else
+            logs.report("fileio","- readable: %s", name)
+        end
+    end
+    return readable
+end
+
+resolvers.isreadable.tex = resolvers.isreadable.file
+
+-- name
+-- name/name
+
+local function collect_files(names)
+    local filelist = { }
+    for k=1,#names do
+        local fname = names[k]
+        if trace_detail then
+            logs.report("fileio","? blobpath asked: %s",fname)
+        end
+        local bname = file.basename(fname)
+        local dname = file.dirname(fname)
+        if dname == "" or find(dname,"^%.") then
+            dname = false
+        else
+            dname = "/" .. dname .. "$"
+        end
+        local hashes = instance.hashes
+        for h=1,#hashes do
+            local hash = hashes[h]
+            local blobpath = hash.tag
+            local files = blobpath and instance.files[blobpath]
+            if files then
+                if trace_detail then
+                    logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname)
+                end
+                local blobfile = files[bname]
+                if not blobfile then
+                    local rname = "remap:"..bname
+                    blobfile = files[rname]
+                    if blobfile then
+                        bname = files[rname]
+                        blobfile = files[bname]
+                    end
+                end
+                if blobfile then
+                    if type(blobfile) == 'string' then
+                        if not dname or find(blobfile,dname) then
+                            filelist[#filelist+1] = {
+                                hash.type,
+                                file.join(blobpath,blobfile,bname), -- search
+                                resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result
+                            }
+                        end
+                    else
+                        for kk=1,#blobfile do
+                            local vv = blobfile[kk]
+                            if not dname or find(vv,dname) then
+                                filelist[#filelist+1] = {
+                                    hash.type,
+                                    file.join(blobpath,vv,bname), -- search
+                                    resolvers.concatinators[hash.type](blobpath,vv,bname) -- result
+                                }
+                            end
+                        end
+                    end
+                end
+            elseif trace_locating then
+                logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname)
+            end
+        end
+    end
+    if #filelist > 0 then
+        return filelist
+    else
+        return nil
+    end
+end
+
+function resolvers.suffix_of_format(str)
+    if suffixes[str] then
+        return suffixes[str][1]
+    else
+        return ""
+    end
+end
+
+function resolvers.suffixes_of_format(str)
+    if suffixes[str] then
+        return suffixes[str]
+    else
+        return {}
+    end
+end
+
+function resolvers.register_in_trees(name)
+    if not find(name,"^%.") then
+        instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one
+    end
+end
+
+-- split the next one up for readability (bu this module needs a cleanup anyway)
+
+local function can_be_dir(name) -- can become local
+    local fakepaths = instance.fakepaths
+    if not fakepaths[name] then
+        if lfs.isdir(name) then
+            fakepaths[name] = 1 -- directory
+        else
+            fakepaths[name] = 2 -- no directory
+        end
+    end
+    return (fakepaths[name] == 1)
+end
+
+local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc)
+    local result = collected or { }
+    local stamp  = nil
+    filename = file.collapse_path(filename)  -- elsewhere
+    filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere
+    -- speed up / beware: format problem
+    if instance.remember then
+        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format
+        if instance.found[stamp] then
+            if trace_locating then
+                logs.report("fileio",'! remembered: %s',filename)
+            end
+            return instance.found[stamp]
+        end
+    end
+    if not dangerous[instance.format or "?"] then
+        if resolvers.isreadable.file(filename) then
+            if trace_detail then
+                logs.report("fileio",'= found directly: %s',filename)
+            end
+            instance.found[stamp] = { filename }
+            return { filename }
+        end
+    end
+    if find(filename,'%*') then
+        if trace_locating then
+            logs.report("fileio",'! wildcard: %s', filename)
+        end
+        result = resolvers.find_wildcard_files(filename)
+    elseif file.is_qualified_path(filename) then
+        if resolvers.isreadable.file(filename) then
+            if trace_locating then
+                logs.report("fileio",'! qualified: %s', filename)
+            end
+            result = { filename }
+        else
+            local forcedname, ok, suffix = "", false, file.extname(filename)
+            if suffix == "" then -- why
+                if instance.format == "" then
+                    forcedname = filename .. ".tex"
+                    if resolvers.isreadable.file(forcedname) then
+                        if trace_locating then
+                            logs.report("fileio",'! no suffix, forcing standard filetype: tex')
+                        end
+                        result, ok = { forcedname }, true
+                    end
+                else
+                    local suffixes = resolvers.suffixes_of_format(instance.format)
+                    for _, s in next, suffixes do
+                        forcedname = filename .. "." .. s
+                        if resolvers.isreadable.file(forcedname) then
+                            if trace_locating then
+                                logs.report("fileio",'! no suffix, forcing format filetype: %s', s)
+                            end
+                            result, ok = { forcedname }, true
+                            break
+                        end
+                    end
+                end
+            end
+            if not ok and suffix ~= "" then
+                -- try to find in tree (no suffix manipulation), here we search for the
+                -- matching last part of the name
+                local basename = file.basename(filename)
+                local pattern = (filename .. "$"):gsub("([%.%-])","%%%1")
+                local savedformat = instance.format
+                local format = savedformat or ""
+                if format == "" then
+                    instance.format = resolvers.format_of_suffix(suffix)
+                end
+                if not format then
+                    instance.format = "othertextfiles" -- kind of everything, maybe texinput is better
+                end
+                --
+                local resolved = collect_instance_files(basename)
+                if #result == 0 then
+                    local lowered = lower(basename)
+                    if filename ~= lowered then
+                        resolved = collect_instance_files(lowered)
+                    end
+                end
+                resolvers.format = savedformat
+                --
+                for r=1,#resolved do
+                    local rr = resolved[r]
+                    if rr:find(pattern) then
+                        result[#result+1], ok = rr, true
+                    end
+                end
+                -- a real wildcard:
+                --
+                -- if not ok then
+                --     local filelist = collect_files({basename})
+                --     for f=1,#filelist do
+                --         local ff = filelist[f][3] or ""
+                --         if ff:find(pattern) then
+                --             result[#result+1], ok = ff, true
+                --         end
+                --     end
+                -- end
+            end
+            if not ok and trace_locating then
+                logs.report("fileio",'? qualified: %s', filename)
+            end
+        end
+    else
+        -- search spec
+        local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename)
+        if ext == "" then
+            if not instance.force_suffixes then
+                wantedfiles[#wantedfiles+1] = filename
+            end
+        else
+            wantedfiles[#wantedfiles+1] = filename
+        end
+        if instance.format == "" then
+            if ext == "" then
+                local forcedname = filename .. '.tex'
+                wantedfiles[#wantedfiles+1] = forcedname
+                filetype = resolvers.format_of_suffix(forcedname)
+                if trace_locating then
+                    logs.report("fileio",'! forcing filetype: %s',filetype)
+                end
+            else
+                filetype = resolvers.format_of_suffix(filename)
+                if trace_locating then
+                    logs.report("fileio",'! using suffix based filetype: %s',filetype)
+                end
+            end
+        else
+            if ext == "" then
+                local suffixes = resolvers.suffixes_of_format(instance.format)
+                for _, s in next, suffixes do
+                    wantedfiles[#wantedfiles+1] = filename .. "." .. s
+                end
+            end
+            filetype = instance.format
+            if trace_locating then
+                logs.report("fileio",'! using given filetype: %s',filetype)
+            end
+        end
+        local typespec = resolvers.variable_of_format(filetype)
+        local pathlist = resolvers.expanded_path_list(typespec)
+        if not pathlist or #pathlist == 0 then
+            -- no pathlist, access check only / todo == wildcard
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+                logs.report("fileio",'? filetype: %s',filetype or '?')
+                logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | "))
+            end
+            for k=1,#wantedfiles do
+                local fname = wantedfiles[k]
+                if fname and resolvers.isreadable.file(fname) then
+                    filename, done = fname, true
+                    result[#result+1] = file.join('.',fname)
+                    break
+                end
+            end
+            -- this is actually 'other text files' or 'any' or 'whatever'
+            local filelist = collect_files(wantedfiles)
+            local fl = filelist and filelist[1]
+            if fl then
+                filename = fl[3]
+                result[#result+1] = filename
+                done = true
+            end
+        else
+            -- list search
+            local filelist = collect_files(wantedfiles)
+            local doscan, recurse
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+            end
+            -- a bit messy ... esp the doscan setting here
+            for k=1,#pathlist do
+                local path = pathlist[k]
+                if find(path,"^!!") then doscan  = false else doscan  = true  end
+                if find(path,"//$") then recurse = true  else recurse = false end
+                local pathname = gsub(path,"^!+", '')
+                done = false
+                -- using file list
+                if filelist and not (done and not instance.allresults) and recurse then
+                    -- compare list entries with permitted pattern
+                    pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences
+                    pathname = gsub(pathname,"/+$", '/.*')      -- later usage of pathname
+                    pathname = gsub(pathname,"//", '/.-/')      -- not ok for /// but harmless
+                    local expr = "^" .. pathname
+                    for k=1,#filelist do
+                        local fl = filelist[k]
+                        local f = fl[2]
+                        if find(f,expr) then
+                            if trace_detail then
+                                logs.report("fileio",'= found in hash: %s',f)
+                            end
+                            --- todo, test for readable
+                            result[#result+1] = fl[3]
+                            resolvers.register_in_trees(f) -- for tracing used files
+                            done = true
+                            if not instance.allresults then break end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- check if on disk / unchecked / does not work at all / also zips
+                    if resolvers.splitmethod(pathname).scheme == 'file' then -- ?
+                        local pname = gsub(pathname,"%.%*$",'')
+                        if not find(pname,"%*") then
+                            local ppname = gsub(pname,"/+$","")
+                            if can_be_dir(ppname) then
+                                for k=1,#wantedfiles do
+                                    local w = wantedfiles[k]
+                                    local fname = file.join(ppname,w)
+                                    if resolvers.isreadable.file(fname) then
+                                        if trace_detail then
+                                            logs.report("fileio",'= found by scanning: %s',fname)
+                                        end
+                                        result[#result+1] = fname
+                                        done = true
+                                        if not instance.allresults then break end
+                                    end
+                                end
+                            else
+                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
+                            end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- todo: slow path scanning
+                end
+                if done and not instance.allresults then break end
+            end
+        end
+    end
+    for k=1,#result do
+        result[k] = file.collapse_path(result[k])
+    end
+    if instance.remember then
+        instance.found[stamp] = result
+    end
+    return result
+end
+
+if not resolvers.concatinators  then resolvers.concatinators = { } end
+
+resolvers.concatinators.tex  = file.join
+resolvers.concatinators.file = resolvers.concatinators.tex
+
+function resolvers.find_files(filename,filetype,mustexist)
+    if type(mustexist) == boolean then
+        -- all set
+    elseif type(filetype) == 'boolean' then
+        filetype, mustexist = nil, false
+    elseif type(filetype) ~= 'string' then
+        filetype, mustexist = nil, false
+    end
+    instance.format = filetype or ''
+    local result = collect_instance_files(filename)
+    if #result == 0 then
+        local lowered = lower(filename)
+        if filename ~= lowered then
+            return collect_instance_files(lowered)
+        end
+    end
+    instance.format = ''
+    return result
+end
+
+function resolvers.find_file(filename,filetype,mustexist)
+    return (resolvers.find_files(filename,filetype,mustexist)[1] or "")
+end
+
+function resolvers.find_given_files(filename)
+    local bname, result = file.basename(filename), { }
+    local hashes = instance.hashes
+    for k=1,#hashes do
+        local hash = hashes[k]
+        local files = instance.files[hash.tag]
+        local blist = files[bname]
+        if not blist then
+            local rname = "remap:"..bname
+            blist = files[rname]
+            if blist then
+                bname = files[rname]
+                blist = files[bname]
+            end
+        end
+        if blist then
+            if type(blist) == 'string' then
+                result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or ""
+                if not instance.allresults then break end
+            else
+                for kk=1,#blist do
+                    local vv = blist[kk]
+                    result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or ""
+                    if not instance.allresults then break end
+                end
+            end
+        end
+    end
+    return result
+end
+
+function resolvers.find_given_file(filename)
+    return (resolvers.find_given_files(filename)[1] or "")
+end
+
+local function doit(path,blist,bname,tag,kind,result,allresults)
+    local done = false
+    if blist and kind then
+        if type(blist) == 'string' then
+            -- make function and share code
+            if find(lower(blist),path) then
+                result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or ""
+                done = true
+            end
+        else
+            for kk=1,#blist do
+                local vv = blist[kk]
+                if find(lower(vv),path) then
+                    result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or ""
+                    done = true
+                    if not allresults then break end
+                end
+            end
+        end
+    end
+    return done
+end
+
+function resolvers.find_wildcard_files(filename) -- todo: remap:
+    local result = { }
+    local bname, dname = file.basename(filename), file.dirname(filename)
+    local path = gsub(dname,"^*/","")
+    path = gsub(path,"*",".*")
+    path = gsub(path,"-","%%-")
+    if dname == "" then
+        path = ".*"
+    end
+    local name = bname
+    name = gsub(name,"*",".*")
+    name = gsub(name,"-","%%-")
+    path = lower(path)
+    name = lower(name)
+    local files, allresults, done = instance.files, instance.allresults, false
+    if find(name,"%*") then
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            for kk, hh in next, files[hash.tag] do
+                if not find(kk,"^remap:") then
+                    if find(lower(kk),name) then
+                        if doit(path,hh,kk,tag,kind,result,allresults) then done = true end
+                        if done and not allresults then break end
+                    end
+                end
+            end
+        end
+    else
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end
+            if done and not allresults then break end
+        end
+    end
+    -- we can consider also searching the paths not in the database, but then
+    -- we end up with a messy search (all // in all path specs)
+    return result
+end
+
+function resolvers.find_wildcard_file(filename)
+    return (resolvers.find_wildcard_files(filename)[1] or "")
+end
+
+-- main user functions
+
+function resolvers.automount()
+    -- implemented later
+end
+
+function resolvers.load(option)
+    statistics.starttiming(instance)
+    resolvers.resetconfig()
+    resolvers.identify_cnf()
+    resolvers.load_lua()
+    resolvers.expand_variables()
+    resolvers.load_cnf()
+    resolvers.expand_variables()
+    if option ~= "nofiles" then
+        resolvers.load_hash()
+        resolvers.automount()
+    end
+    statistics.stoptiming(instance)
+end
+
+function resolvers.for_files(command, files, filetype, mustexist)
+    if files and #files > 0 then
+        local function report(str)
+            if trace_verbose then
+                logs.report("fileio",str) -- has already verbose
+            else
+                print(str)
+            end
+        end
+        if trace_verbose then
+            report('')
+        end
+        for _, file in ipairs(files) do
+            local result = command(file,filetype,mustexist)
+            if type(result) == 'string' then
+                report(result)
+            else
+                for _,v in ipairs(result) do
+                    report(v)
+                end
+            end
+        end
+    end
+end
+
+-- strtab
+
+resolvers.var_value  = resolvers.variable   -- output the value of variable $STRING.
+resolvers.expand_var = resolvers.expansion  -- output variable expansion of STRING.
+
+function resolvers.show_path(str)     -- output search path for file type NAME
+    return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str)))
+end
+
+-- resolvers.find_file(filename)
+-- resolvers.find_file(filename, filetype, mustexist)
+-- resolvers.find_file(filename, mustexist)
+-- resolvers.find_file(filename, filetype)
+
+function resolvers.register_file(files, name, path)
+    if files[name] then
+        if type(files[name]) == 'string' then
+            files[name] = { files[name], path }
+        else
+            files[name] = path
+        end
+    else
+        files[name] = path
+    end
+end
+
+function resolvers.splitmethod(filename)
+    if not filename then
+        return { } -- safeguard
+    elseif type(filename) == "table" then
+        return filename -- already split
+    elseif not find(filename,"://") then
+        return { scheme="file", path = filename, original=filename } -- quick hack
+    else
+        return url.hashed(filename)
+    end
+end
+
+function table.sequenced(t,sep) -- temp here
+    local s = { }
+    for k, v in pairs(t) do -- pairs?
+        s[#s+1] = k .. "=" .. v
+    end
+    return concat(s, sep or " | ")
+end
+
+function resolvers.methodhandler(what, filename, filetype) -- ...
+    local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb
+    local scheme = specification.scheme
+    if resolvers[what][scheme] then
+        if trace_locating then
+            logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification))
+        end
+        return resolvers[what][scheme](filename,filetype) -- todo: specification
+    else
+        return resolvers[what].tex(filename,filetype) -- todo: specification
+    end
+end
+
+function resolvers.clean_path(str)
+    if str then
+        str = gsub(str,"\\","/")
+        str = gsub(str,"^!+","")
+        str = gsub(str,"^~",resolvers.homedir)
+        return str
+    else
+        return nil
+    end
+end
+
+function resolvers.do_with_path(name,func)
+    for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs?
+        func("^"..resolvers.clean_path(v))
+    end
+end
+
+function resolvers.do_with_var(name,func)
+    func(expanded_var(name))
+end
+
+function resolvers.with_files(pattern,handle)
+    for _, hash in ipairs(instance.hashes) do
+        local blobpath = hash.tag
+        local blobtype = hash.type
+        if blobpath then
+            local files = instance.files[blobpath]
+            if files then
+                for k,v in next, files do
+                    if find(k,"^remap:") then
+                        k = files[k]
+                        v = files[k] -- chained
+                    end
+                    if find(k,pattern) then
+                        if type(v) == "string" then
+                            handle(blobtype,blobpath,v,k)
+                        else
+                            for _,vv in pairs(v) do -- ipairs?
+                                handle(blobtype,blobpath,vv,k)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+function resolvers.locate_format(name)
+    local barename, fmtname = name:gsub("%.%a+$",""), ""
+    if resolvers.usecache then
+        local path = file.join(caches.setpath("formats")) -- maybe platform
+        fmtname = file.join(path,barename..".fmt") or ""
+    end
+    if fmtname == "" then
+        fmtname = resolvers.find_files(barename..".fmt")[1] or ""
+    end
+    fmtname = resolvers.clean_path(fmtname)
+    if fmtname ~= "" then
+        local barename = file.removesuffix(fmtname)
+        local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui"
+        if lfs.isfile(luiname) then
+            return barename, luiname
+        elseif lfs.isfile(lucname) then
+            return barename, lucname
+        elseif lfs.isfile(luaname) then
+            return barename, luaname
+        end
+    end
+    return nil, nil
+end
+
+function resolvers.boolean_variable(str,default)
+    local b = resolvers.expansion(str)
+    if b == "" then
+        return default
+    else
+        b = toboolean(b)
+        return (b == nil and default) or b
+    end
+end
+
+texconfig.kpse_init = false
+
+kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } )
+
+-- for a while
+
+input = resolvers
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This module deals with caching data. It sets up the paths and
+implements loaders and savers for tables. Best is to set the
+following variable. When not set, the usual paths will be
+checked. Personally I prefer the (users) temporary path.

+
+
+TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
+
+
+
Currently we do no locking when we write files. This is no real
+problem because most caching involves fonts and the chance of them
+being written at the same time is small. We also need to extend
+luatools with a recache feature.

+--ldx]]--
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache = false  trackers.register("resolvers.cache", function(v) trace_cache = v end)
+
+caches = caches or { }
+
+caches.path     = caches.path or nil
+caches.base     = caches.base or "luatex-cache"
+caches.more     = caches.more or "context"
+caches.direct   = false -- true is faster but may need huge amounts of memory
+caches.tree     = false
+caches.paths    = caches.paths or nil
+caches.force    = false
+caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" }
+
+function caches.cleanname(name)
+    return (gsub(lower(name),"[^%w%d]+","-"))
+end
+
+function caches.temp()
+    local cachepath = nil
+    local function check(list,isenv)
+        if not cachepath then
+            for k=1,#list do
+                local v = list[k]
+                cachepath = (isenv and (os.env[v] or "")) or v or ""
+                if cachepath == "" then
+                    -- next
+                else
+                    cachepath = resolvers.clean_path(cachepath)
+                    if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory"
+                        break
+                    elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
+                        dir.mkdirs(cachepath)
+                        if lfs.isdir(cachepath) and file.iswritable(cachepath) then
+                            break
+                        end
+                    end
+                end
+                cachepath = nil
+            end
+        end
+    end
+    check(resolvers.clean_path_list("TEXMFCACHE") or { })
+    check(caches.defaults,true)
+    if not cachepath then
+        print("\nfatal error: there is no valid (writable) cache path defined\n")
+        os.exit()
+    elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory"
+        print(format("\nfatal error: cache path %s is not a directory\n",cachepath))
+        os.exit()
+    end
+    cachepath = file.collapse_path(cachepath)
+    function caches.temp()
+        return cachepath
+    end
+    return cachepath
+end
+
+function caches.configpath()
+    return table.concat(resolvers.instance.cnffiles,";")
+end
+
+function caches.hashed(tree)
+    return md5.hex(gsub(lower(tree),"[\\\/]+","/"))
+end
+
+function caches.treehash()
+    local tree = caches.configpath()
+    if not tree or tree == "" then
+        return false
+    else
+        return caches.hashed(tree)
+    end
+end
+
+function caches.setpath(...)
+    if not caches.path then
+        if not caches.path then
+            caches.path = caches.temp()
+        end
+        caches.path = resolvers.clean_path(caches.path) -- to be sure
+        caches.tree = caches.tree or caches.treehash()
+        if caches.tree then
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
+        else
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
+        end
+    end
+    if not caches.path then
+        caches.path = '.'
+    end
+    caches.path = resolvers.clean_path(caches.path)
+    if not table.is_empty({...}) then
+        local pth = dir.mkdirs(caches.path,...)
+        return pth
+    end
+    caches.path = dir.expand_name(caches.path)
+    return caches.path
+end
+
+function caches.definepath(category,subcategory)
+    return function()
+        return caches.setpath(category,subcategory)
+    end
+end
+
+function caches.setluanames(path,name)
+    return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
+end
+
+function caches.loaddata(path,name)
+    local tmaname, tmcname = caches.setluanames(path,name)
+    local loader = loadfile(tmcname) or loadfile(tmaname)
+    if loader then
+        return loader()
+    else
+        return false
+    end
+end
+
+--~ function caches.loaddata(path,name)
+--~     local tmaname, tmcname = caches.setluanames(path,name)
+--~     return dofile(tmcname) or dofile(tmaname)
+--~ end
+
+function caches.iswritable(filepath,filename)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    return file.iswritable(tmaname)
+end
+
+function caches.savedata(filepath,filename,data,raw)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    local reduce, simplify = true, true
+    if raw then
+        reduce, simplify = false, false
+    end
+    if caches.direct then
+        file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex
+    else
+        table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true
+    end
+    local cleanup = resolvers.boolean_variable("PURGECACHE", false)
+    local strip = resolvers.boolean_variable("LUACSTRIP", true)
+    utils.lua.compile(tmaname, tmcname, cleanup, strip)
+end
+
+-- here we use the cache for format loading (texconfig.[formatname|jobname])
+
+--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then
+if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then
+    if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc
+    texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt")
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+resolvers.finders = resolvers.finders or { }
+resolvers.openers = resolvers.openers or { }
+resolvers.loaders = resolvers.loaders or { }
+
+resolvers.finders.notfound  = { nil }
+resolvers.openers.notfound  = { nil }
+resolvers.loaders.notfound  = { false, nil, 0 }
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-out'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+outputs = outputs or { }
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-con'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache      = false  trackers.register("resolvers.cache",      function(v) trace_cache      = v end)
+local trace_containers = false  trackers.register("resolvers.containers", function(v) trace_containers = v end)
+local trace_storage    = false  trackers.register("resolvers.storage",    function(v) trace_storage    = v end)
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+--[[ldx--
+
Once we found ourselves defining similar cache constructs
+several times, containers were introduced. Containers are used
+to collect tables in memory and reuse them when possible based
+on (unique) hashes (to be provided by the calling function).

+
+
Caching to disk is disabled by default. Version numbers are
+stored in the saved table which makes it possible to change the
+table structures without bothering about the disk cache.

+
+
Examples of usage can be found in the font related code.

+--ldx]]--
+
+containers = containers or { }
+
+containers.usecache = true
+
+local function report(container,tag,name)
+    if trace_cache or trace_containers then
+        logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid')
+    end
+end
+
+local allocated = { }
+
+-- tracing
+
+function containers.define(category, subcategory, version, enabled)
+    return function()
+        if category and subcategory then
+            local c = allocated[category]
+            if not c then
+                c  = { }
+                allocated[category] = c
+            end
+            local s = c[subcategory]
+            if not s then
+                s = {
+                    category = category,
+                    subcategory = subcategory,
+                    storage = { },
+                    enabled = enabled,
+                    version = version or 1.000,
+                    trace = false,
+                    path = caches and caches.setpath(category,subcategory),
+                }
+                c[subcategory] = s
+            end
+            return s
+        else
+            return nil
+        end
+    end
+end
+
+function containers.is_usable(container, name)
+    return container.enabled and caches and caches.iswritable(container.path, name)
+end
+
+function containers.is_valid(container, name)
+    if name and name ~= "" then
+        local storage = container.storage[name]
+        return storage and not table.is_empty(storage) and storage.cache_version == container.version
+    else
+        return false
+    end
+end
+
+function containers.read(container,name)
+    if container.enabled and caches and not container.storage[name] and containers.usecache then
+        container.storage[name] = caches.loaddata(container.path,name)
+        if containers.is_valid(container,name) then
+            report(container,"loaded",name)
+        else
+            container.storage[name] = nil
+        end
+    end
+    if container.storage[name] then
+        report(container,"reusing",name)
+    end
+    return container.storage[name]
+end
+
+function containers.write(container, name, data)
+    if data then
+        data.cache_version = container.version
+        if container.enabled and caches then
+            local unique, shared = data.unique, data.shared
+            data.unique, data.shared = nil, nil
+            caches.savedata(container.path, name, data)
+            report(container,"saved",name)
+            data.unique, data.shared = unique, shared
+        end
+        report(container,"stored",name)
+        container.storage[name] = data
+    end
+    return data
+end
+
+function containers.content(container,name)
+    return container.storage[name]
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-use'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+-- since we want to use the cache instead of the tree, we will now
+-- reimplement the saver.
+
+local save_data = resolvers.save_data
+local load_data = resolvers.load_data
+
+resolvers.cachepath = nil  -- public, for tracing
+resolvers.usecache  = true -- public, for tracing
+
+function resolvers.save_data(dataname)
+    save_data(dataname, function(cachename,dataname)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(cachename))
+        else
+            return file.join(cachename,dataname)
+        end
+    end)
+end
+
+function resolvers.load_data(pathname,dataname,filename)
+    load_data(pathname,dataname,filename,function(dataname,filename)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(pathname))
+        else
+            if not filename or (filename == "") then
+                filename = dataname
+            end
+            return file.join(pathname,filename)
+        end
+    end)
+end
+
+-- we will make a better format, maybe something xml or just text or lua
+
+resolvers.automounted = resolvers.automounted or { }
+
+function resolvers.automount(usecache)
+    local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT'))
+    if table.is_empty(mountpaths) and usecache then
+        mountpaths = { caches.setpath("mount") }
+    end
+    if not table.is_empty(mountpaths) then
+        statistics.starttiming(resolvers.instance)
+        for k, root in pairs(mountpaths) do
+            local f = io.open(root.."/url.tmi")
+            if f then
+                for line in f:lines() do
+                    if line then
+                        if line:find("^[%%#%-]") then -- or %W
+                            -- skip
+                        elseif line:find("^zip://") then
+                            if trace_locating then
+                                logs.report("fileio","mounting %s",line)
+                            end
+                            table.insert(resolvers.automounted,line)
+                            resolvers.usezipfile(line)
+                        end
+                    end
+                end
+                f:close()
+            end
+        end
+        statistics.stoptiming(resolvers.instance)
+    end
+end
+
+-- status info
+
+statistics.register("used config path", function() return caches.configpath()  end)
+statistics.register("used cache path",  function() return caches.temp() or "?" end)
+
+-- experiment (code will move)
+
+function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname
+    local enginebanner = status.list().banner
+    if formatbanner and enginebanner and sourcefile then
+        local luvname = file.replacesuffix(texname,"luv")
+        local luvdata = {
+            enginebanner = enginebanner,
+            formatbanner = formatbanner,
+            sourcehash   = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"),
+            sourcefile   = sourcefile,
+        }
+        io.savedata(luvname,table.serialize(luvdata,true))
+    end
+end
+
+function statistics.check_fmt_status(texname)
+    local enginebanner = status.list().banner
+    if enginebanner and texname then
+        local luvname = file.replacesuffix(texname,"luv")
+        if lfs.isfile(luvname) then
+            local luv = dofile(luvname)
+            if luv and luv.sourcefile then
+                local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown")
+                if luv.enginebanner and luv.enginebanner ~= enginebanner then
+                    return "engine mismatch"
+                end
+                if luv.sourcehash and luv.sourcehash ~= sourcehash then
+                    return "source mismatch"
+                end
+            else
+                return "invalid status file"
+            end
+        else
+            return "missing status file"
+        end
+    end
+    return true
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-kps'] = {
+    version   = 1.001,
+    comment   = "companion to luatools.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This file is used when we want the input handlers to behave like
+kpsewhich. What to do with the following:

+
+
+{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}
+$SELFAUTOLOC    : /usr/tex/bin/platform
+$SELFAUTODIR    : /usr/tex/bin
+$SELFAUTOPARENT : /usr/tex
+
+
+
How about just forgetting about them?

+--ldx]]--
+
+local suffixes = resolvers.suffixes
+local formats  = resolvers.formats
+
+suffixes['gf']                       = { 'gf' }
+suffixes['pk']                       = { 'pk' }
+suffixes['base']                     = { 'base' }
+suffixes['bib']                      = { 'bib' }
+suffixes['bst']                      = { 'bst' }
+suffixes['cnf']                      = { 'cnf' }
+suffixes['mem']                      = { 'mem' }
+suffixes['mf']                       = { 'mf' }
+suffixes['mfpool']                   = { 'pool' }
+suffixes['mft']                      = { 'mft' }
+suffixes['mppool']                   = { 'pool' }
+suffixes['graphic/figure']           = { 'eps', 'epsi' }
+suffixes['texpool']                  = { 'pool' }
+suffixes['PostScript header']        = { 'pro' }
+suffixes['ist']                      = { 'ist' }
+suffixes['web']                      = { 'web', 'ch' }
+suffixes['cweb']                     = { 'w', 'web', 'ch' }
+suffixes['cmap files']               = { 'cmap' }
+suffixes['lig files']                = { 'lig' }
+suffixes['bitmap font']              = { }
+suffixes['MetaPost support']         = { }
+suffixes['TeX system documentation'] = { }
+suffixes['TeX system sources']       = { }
+suffixes['dvips config']             = { }
+suffixes['type42 fonts']             = { }
+suffixes['web2c files']              = { }
+suffixes['other text files']         = { }
+suffixes['other binary files']       = { }
+suffixes['opentype fonts']           = { 'otf' }
+
+suffixes['fmt']                      = { 'fmt' }
+suffixes['texmfscripts']             = { 'rb','lua','py','pl' }
+
+suffixes['pdftex config']            = { }
+suffixes['Troff fonts']              = { }
+
+suffixes['ls-R']                     = { }
+
+--[[ldx--
+
If you wondered abou tsome of the previous mappings, how about
+the next bunch:

+--ldx]]--
+
+formats['bib']                      = ''
+formats['bst']                      = ''
+formats['mft']                      = ''
+formats['ist']                      = ''
+formats['web']                      = ''
+formats['cweb']                     = ''
+formats['MetaPost support']         = ''
+formats['TeX system documentation'] = ''
+formats['TeX system sources']       = ''
+formats['Troff fonts']              = ''
+formats['dvips config']             = ''
+formats['graphic/figure']           = ''
+formats['ls-R']                     = ''
+formats['other text files']         = ''
+formats['other binary files']       = ''
+
+formats['gf']                       = ''
+formats['pk']                       = ''
+formats['base']                     = 'MFBASES'
+formats['cnf']                      = ''
+formats['mem']                      = 'MPMEMS'
+formats['mf']                       = 'MFINPUTS'
+formats['mfpool']                   = 'MFPOOL'
+formats['mppool']                   = 'MPPOOL'
+formats['texpool']                  = 'TEXPOOL'
+formats['PostScript header']        = 'TEXPSHEADERS'
+formats['cmap files']               = 'CMAPFONTS'
+formats['type42 fonts']             = 'T42FONTS'
+formats['web2c files']              = 'WEB2C'
+formats['pdftex config']            = 'PDFTEXCONFIG'
+formats['texmfscripts']             = 'TEXMFSCRIPTS'
+formats['bitmap font']              = ''
+formats['lig files']                = 'LIGFONTS'
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-aux'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+local trace_verbose = false  trackers.register("resolvers.verbose", function(v) trace_verbose = v end)
+
+function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix
+    local scriptpath = "scripts/context/lua"
+    newname = file.addsuffix(newname,"lua")
+    local oldscript = resolvers.clean_path(oldname)
+    if trace_verbose then
+        logs.report("fileio","to be replaced old script %s", oldscript)
+    end
+    local newscripts = resolvers.find_files(newname) or { }
+    if #newscripts == 0 then
+        if trace_verbose then
+            logs.report("fileio","unable to locate new script")
+        end
+    else
+        for i=1,#newscripts do
+            local newscript = resolvers.clean_path(newscripts[i])
+            if trace_verbose then
+                logs.report("fileio","checking new script %s", newscript)
+            end
+            if oldscript == newscript then
+                if trace_verbose then
+                    logs.report("fileio","old and new script are the same")
+                end
+            elseif not find(newscript,scriptpath) then
+                if trace_verbose then
+                    logs.report("fileio","new script should come from %s",scriptpath)
+                end
+            elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then
+                if trace_verbose then
+                    logs.report("fileio","invalid new script name")
+                end
+            else
+                local newdata = io.loaddata(newscript)
+                if newdata then
+                    if trace_verbose then
+                        logs.report("fileio","old script content replaced by new content")
+                    end
+                    io.savedata(oldscript,newdata)
+                    break
+                elseif trace_verbose then
+                    logs.report("fileio","unable to load new script")
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-lst'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- used in mtxrun
+
+local find, concat, upper, format = string.find, table.concat, string.upper, string.format
+
+resolvers.listers = resolvers.listers or { }
+
+local function tabstr(str)
+    if type(str) == 'table' then
+        return concat(str," | ")
+    else
+        return str
+    end
+end
+
+local function list(list,report)
+    local instance = resolvers.instance
+    local pat = upper(pattern or "","")
+    local report = report or texio.write_nl
+    for _,key in pairs(table.sortedkeys(list)) do
+        if instance.pattern == "" or find(upper(key),pat) then
+            if instance.kpseonly then
+                if instance.kpsevars[key] then
+                    report(format("%s=%s",key,tabstr(list[key])))
+                end
+            else
+                report(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key])))
+            end
+        end
+    end
+end
+
+function resolvers.listers.variables () list(resolvers.instance.variables ) end
+function resolvers.listers.expansions() list(resolvers.instance.expansions) end
+
+function resolvers.listers.configurations(report)
+    local report = report or texio.write_nl
+    local instance = resolvers.instance
+    for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do
+        if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then
+            report(format("%s\n",key))
+            for i,c in ipairs(instance.order) do
+                local str = c[key]
+                if str then
+                    report(format("\t%s\t%s",i,str))
+                end
+            end
+            report("")
+        end
+    end
+end
+
+
+end -- of closure
+-- end library merge
+
+-- We initialize some characteristics of this program. We need to
+-- do this before we load the libraries, else own.name will not be
+-- properly set (handy for selfcleaning the file). It's an ugly
+-- looking piece of code.
+
+own = { }
+
+own.libs = { -- todo: check which ones are really needed
+    'l-string.lua',
+    'l-lpeg.lua',
+    'l-table.lua',
+    'l-io.lua',
+    'l-number.lua',
+    'l-set.lua',
+    'l-os.lua',
+    'l-file.lua',
+    'l-md5.lua',
+    'l-url.lua',
+    'l-dir.lua',
+    'l-boolean.lua',
+    'l-unicode.lua',
+    'l-math.lua',
+    'l-utils.lua',
+    'trac-tra.lua',
+    'luat-env.lua',
+    'trac-inf.lua',
+    'trac-log.lua',
+    'data-res.lua',
+    'data-tmp.lua',
+--  'data-pre.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-tex.lua',
+--  'data-bin.lua',
+--  'data-zip.lua',
+--  'data-crl.lua',
+--  'data-lua.lua',
+    'data-kps.lua', -- so that we can replace kpsewhich
+    'data-aux.lua', -- updater
+    'data-lst.lua', -- lister
+}
+
+-- We need this hack till luatex is fixed.
+
+if arg and arg[0] == 'luatex' and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+-- End of hack.
+
+own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua'
+own.path = string.match(own.name,"^(.+)[\\/].-$") or "."
+own.list = { '.' }
+
+if own.path ~= '.' then
+    table.insert(own.list,own.path)
+end
+
+table.insert(own.list,own.path.."/../../../tex/context/base")
+table.insert(own.list,own.path.."/mtx")
+table.insert(own.list,own.path.."/../sources")
+
+function locate_libs()
+    for _, lib in pairs(own.libs) do
+        for _, pth in pairs(own.list) do
+            local filename = string.gsub(pth .. "/" .. lib,"\\","/")
+            local codeblob = loadfile(filename)
+            if codeblob then
+                codeblob()
+                own.list = { pth } -- speed up te search
+                break
+            end
+        end
+    end
+end
+
+if not resolvers then
+    locate_libs()
+end
+
+if not resolvers then
+    print("")
+    print("Luatools is unable to start up due to lack of libraries. You may")
+    print("try to run 'lua luatools.lua --selfmerge' in the path where this")
+    print("script is located (normally under ..../scripts/context/lua) which")
+    print("will make luatools library independent.")
+    os.exit()
+end
+
+logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false)
+
+local instance = resolvers.reset()
+
+resolvers.defaultlibs = { -- not all are needed
+    'l-string.lua',
+    'l-lpeg.lua',
+    'l-table.lua',
+    'l-boolean.lua',
+    'l-number.lua',
+    'l-unicode.lua',
+    'l-os.lua',
+    'l-io.lua',
+    'l-file.lua',
+    'l-md5.lua',
+    'l-url.lua',
+    'l-dir.lua',
+    'l-utils.lua',
+    'l-dimen.lua',
+    'trac-inf.lua',
+    'trac-tra.lua',
+    'trac-log.lua',
+    'luat-env.lua', -- here ?
+    'data-res.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-tmp.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-pre.lua',
+    'data-tex.lua',
+    'data-bin.lua',
+--  'data-zip.lua',
+--  'data-clr.lua',
+    'data-lua.lua',
+    'data-ctx.lua',
+    'luat-fio.lua',
+    'luat-cnf.lua',
+}
+
+instance.engine     =     environment.arguments["engine"]   or 'luatex'
+instance.progname   =     environment.arguments["progname"] or 'context'
+instance.luaname    =     environment.arguments["luafile"]  or "" -- environment.ownname or ""
+instance.lualibs    =     environment.arguments["lualibs"]  or table.concat(resolvers.defaultlibs,",")
+instance.allresults =     environment.arguments["all"]      or false
+instance.pattern    =     environment.arguments["pattern"]  or nil
+instance.sortdata   =     environment.arguments["sort"]     or false
+instance.kpseonly   = not environment.arguments["all"]      or false
+instance.my_format  =     environment.arguments["format"]   or instance.format
+
+if type(instance.pattern) == 'boolean' then
+    logs.simple("invalid pattern specification")
+    instance.pattern = nil
+end
+
+if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end
+
+runners  = runners  or { }
+messages = messages or { }
+
+messages.no_ini_file = [[
+There is no lua initialization file found. This file can be forced by the
+"--progname" directive, or specified with "--luaname", or it is derived
+automatically from the formatname (aka jobname). It may be that you have
+to regenerate the file database using "luatools --generate".
+]]
+
+messages.help = [[
+--generate        generate file database
+--variables       show configuration variables
+--expansions      show expanded variables
+--configurations  show configuration order
+--expand-braces   expand complex variable
+--expand-path     expand variable (resolve paths)
+--expand-var      expand variable (resolve references)
+--show-path       show path expansion of ...
+--var-value       report value of variable
+--find-file       report file location
+--find-path       report path of file
+--make or --ini   make luatex format
+--run or --fmt=   run luatex format
+--luafile=str     lua inifile (default is .lua)
+--lualibs=list    libraries to assemble (optional when --compile)
+--compile         assemble and compile lua inifile
+--verbose         give a bit more info
+--all             show all found files
+--sort            sort cached data
+--engine=str      target engine
+--progname=str    format or backend
+--pattern=str     filter variables
+]]
+
+function runners.make_format(texname)
+    local instance = resolvers.instance
+    if texname and texname ~= "" then
+        if resolvers.usecache then
+            local path = file.join(caches.setpath("formats")) -- maybe platform
+            if path and lfs then
+                lfs.chdir(path)
+            end
+        end
+        local barename = texname:gsub("%.%a+$","")
+        if barename == texname then
+            texname = texname .. ".tex"
+        end
+        local fullname = resolvers.find_files(texname)[1] or ""
+        if fullname == "" then
+            logs.simple("no tex file with name: %s",texname)
+        else
+            local luaname, lucname, luapath, lualibs = "", "", "", { }
+            -- the following is optional, since context.lua can also
+            -- handle this collect and compile business
+            if environment.arguments["compile"] then
+                if luaname == "" then luaname = barename end
+                logs.simple("creating initialization file: %s",luaname)
+                luapath = file.dirname(luaname)
+                if luapath == "" then
+                    luapath = file.dirname(texname)
+                end
+                if luapath == "" then
+                    luapath = file.dirname(resolvers.find_files(texname)[1] or "")
+                end
+                lualibs = string.split(instance.lualibs,",")
+                luaname = file.basename(barename .. ".lua")
+                lucname = file.basename(barename .. ".luc")
+                -- todo: when this fails, we can just copy the merged libraries from
+                -- luatools since they are normally the same, at least for context
+                if lualibs[1] then
+                    local firstlib = file.join(luapath,lualibs[1])
+                    if not lfs.isfile(firstlib) then
+                        local foundname = resolvers.find_files(lualibs[1])[1]
+                        if foundname then
+                            logs.simple("located library path: %s",luapath)
+                            luapath = file.dirname(foundname)
+                        end
+                    end
+                end
+                logs.simple("using library path: %s",luapath)
+                logs.simple("using lua libraries: %s",table.join(lualibs," "))
+                utils.merger.selfcreate(lualibs,luapath,luaname)
+                local strip = resolvers.boolean_variable("LUACSTRIP", true)
+                if utils.lua.compile(luaname,lucname,false,strip) and io.exists(lucname) then
+                    luaname = lucname
+                    logs.simple("using compiled initialization file: %s",lucname)
+                else
+                    logs.simple("using uncompiled initialization file: %s",luaname)
+                end
+            else
+                for _, v in pairs({instance.luaname, instance.progname, barename}) do
+                    v = string.gsub(v..".lua","%.lua%.lua$",".lua")
+                    if v and (v ~= "") then
+                        luaname = resolvers.find_files(v)[1] or ""
+                        if luaname ~= "" then
+                            break
+                        end
+                    end
+                end
+            end
+            if environment.arguments["noluc"] then
+                luaname = luaname:gsub("%.luc$",".lua") -- make this an option
+            end
+            if luaname == "" then
+                if logs.verbose then
+                    logs.simplelines(messages.no_ini_file)
+                    logs.simple("texname : %s",texname)
+                    logs.simple("luaname : %s",instance.luaname)
+                    logs.simple("progname: %s",instance.progname)
+                    logs.simple("barename: %s",barename)
+                end
+            else
+                logs.simple("using lua initialization file: %s",luaname)
+                local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem")
+                if mp and #mp > 0 then
+                    for _, name in ipairs(mp) do
+                        logs.simple("removing related mplib format %s", file.basename(name))
+                        os.remove(name)
+                    end
+                end
+                local flags = {
+                    "--ini",
+                    "--lua=" .. string.quote(luaname)
+                }
+                local bs = (os.platform == "unix" and "\\\\") or "\\" -- todo: make a function
+                local command = "luatex ".. table.concat(flags," ")  .. " " .. string.quote(fullname) .. " " .. bs .. "dump"
+                logs.simple("running command: %s\n",command)
+                os.spawn(command)
+                -- todo: do a dummy run that generates the related metafun and mfplain formats
+            end
+        end
+    else
+        logs.simple("no tex file given")
+    end
+end
+
+function runners.run_format(name,data,more)
+ -- hm, rather old code here; we can now use the file.whatever functions
+    if name and (name ~= "") then
+        local barename = name:gsub("%.%a+$","")
+        local fmtname = ""
+        if resolvers.usecache then
+            local path = file.join(caches.setpath("formats")) -- maybe platform
+            fmtname = file.join(path,barename..".fmt") or ""
+        end
+        if fmtname == "" then
+            fmtname = resolvers.find_files(barename..".fmt")[1] or ""
+        end
+        fmtname = resolvers.clean_path(fmtname)
+        barename = fmtname:gsub("%.%a+$","")
+        if fmtname == "" then
+            logs.simple("no format with name: %s",name)
+        else
+            local luaname = barename .. ".luc"
+            local f = io.open(luaname)
+            if not f then
+                luaname = barename .. ".lua"
+                f = io.open(luaname)
+            end
+            if f then
+                f:close()
+                local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. (more ~= "" and string.quote(more) or "")
+                logs.simple("running command: %s",command)
+                os.spawn(command)
+            else
+                logs.simple("using format name: %s",fmtname)
+                logs.simple("no luc/lua with name: %s",barename)
+            end
+        end
+    end
+end
+
+local ok = true
+
+-- private option --noluc for testing errors in the stub
+
+if environment.arguments["find-file"] then
+    resolvers.load()
+    instance.format  = environment.arguments["format"] or instance.format
+    if instance.pattern then
+        instance.allresults = true
+        resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format)
+    else
+        resolvers.for_files(resolvers.find_files, environment.files, instance.my_format)
+    end
+elseif environment.arguments["find-path"] then
+    resolvers.load()
+    local path = resolvers.find_file(environment.files[1], instance.my_format)
+    if logs.verbose then
+        logs.simple(file.dirname(path))
+    else
+        print(file.dirname(path))
+    end
+elseif environment.arguments["run"] then
+    resolvers.load("nofiles") -- ! no need for loading databases
+    logs.setverbose(true)
+    runners.run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "")
+elseif environment.arguments["fmt"] then
+    resolvers.load("nofiles") -- ! no need for loading databases
+    logs.setverbose(true)
+    runners.run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "")
+elseif environment.arguments["expand-braces"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.expand_braces, environment.files)
+elseif environment.arguments["expand-path"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.expand_path, environment.files)
+elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.expand_var, environment.files)
+elseif environment.arguments["show-path"] or environment.arguments["path-value"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.show_path, environment.files)
+elseif environment.arguments["var-value"] or environment.arguments["show-value"] then
+    resolvers.load("nofiles")
+    resolvers.for_files(resolvers.var_value, environment.files)
+elseif environment.arguments["format-path"] then
+    resolvers.load()
+    logs.simple(caches.setpath("format"))
+elseif instance.pattern then -- brrr
+    resolvers.load()
+    instance.format = environment.arguments["format"] or instance.format
+    instance.allresults = true
+    resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format)
+elseif environment.arguments["generate"] then
+    instance.renewcache = true
+    logs.setverbose(true)
+    resolvers.load()
+elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then
+    resolvers.load()
+    logs.setverbose(true)
+    runners.make_format(environment.files[1] or "")
+elseif environment.arguments["selfmerge"] then
+    utils.merger.selfmerge(own.name,own.libs,own.list)
+elseif environment.arguments["selfclean"] then
+    utils.merger.selfclean(own.name)
+elseif environment.arguments["selfupdate"] then
+    resolvers.load()
+    logs.setverbose(true)
+    resolvers.update_script(own.name,"luatools")
+elseif environment.arguments["variables"] or environment.arguments["show-variables"] then
+    resolvers.load("nofiles")
+    resolvers.listers.variables()
+elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then
+    resolvers.load("nofiles")
+    resolvers.listers.expansions()
+elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then
+    resolvers.load("nofiles")
+    resolvers.listers.configurations()
+elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then
+    logs.help(messages.help)
+else
+    resolvers.load()
+    resolvers.for_files(resolvers.find_files, environment.files, instance.my_format)
+end
+
+if logs.verbose then
+    logs.simpleline()
+    logs.simple("runtime: %0.3f seconds",os.runtime())
+end
+
+if os.platform == "unix" then
+    io.write("\n")
+end
diff --git a/scripts/context/stubs/unix/makempy b/scripts/context/stubs/unix/makempy
index 4bf7a1af2..34892b284 100755
--- a/scripts/context/stubs/unix/makempy
+++ b/scripts/context/stubs/unix/makempy
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart makempy.pl "$@"
+mtxrun --usekpse --execute makempy.pl "$@"
diff --git a/scripts/context/stubs/unix/metatex b/scripts/context/stubs/unix/metatex
new file mode 100755
index 000000000..f0c6b65d4
--- /dev/null
+++ b/scripts/context/stubs/unix/metatex
@@ -0,0 +1,2 @@
+#!/bin/sh
+mtxrun --script metatex "$@"
diff --git a/scripts/context/stubs/unix/mpstools b/scripts/context/stubs/unix/mpstools
index b4c8f6345..1a64d90b0 100755
--- a/scripts/context/stubs/unix/mpstools
+++ b/scripts/context/stubs/unix/mpstools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart mpstools.rb "$@"
+mtxrun --usekpse --execute mpstools.rb "$@"
diff --git a/scripts/context/stubs/unix/mptopdf b/scripts/context/stubs/unix/mptopdf
index 980a3123d..f57a8b7a7 100755
--- a/scripts/context/stubs/unix/mptopdf
+++ b/scripts/context/stubs/unix/mptopdf
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart mptopdf.pl "$@"
+mtxrun --usekpse --execute mptopdf.pl "$@"
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
new file mode 100755
index 000000000..0af429bf1
--- /dev/null
+++ b/scripts/context/stubs/unix/mtxrun
@@ -0,0 +1,10190 @@
+#!/usr/bin/env texlua
+
+if not modules then modules = { } end modules ['mtxrun'] = {
+    version   = 1.001,
+    comment   = "runner, lua replacement for texmfstart.rb",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+
+-- one can make a stub:
+--
+-- #!/bin/sh
+-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly mtxrun.lua "$@"
+
+-- filename : mtxrun.lua
+-- comment  : companion to context.tex
+-- author   : Hans Hagen, PRAGMA-ADE, Hasselt NL
+-- copyright: PRAGMA ADE / ConTeXt Development Team
+-- license  : see context related readme files
+
+-- This script is based on texmfstart.rb but does not use kpsewhich to
+-- locate files. Although kpse is a library it never came to opening up
+-- its interface to other programs (esp scripting languages) and so we
+-- do it ourselves. The lua variant evolved out of an experimental ruby
+-- one. Interesting is that using a scripting language instead of c does
+-- not have a speed penalty. Actually the lua variant is more efficient,
+-- especially when multiple calls to kpsewhich are involved. The lua
+-- library also gives way more control.
+
+-- to be done / considered
+--
+-- support for --exec or make it default
+-- support for jar files (or maybe not, never used, too messy)
+-- support for $RUBYINPUTS cum suis (if still needed)
+-- remember for subruns: _CTX_K_V_#{original}_
+-- remember for subruns: _CTX_K_S_#{original}_
+-- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb]
+
+texlua = true
+
+-- begin library merge
+
+
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-string'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep
+
+if not string.split then
+
+    -- this will be overloaded by a faster lpeg variant
+
+    function string:split(pattern)
+        if #self > 0 then
+            local t = { }
+            for s in gmatch(self..pattern,"(.-)"..pattern) do
+                t[#t+1] = s
+            end
+            return t
+        else
+            return { }
+        end
+    end
+
+end
+
+local chr_to_esc = {
+    ["%"] = "%%",
+    ["."] = "%.",
+    ["+"] = "%+", ["-"] = "%-", ["*"] = "%*",
+    ["^"] = "%^", ["$"] = "%$",
+    ["["] = "%[", ["]"] = "%]",
+    ["("] = "%(", [")"] = "%)",
+    ["{"] = "%{", ["}"] = "%}"
+}
+
+string.chr_to_esc = chr_to_esc
+
+function string:esc() -- variant 2
+    return (gsub(self,"(.)",chr_to_esc))
+end
+
+function string:unquote()
+    return (gsub(self,"^([\"\'])(.*)%1$","%2"))
+end
+
+function string:quote() -- we could use format("%q")
+    return '"' .. self:unquote() .. '"'
+end
+
+function string:count(pattern) -- variant 3
+    local n = 0
+    for _ in gmatch(self,pattern) do
+        n = n + 1
+    end
+    return n
+end
+
+function string:limit(n,sentinel)
+    if #self > n then
+        sentinel = sentinel or " ..."
+        return sub(self,1,(n-#sentinel)) .. sentinel
+    else
+        return self
+    end
+end
+
+function string:strip()
+    return (gsub(self,"^%s*(.-)%s*$", "%1"))
+end
+
+function string:is_empty()
+    return not find(find,"%S")
+end
+
+function string:enhance(pattern,action)
+    local ok, n = true, 0
+    while ok do
+        ok = false
+        self = gsub(self,pattern, function(...)
+            ok, n = true, n + 1
+            return action(...)
+        end)
+    end
+    return self, n
+end
+
+local chr_to_hex, hex_to_chr = { }, { }
+
+for i=0,255 do
+    local c, h = char(i), format("%02X",i)
+    chr_to_hex[c], hex_to_chr[h] = h, c
+end
+
+function string:to_hex()
+    return (gsub(self or "","(.)",chr_to_hex))
+end
+
+function string:from_hex()
+    return (gsub(self or "","(..)",hex_to_chr))
+end
+
+if not string.characters then
+
+    local function nextchar(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, str:sub(index,index)
+    end
+    function string:characters()
+        return nextchar, self, 0
+    end
+    local function nextbyte(str, index)
+        index = index + 1
+        return (index <= #str) and index or nil, byte(str:sub(index,index))
+    end
+    function string:bytes()
+        return nextbyte, self, 0
+    end
+
+end
+
+-- we can use format for this (neg n)
+
+function string:rpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self .. self.rep(chr or " ",m)
+    else
+        return self
+    end
+end
+
+function string:lpadd(n,chr)
+    local m = n-#self
+    if m > 0 then
+        return self.rep(chr or " ",m) .. self
+    else
+        return self
+    end
+end
+
+string.padd = string.rpadd
+
+function is_number(str) -- tonumber
+    return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1
+end
+
+--~ print(is_number("1"))
+--~ print(is_number("1.1"))
+--~ print(is_number(".1"))
+--~ print(is_number("-0.1"))
+--~ print(is_number("+0.1"))
+--~ print(is_number("-.1"))
+--~ print(is_number("+.1"))
+
+function string:split_settings() -- no {} handling, see l-aux for lpeg variant
+    if find(self,"=") then
+        local t = { }
+        for k,v in gmatch(self,"(%a+)=([^%,]*)") do
+            t[k] = v
+        end
+        return t
+    else
+        return nil
+    end
+end
+
+local patterns_escapes = {
+    ["-"] = "%-",
+    ["."] = "%.",
+    ["+"] = "%+",
+    ["*"] = "%*",
+    ["%"] = "%%",
+    ["("] = "%)",
+    [")"] = "%)",
+    ["["] = "%[",
+    ["]"] = "%]",
+}
+
+function string:pattesc()
+    return (gsub(self,".",patterns_escapes))
+end
+
+function string:tohash()
+    local t = { }
+    for s in gmatch(self,"([^, ]+)") do -- lpeg
+        t[s] = true
+    end
+    return t
+end
+
+local pattern = lpeg.Ct(lpeg.C(1)^0)
+
+function string:totable()
+    return pattern:match(self)
+end
+
+--~ for _, str in ipairs {
+--~     "1234567123456712345671234567",
+--~     "a\tb\tc",
+--~     "aa\tbb\tcc",
+--~     "aaa\tbbb\tccc",
+--~     "aaaa\tbbbb\tcccc",
+--~     "aaaaa\tbbbbb\tccccc",
+--~     "aaaaaa\tbbbbbb\tcccccc",
+--~ } do print(string.tabtospace(str)) end
+
+function string.tabtospace(str,tab)
+    -- we don't handle embedded newlines
+    while true do
+        local s = find(str,"\t")
+        if s then
+            if not tab then tab = 7 end -- only when found
+            local d = tab-(s-1)%tab
+            if d > 0 then
+                str = gsub(str,"\t",rep(" ",d),1)
+            else
+                str = gsub(str,"\t","",1)
+            end
+        else
+            break
+        end
+    end
+    return str
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-lpeg'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
+
+--~ l-lpeg.lua :
+
+--~ lpeg.digit         = lpeg.R('09')^1
+--~ lpeg.sign          = lpeg.S('+-')^1
+--~ lpeg.cardinal      = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.integer       = lpeg.P(lpeg.sign^0 * lpeg.digit^1)
+--~ lpeg.float         = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1)
+--~ lpeg.number        = lpeg.float + lpeg.integer
+--~ lpeg.oct           = lpeg.P("0") * lpeg.R('07')^1
+--~ lpeg.hex           = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1
+--~ lpeg.uppercase     = lpeg.P("AZ")
+--~ lpeg.lowercase     = lpeg.P("az")
+
+--~ lpeg.eol           = lpeg.S('\r\n\f')^1 -- includes formfeed
+--~ lpeg.space         = lpeg.S(' ')^1
+--~ lpeg.nonspace      = lpeg.P(1-lpeg.space)^1
+--~ lpeg.whitespace    = lpeg.S(' \r\n\f\t')^1
+--~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1
+
+local hash = { }
+
+function lpeg.anywhere(pattern) --slightly adapted from website
+    return P { P(pattern) + 1 * lpeg.V(1) }
+end
+
+function lpeg.startswith(pattern) --slightly adapted
+    return P(pattern)
+end
+
+function lpeg.splitter(pattern, action)
+    return (((1-P(pattern))^1)/action+1)^0
+end
+
+-- variant:
+
+--~ local parser = lpeg.Ct(lpeg.splitat(newline))
+
+local crlf     = P("\r\n")
+local cr       = P("\r")
+local lf       = P("\n")
+local space    = S(" \t\f\v")  -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local newline  = crlf + cr + lf
+local spacing  = space^0 * newline
+
+local empty    = spacing * Cc("")
+local nonempty = Cs((1-spacing)^1) * spacing^-1
+local content  = (empty + nonempty)^1
+
+local capture = Ct(content^0)
+
+function string:splitlines()
+    return capture:match(self)
+end
+
+lpeg.linebyline = content -- better make a sublibrary
+
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps->what->more"))  -- oeps what more
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps->what->more"))  -- oeps what->more
+--~ local p = lpeg.splitat("->",false)  print(p:match("oeps"))              -- oeps
+--~ local p = lpeg.splitat("->",true)   print(p:match("oeps"))              -- oeps
+
+local splitters_s, splitters_m = { }, { }
+
+local function splitat(separator,single)
+    local splitter = (single and splitters_s[separator]) or splitters_m[separator]
+    if not splitter then
+        separator = P(separator)
+        if single then
+            local other, any = C((1 - separator)^0), P(1)
+            splitter = other * (separator * C(any^0) + "")
+            splitters_s[separator] = splitter
+        else
+            local other = C((1 - separator)^0)
+            splitter = other * (separator * other)^0
+            splitters_m[separator] = splitter
+        end
+    end
+    return splitter
+end
+
+lpeg.splitat = splitat
+
+local cache = { }
+
+function string:split(separator)
+    local c = cache[separator]
+    if not c then
+        c = Ct(splitat(separator))
+        cache[separator] = c
+    end
+    return c:match(self)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-table'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+table.join = table.concat
+
+local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove
+local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump
+local getmetatable, setmetatable = getmetatable, setmetatable
+local type, next, tostring, ipairs = type, next, tostring, ipairs
+
+function table.strip(tab)
+    local lst = { }
+    for i=1,#tab do
+        local s = gsub(tab[i],"^%s*(.-)%s*$","%1")
+        if s == "" then
+            -- skip this one
+        else
+            lst[#lst+1] = s
+        end
+    end
+    return lst
+end
+
+local function sortedkeys(tab)
+    local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+        if kind == 3 then
+            -- no further check
+        else
+            local tkey = type(key)
+            if tkey == "string" then
+            --  if kind == 2 then kind = 3 else kind = 1 end
+                kind = (kind == 2 and 3) or 1
+            elseif tkey == "number" then
+            --  if kind == 1 then kind = 3 else kind = 2 end
+                kind = (kind == 1 and 3) or 2
+            else
+                kind = 3
+            end
+        end
+    end
+    if kind == 0 or kind == 3 then
+        sort(srt,function(a,b) return (tostring(a) < tostring(b)) end)
+    else
+        sort(srt)
+    end
+    return srt
+end
+
+local function sortedhashkeys(tab) -- fast one
+    local srt = { }
+    for key,_ in next, tab do
+        srt[#srt+1] = key
+    end
+    sort(srt)
+    return srt
+end
+
+table.sortedkeys     = sortedkeys
+table.sortedhashkeys = sortedhashkeys
+
+function table.sortedpairs(t)
+    local s = sortedhashkeys(t) -- maybe just sortedkeys
+    local n = 0
+    local function kv(s)
+        n = n + 1
+        local k = s[n]
+        return k, t[k]
+    end
+    return kv, s
+end
+
+function table.append(t, list)
+    for _,v in next, list do
+        insert(t,v)
+    end
+end
+
+function table.prepend(t, list)
+    for k,v in next, list do
+        insert(t,k,v)
+    end
+end
+
+function table.merge(t, ...) -- first one is target
+    t = t or {}
+    local lst = {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            t[k] = v
+        end
+    end
+    return t
+end
+
+function table.merged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        for k, v in next, lst[i] do
+            tmp[k] = v
+        end
+    end
+    return tmp
+end
+
+function table.imerge(t, ...)
+    local lst = {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            t[#t+1] = nst[j]
+        end
+    end
+    return t
+end
+
+function table.imerged(...)
+    local tmp, lst = { }, {...}
+    for i=1,#lst do
+        local nst = lst[i]
+        for j=1,#nst do
+            tmp[#tmp+1] = nst[j]
+        end
+    end
+    return tmp
+end
+
+local function fastcopy(old) -- fast one
+    if old then
+        local new = { }
+        for k,v in next, old do
+            if type(v) == "table" then
+                new[k] = fastcopy(v) -- was just table.copy
+            else
+                new[k] = v
+            end
+        end
+        -- optional second arg
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
+        end
+        return new
+    else
+        return { }
+    end
+end
+
+local function copy(t, tables) -- taken from lua wiki, slightly adapted
+    tables = tables or { }
+    local tcopy = {}
+    if not tables[t] then
+        tables[t] = tcopy
+    end
+    for i,v in next, t do -- brrr, what happens with sparse indexed
+        if type(i) == "table" then
+            if tables[i] then
+                i = tables[i]
+            else
+                i = copy(i, tables)
+            end
+        end
+        if type(v) ~= "table" then
+            tcopy[i] = v
+        elseif tables[v] then
+            tcopy[i] = tables[v]
+        else
+            tcopy[i] = copy(v, tables)
+        end
+    end
+    local mt = getmetatable(t)
+    if mt then
+        setmetatable(tcopy,mt)
+    end
+    return tcopy
+end
+
+table.fastcopy = fastcopy
+table.copy     = copy
+
+-- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack)
+
+function table.sub(t,i,j)
+    return { unpack(t,i,j) }
+end
+
+function table.replace(a,b)
+    for k,v in next, b do
+        a[k] = v
+    end
+end
+
+-- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice)
+
+function table.is_empty(t)
+    return not t or not next(t)
+end
+
+function table.one_entry(t)
+    local n = next(t)
+    return n and not next(t,n)
+end
+
+function table.starts_at(t)
+    return ipairs(t,1)(t,0)
+end
+
+function table.tohash(t,value)
+    local h = { }
+    if t then
+        if value == nil then value = true end
+        for _, v in next, t do -- no ipairs here
+            h[v] = value
+        end
+    end
+    return h
+end
+
+function table.fromhash(t)
+    local h = { }
+    for k, v in next, t do -- no ipairs here
+        if v then h[#h+1] = k end
+    end
+    return h
+end
+
+--~ print(table.serialize(t), "\n")
+--~ print(table.serialize(t,"name"), "\n")
+--~ print(table.serialize(t,false), "\n")
+--~ print(table.serialize(t,true), "\n")
+--~ print(table.serialize(t,"name",true), "\n")
+--~ print(table.serialize(t,"name",true,true), "\n")
+
+table.serialize_functions = true
+table.serialize_compact   = true
+table.serialize_inline    = true
+
+local noquotes, hexify, handle, reduce, compact, inline, functions
+
+local reserved = table.tohash { -- intercept a language flaw, no reserved words as key
+    'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
+    'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
+}
+
+local function simple_table(t)
+    if #t > 0 then
+        local n = 0
+        for _,v in next, t do
+            n = n + 1
+        end
+        if n == #t then
+            local tt = { }
+            for i=1,#t do
+                local v = t[i]
+                local tv = type(v)
+                if tv == "number" then
+                    if hexify then
+                        tt[#tt+1] = format("0x%04X",v)
+                    else
+                        tt[#tt+1] = tostring(v) -- tostring not needed
+                    end
+                elseif tv == "boolean" then
+                    tt[#tt+1] = tostring(v)
+                elseif tv == "string" then
+                    tt[#tt+1] = format("%q",v)
+                else
+                    tt = nil
+                    break
+                end
+            end
+            return tt
+        end
+    end
+    return nil
+end
+
+-- Because this is a core function of mkiv I moved some function calls
+-- inline.
+--
+-- twice as fast in a test:
+--
+-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) )
+
+local function do_serialize(root,name,depth,level,indexed)
+    if level > 0 then
+        depth = depth .. " "
+        if indexed then
+            handle(format("%s{",depth))
+        elseif name then
+        --~ handle(format("%s%s={",depth,key(name)))
+            if type(name) == "number" then -- or find(k,"^%d+$") then
+                if hexify then
+                    handle(format("%s[0x%04X]={",depth,name))
+                else
+                    handle(format("%s[%s]={",depth,name))
+                end
+            elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then
+                handle(format("%s%s={",depth,name))
+            else
+                handle(format("%s[%q]={",depth,name))
+            end
+        else
+            handle(format("%s{",depth))
+        end
+    end
+    if root and next(root) then
+        local first, last = nil, 0 -- #root cannot be trusted here
+        if compact then
+            -- NOT: for k=1,#root do (we need to quit at nil)
+            for k,v in ipairs(root) do -- can we use next?
+                if not first then first = k end
+                last = last + 1
+            end
+        end
+        local sk = sortedkeys(root)
+        for i=1,#sk do
+            local k = sk[i]
+            local v = root[k]
+            --~ if v == root then
+                -- circular
+            --~ else
+            local t = type(v)
+            if compact and first and type(k) == "number" and k >= first and k <= last then
+                if t == "number" then
+                    if hexify then
+                        handle(format("%s 0x%04X,",depth,v))
+                    else
+                        handle(format("%s %s,",depth,v))
+                    end
+                elseif t == "string" then
+                    if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                        handle(format("%s %s,",depth,v))
+                    else
+                        handle(format("%s %q,",depth,v))
+                    end
+                elseif t == "table" then
+                    if not next(v) then
+                        handle(format("%s {},",depth))
+                    elseif inline then -- and #t > 0
+                        local st = simple_table(v)
+                        if st then
+                            handle(format("%s { %s },",depth,concat(st,", ")))
+                        else
+                            do_serialize(v,k,depth,level+1,true)
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1,true)
+                    end
+                elseif t == "boolean" then
+                    handle(format("%s %s,",depth,tostring(v)))
+                elseif t == "function" then
+                    if functions then
+                        handle(format('%s loadstring(%q),',depth,dump(v)))
+                    else
+                        handle(format('%s "function",',depth))
+                    end
+                else
+                    handle(format("%s %q,",depth,tostring(v)))
+                end
+            elseif k == "__p__" then -- parent
+                if false then
+                    handle(format("%s __p__=nil,",depth))
+                end
+            elseif t == "number" then
+            --~ if hexify then
+            --~     handle(format("%s %s=0x%04X,",depth,key(k),v))
+            --~ else
+            --~     handle(format("%s %s=%s,",depth,key(k),v))
+            --~ end
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,v))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    if hexify then
+                        handle(format("%s %s=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s %s=%s,",depth,k,v))
+                    end
+                else
+                    if hexify then
+                        handle(format("%s [%q]=0x%04X,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                end
+            elseif t == "string" then
+                if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then
+                --~ handle(format("%s %s=%s,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%s,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%s,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%s,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%s,",depth,k,v))
+                    end
+                else
+                --~ handle(format("%s %s=%q,",depth,key(k),v))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=%q,",depth,k,v))
+                        else
+                            handle(format("%s [%s]=%q,",depth,k,v))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=%q,",depth,k,v))
+                    else
+                        handle(format("%s [%q]=%q,",depth,k,v))
+                    end
+                end
+            elseif t == "table" then
+                if not next(v) then
+                    --~ handle(format("%s %s={},",depth,key(k)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]={},",depth,k))
+                        else
+                            handle(format("%s [%s]={},",depth,k))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s={},",depth,k))
+                    else
+                        handle(format("%s [%q]={},",depth,k))
+                    end
+                elseif inline then
+                    local st = simple_table(v)
+                    if st then
+                    --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", ")))
+                        if type(k) == "number" then -- or find(k,"^%d+$") then
+                            if hexify then
+                                handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", ")))
+                            else
+                                handle(format("%s [%s]={ %s },",depth,k,concat(st,", ")))
+                            end
+                        elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                            handle(format("%s %s={ %s },",depth,k,concat(st,", ")))
+                        else
+                            handle(format("%s [%q]={ %s },",depth,k,concat(st,", ")))
+                        end
+                    else
+                        do_serialize(v,k,depth,level+1)
+                    end
+                else
+                    do_serialize(v,k,depth,level+1)
+                end
+            elseif t == "boolean" then
+            --~ handle(format("%s %s=%s,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%s,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%s,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%s,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%s,",depth,k,tostring(v)))
+                end
+            elseif t == "function" then
+                if functions then
+                    --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v)))
+                    if type(k) == "number" then -- or find(k,"^%d+$") then
+                        if hexify then
+                            handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v)))
+                        else
+                            handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v)))
+                        end
+                    elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                        handle(format("%s %s=loadstring(%q),",depth,k,dump(v)))
+                    else
+                        handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v)))
+                    end
+                end
+            else
+                --~ handle(format("%s %s=%q,",depth,key(k),tostring(v)))
+                if type(k) == "number" then -- or find(k,"^%d+$") then
+                    if hexify then
+                        handle(format("%s [0x%04X]=%q,",depth,k,tostring(v)))
+                    else
+                        handle(format("%s [%s]=%q,",depth,k,tostring(v)))
+                    end
+                elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then
+                    handle(format("%s %s=%q,",depth,k,tostring(v)))
+                else
+                    handle(format("%s [%q]=%q,",depth,k,tostring(v)))
+                end
+            end
+            --~ end
+        end
+    end
+   if level > 0 then
+        handle(format("%s},",depth))
+    end
+end
+
+-- replacing handle by a direct t[#t+1] = ... (plus test) is not much
+-- faster (0.03 on 1.00 for zapfino.tma)
+
+local function serialize(root,name,_handle,_reduce,_noquotes,_hexify)
+    noquotes = _noquotes
+    hexify = _hexify
+    handle = _handle or print
+    reduce = _reduce or false
+    compact = table.serialize_compact
+    inline  = compact and table.serialize_inline
+    functions = table.serialize_functions
+    local tname = type(name)
+    if tname == "string" then
+        if name == "return" then
+            handle("return {")
+        else
+            handle(name .. "={")
+        end
+    elseif tname == "number" then
+        if hexify then
+            handle(format("[0x%04X]={",name))
+        else
+            handle("[" .. name .. "]={")
+        end
+    elseif tname == "boolean" then
+        if name then
+            handle("return {")
+        else
+            handle("{")
+        end
+    else
+        handle("t={")
+    end
+    if root and next(root) then
+        do_serialize(root,name,"",0,indexed)
+    end
+    handle("}")
+end
+
+--~ name:
+--~
+--~ true     : return     { }
+--~ false    :            { }
+--~ nil      : t        = { }
+--~ string   : string   = { }
+--~ 'return' : return     { }
+--~ number   : [number] = { }
+
+function table.serialize(root,name,reduce,noquotes,hexify)
+    local t = { }
+    local function flush(s)
+        t[#t+1] = s
+    end
+    serialize(root,name,flush,reduce,noquotes,hexify)
+    return concat(t,"\n")
+end
+
+function table.tohandle(handle,root,name,reduce,noquotes,hexify)
+    serialize(root,name,handle,reduce,noquotes,hexify)
+end
+
+-- sometimes tables are real use (zapfino extra pro is some 85M) in which
+-- case a stepwise serialization is nice; actually, we could consider:
+--
+-- for line in table.serializer(root,name,reduce,noquotes) do
+--    ...(line)
+-- end
+--
+-- so this is on the todo list
+
+table.tofile_maxtab = 2*1024
+
+function table.tofile(filename,root,name,reduce,noquotes,hexify)
+    local f = io.open(filename,'w')
+    if f then
+        local maxtab = table.tofile_maxtab
+        if maxtab > 1 then
+            local t = { }
+            local function flush(s)
+                t[#t+1] = s
+                if #t > maxtab then
+                    f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice
+                    t = { }
+                end
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+            f:write(concat(t,"\n"),"\n")
+        else
+            local function flush(s)
+                f:write(s,"\n")
+            end
+            serialize(root,name,flush,reduce,noquotes,hexify)
+        end
+        f:close()
+    end
+end
+
+local function flatten(t,f,complete)
+    for i=1,#t do
+        local v = t[i]
+        if type(v) == "table" then
+            if complete or type(v[1]) == "table" then
+                flatten(v,f,complete)
+            else
+                f[#f+1] = v
+            end
+        else
+            f[#f+1] = v
+        end
+    end
+end
+
+function table.flatten(t)
+    local f = { }
+    flatten(t,f,true)
+    return f
+end
+
+function table.unnest(t) -- bad name
+    local f = { }
+    flatten(t,f,false)
+    return f
+end
+
+table.flatten_one_level = table.unnest
+
+-- the next three may disappear
+
+function table.remove_value(t,value) -- todo: n
+    if value then
+        for i=1,#t do
+            if t[i] == value then
+                remove(t,i)
+                -- remove all, so no: return
+            end
+        end
+    end
+end
+
+function table.insert_before_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i,str)
+                    return
+                end
+            end
+        end
+        insert(t,1,str)
+    elseif value then
+        insert(t,1,value)
+    end
+end
+
+function table.insert_after_value(t,value,str)
+    if str then
+        if value then
+            for i=1,#t do
+                if t[i] == value then
+                    insert(t,i+1,str)
+                    return
+                end
+            end
+        end
+        t[#t+1] = str
+    elseif value then
+        t[#t+1] = value
+    end
+end
+
+local function are_equal(a,b,n,m) -- indexed
+    if #a == #b then
+        n = n or 1
+        m = m or #a
+        for i=n,m do
+            local ai, bi = a[i], b[i]
+            if ai==bi then
+                -- same
+            elseif type(ai)=="table" and type(bi)=="table" then
+                if not are_equal(ai,bi) then
+                    return false
+                end
+            else
+                return false
+            end
+        end
+        return true
+    else
+        return false
+    end
+end
+
+local function identical(a,b) -- assumes same structure
+    for ka, va in next, a do
+        local vb = b[k]
+        if va == vb then
+            -- same
+        elseif type(va) == "table" and  type(vb) == "table" then
+            if not identical(va,vb) then
+                return false
+            end
+        else
+            return false
+        end
+    end
+    return true
+end
+
+table.are_equal = are_equal
+table.identical = identical
+
+-- maybe also make a combined one
+
+function table.compact(t)
+    if t then
+        for k,v in next, t do
+            if not next(v) then
+                t[k] = nil
+            end
+        end
+    end
+end
+
+function table.contains(t, v)
+    if t then
+        for i=1, #t do
+            if t[i] == v then
+                return i
+            end
+        end
+    end
+    return false
+end
+
+function table.count(t)
+    local n, e = 0, next(t)
+    while e do
+        n, e = n + 1, next(t,e)
+    end
+    return n
+end
+
+function table.swapped(t)
+    local s = { }
+    for k, v in next, t do
+        s[v] = k
+    end
+    return s
+end
+
+--~ function table.are_equal(a,b)
+--~     return table.serialize(a) == table.serialize(b)
+--~ end
+
+function table.clone(t,p) -- t is optional or nil or table
+    if not p then
+        t, p = { }, t or { }
+    elseif not t then
+        t = { }
+    end
+    setmetatable(t, { __index = function(_,key) return p[key] end })
+    return t
+end
+
+function table.hexed(t,seperator)
+    local tt = { }
+    for i=1,#t do tt[i] = format("0x%04X",t[i]) end
+    return concat(tt,seperator or " ")
+end
+
+function table.reverse_hash(h)
+    local r = { }
+    for k,v in next, h do
+        r[v] = lower(gsub(k," ",""))
+    end
+    return r
+end
+
+function table.reverse(t)
+    local tt = { }
+    if #t > 0 then
+        for i=#t,1,-1 do
+            tt[#tt+1] = t[i]
+        end
+    end
+    return tt
+end
+
+--~ function table.keys(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return k
+--~ end
+
+--~ function table.keys_as_string(t)
+--~     local k = { }
+--~     for k,_ in next, t do
+--~         k[#k+1] = k
+--~     end
+--~     return concat(k,"")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-io'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local byte = string.byte
+
+if string.find(os.getenv("PATH"),";") then
+    io.fileseparator, io.pathseparator = "\\", ";"
+else
+    io.fileseparator, io.pathseparator = "/" , ":"
+end
+
+function io.loaddata(filename,textmode)
+    local f = io.open(filename,(textmode and 'r') or 'rb')
+    if f then
+        local data = f:read('*all')
+    --  garbagecollector.check(data)
+        f:close()
+        return data
+    else
+        return nil
+    end
+end
+
+function io.savedata(filename,data,joiner)
+    local f = io.open(filename,"wb")
+    if f then
+        if type(data) == "table" then
+            f:write(table.join(data,joiner or ""))
+        elseif type(data) == "function" then
+            data(f)
+        else
+            f:write(data)
+        end
+        f:close()
+        return true
+    else
+        return false
+    end
+end
+
+function io.exists(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return false
+    else
+        assert(f:close())
+        return true
+    end
+end
+
+function io.size(filename)
+    local f = io.open(filename)
+    if f == nil then
+        return 0
+    else
+        local s = f:seek("end")
+        assert(f:close())
+        return s
+    end
+end
+
+function io.noflines(f)
+    local n = 0
+    for _ in f:lines() do
+        n = n + 1
+    end
+    f:seek('set',0)
+    return n
+end
+
+local nextchar = {
+    [ 4] = function(f)
+        return f:read(1,1,1,1)
+    end,
+    [ 2] = function(f)
+        return f:read(1,1)
+    end,
+    [ 1] = function(f)
+        return f:read(1)
+    end,
+    [-2] = function(f)
+        local a, b = f:read(1,1)
+        return b, a
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        return d, c, b, a
+    end
+}
+
+function io.characters(f,n)
+    if f then
+        return nextchar[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+local nextbyte = {
+    [4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(a), byte(b), byte(c), byte(d)
+        else
+            return nil, nil, nil, nil
+        end
+    end,
+    [2] = function(f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(a), byte(b)
+        else
+            return nil, nil
+        end
+    end,
+    [1] = function (f)
+        local a = f:read(1)
+        if a then
+            return byte(a)
+        else
+            return nil
+        end
+    end,
+    [-2] = function (f)
+        local a, b = f:read(1,1)
+        if b then
+            return byte(b), byte(a)
+        else
+            return nil, nil
+        end
+    end,
+    [-4] = function(f)
+        local a, b, c, d = f:read(1,1,1,1)
+        if d then
+            return byte(d), byte(c), byte(b), byte(a)
+        else
+            return nil, nil, nil, nil
+        end
+    end
+}
+
+function io.bytes(f,n)
+    if f then
+        return nextbyte[n or 1], f
+    else
+        return nil, nil
+    end
+end
+
+function io.ask(question,default,options)
+    while true do
+        io.write(question)
+        if options then
+            io.write(string.format(" [%s]",table.concat(options,"|")))
+        end
+        if default then
+            io.write(string.format(" [%s]",default))
+        end
+        io.write(string.format(" "))
+        local answer = io.read()
+        answer = answer:gsub("^%s*(.*)%s*$","%1")
+        if answer == "" and default then
+            return default
+        elseif not options then
+            return answer
+        else
+            for _,v in pairs(options) do
+                if v == answer then
+                    return answer
+                end
+            end
+            local pattern = "^" .. answer
+            for _,v in pairs(options) do
+                if v:find(pattern) then
+                    return v
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-number'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+number = number or { }
+
+-- a,b,c,d,e,f = number.toset(100101)
+
+function number.toset(n)
+    return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)")
+end
+
+function number.toevenhex(n)
+    local s = format("%X",n)
+    if #s % 2 == 0 then
+        return s
+    else
+        return "0" .. s
+    end
+end
+
+-- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5%
+-- on
+--
+-- for i=1,1000000 do
+--     local a,b,c,d,e,f,g,h = number.toset(12345678)
+--     local a,b,c,d         = number.toset(1234)
+--     local a,b,c           = number.toset(123)
+-- end
+--
+-- of course dedicated "(.)(.)(.)(.)" matches are even faster
+
+local one = lpeg.C(1-lpeg.S(''))^1
+
+function number.toset(n)
+    return one:match(tostring(n))
+end
+
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-set'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+set = set or { }
+
+local nums   = { }
+local tabs   = { }
+local concat = table.concat
+
+set.create = table.tohash
+
+function set.tonumber(t)
+    if next(t) then
+        local s = ""
+    --  we could save mem by sorting, but it slows down
+        for k, v in pairs(t) do
+            if v then
+            --  why bother about the leading space
+                s = s .. " " .. k
+            end
+        end
+        if not nums[s] then
+            tabs[#tabs+1] = t
+            nums[s] = #tabs
+        end
+        return nums[s]
+    else
+        return 0
+    end
+end
+
+function set.totable(n)
+    if n == 0 then
+        return { }
+    else
+        return tabs[n] or { }
+    end
+end
+
+function set.contains(n,s)
+    if type(n) == "table" then
+        return n[s]
+    elseif n == 0 then
+        return false
+    else
+        local t = tabs[n]
+        return t and t[s]
+    end
+end
+
+--~ local c = set.create{'aap','noot','mies'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ local c = set.create{'zus','wim','jet'}
+--~ local s = set.tonumber(c)
+--~ local t = set.totable(s)
+--~ print(t['aap'])
+--~ print(t['jet'])
+--~ print(set.contains(t,'jet'))
+--~ print(set.contains(t,'aap'))
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-os'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lub.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+function os.resultof(command)
+    return io.popen(command,"r"):read("*all")
+end
+
+if not os.exec  then os.exec  = os.execute end
+if not os.spawn then os.spawn = os.execute end
+
+--~ os.type : windows | unix (new, we already guessed os.platform)
+--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new)
+
+if not io.fileseparator then
+    if find(os.getenv("PATH"),";") then
+        io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows"
+    else
+        io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix"
+    end
+end
+
+os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix"
+
+function os.launch(str)
+    if os.platform == "windows" then
+        os.execute("start " .. str) -- os.spawn ?
+    else
+        os.execute(str .. " &")     -- os.spawn ?
+    end
+end
+
+if not os.setenv then
+    function os.setenv() return false end
+end
+
+if not os.times then
+    -- utime  = user time
+    -- stime  = system time
+    -- cutime = children user time
+    -- cstime = children system time
+    function os.times()
+        return {
+            utime  = os.gettimeofday(), -- user
+            stime  = 0,                 -- system
+            cutime = 0,                 -- children user
+            cstime = 0,                 -- children system
+        }
+    end
+end
+
+os.gettimeofday = os.gettimeofday or os.clock
+
+local startuptime = os.gettimeofday()
+
+function os.runtime()
+    return os.gettimeofday() - startuptime
+end
+
+--~ print(os.gettimeofday()-os.time())
+--~ os.sleep(1.234)
+--~ print (">>",os.runtime())
+--~ print(os.date("%H:%M:%S",os.gettimeofday()))
+--~ print(os.date("%H:%M:%S",os.time()))
+
+os.arch = os.arch or function()
+    local a = os.resultof("uname -m") or "linux"
+    os.arch = function()
+        return a
+    end
+    return a
+end
+
+local platform
+
+function os.currentplatform(name,default)
+    if not platform then
+        local name = os.name or os.platform or name -- os.name is built in, os.platform is mine
+        if not name then
+            platform = default or "linux"
+        elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then
+            if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then
+                platform = "mswin-64"
+            else
+                platform = "mswin"
+            end
+        else
+            local architecture = os.arch()
+            if name == "linux" then
+                if find(architecture,"x86_64") then
+                    platform = "linux-64"
+                elseif find(architecture,"ppc") then
+                    platform = "linux-ppc"
+                else
+                    platform = "linux"
+                end
+            elseif name == "macosx" then
+                if find(architecture,"i386") then
+                    platform = "osx-intel"
+                else
+                    platform = "osx-ppc"
+                end
+            elseif name == "sunos" then
+                if find(architecture,"sparc") then
+                    platform = "solaris-sparc"
+                else -- if architecture == 'i86pc'
+                    platform = "solaris-intel"
+                end
+            elseif name == "freebsd" then
+                if find(architecture,"amd64") then
+                    platform = "freebsd-amd64"
+                else
+                    platform = "freebsd"
+                end
+            else
+                platform = default or name
+            end
+        end
+        function os.currentplatform()
+            return platform
+        end
+    end
+    return platform
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-file'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- needs a cleanup
+
+file = file or { }
+
+local concat = table.concat
+local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub
+
+function file.removesuffix(filename)
+    return (gsub(filename,"%.[%a%d]+$",""))
+end
+
+function file.addsuffix(filename, suffix)
+    if not find(filename,"%.[%a%d]+$") then
+        return filename .. "." .. suffix
+    else
+        return filename
+    end
+end
+
+function file.replacesuffix(filename, suffix)
+    return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix
+end
+
+function file.dirname(name,default)
+    return match(name,"^(.+)[/\\].-$") or (default or "")
+end
+
+function file.basename(name)
+    return match(name,"^.+[/\\](.-)$") or name
+end
+
+function file.nameonly(name)
+    return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$",""))
+end
+
+function file.extname(name)
+    return match(name,"^.+%.([^/\\]-)$") or  ""
+end
+
+file.suffix = file.extname
+
+--~ print(file.join("x/","/y"))
+--~ print(file.join("http://","/y"))
+--~ print(file.join("http://a","/y"))
+--~ print(file.join("http:///a","/y"))
+--~ print(file.join("//nas-1","/y"))
+
+function file.join(...)
+    local pth = concat({...},"/")
+    pth = gsub(pth,"\\","/")
+    local a, b = match(pth,"^(.*://)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    a, b = match(pth,"^(//)(.*)$")
+    if a and b then
+        return a .. gsub(b,"//+","/")
+    end
+    return (gsub(pth,"//+","/"))
+end
+
+function file.iswritable(name)
+    local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,"."))
+    return a and a.permissions:sub(2,2) == "w"
+end
+
+function file.isreadable(name)
+    local a = lfs.attributes(name)
+    return a and a.permissions:sub(1,1) == "r"
+end
+
+file.is_readable = file.isreadable
+file.is_writable = file.iswritable
+
+-- todo: lpeg
+
+function file.split_path(str)
+    local t = { }
+    str = gsub(str,"\\", "/")
+    str = gsub(str,"(%a):([;/])", "%1\001%2")
+    for name in gmatch(str,"([^;:]+)") do
+        if name ~= "" then
+            t[#t+1] = gsub(name,"\001",":")
+        end
+    end
+    return t
+end
+
+function file.join_path(tab)
+    return concat(tab,io.pathseparator) -- can have trailing //
+end
+
+function file.collapse_path(str)
+    str = gsub(str,"/%./","/")
+    local n, m = 1, 1
+    while n > 0 or m > 0 do
+        str, n = gsub(str,"[^/%.]+/%.%.$","")
+        str, m = gsub(str,"[^/%.]+/%.%./","")
+    end
+    str = gsub(str,"([^/])/$","%1")
+    str = gsub(str,"^%./","")
+    str = gsub(str,"/%.$","")
+    if str == "" then str = "." end
+    return str
+end
+
+--~ print(file.collapse_path("a/./b/.."))
+--~ print(file.collapse_path("a/aa/../b/bb"))
+--~ print(file.collapse_path("a/../.."))
+--~ print(file.collapse_path("a/.././././b/.."))
+--~ print(file.collapse_path("a/./././b/.."))
+--~ print(file.collapse_path("a/b/c/../.."))
+
+function file.robustname(str)
+    return (gsub(str,"[^%a%d%/%-%.\\]+","-"))
+end
+
+file.readdata = io.loaddata
+file.savedata = io.savedata
+
+function file.copy(oldname,newname)
+    file.savedata(newname,io.loaddata(oldname))
+end
+
+-- lpeg variants, slightly faster, not always
+
+--~ local period    = lpeg.P(".")
+--~ local slashes   = lpeg.S("\\/")
+--~ local noperiod  = 1-period
+--~ local noslashes = 1-slashes
+--~ local name      = noperiod^1
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1
+
+--~ function file.extname(name)
+--~     return pattern:match(name) or ""
+--~ end
+
+--~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1)
+
+--~ function file.removesuffix(name)
+--~     return pattern:match(name)
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1
+
+--~ function file.basename(name)
+--~     return pattern:match(name) or name
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1
+
+--~ function file.dirname(name)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2)
+--~     else
+--~         return ""
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.addsuffix(name, suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1
+
+--~ function file.replacesuffix(name,suffix)
+--~     local p = pattern:match(name)
+--~     if p then
+--~         return name:sub(1,p-2) .. "." .. suffix
+--~     else
+--~         return name .. "." .. suffix
+--~     end
+--~ end
+
+--~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1
+
+--~ function file.nameonly(name)
+--~     local a, b = pattern:match(name)
+--~     if b then
+--~         return name:sub(a,b-2)
+--~     elseif a then
+--~         return name:sub(a)
+--~     else
+--~         return name
+--~     end
+--~ end
+
+--~ local test = file.extname
+--~ local test = file.basename
+--~ local test = file.dirname
+--~ local test = file.addsuffix
+--~ local test = file.replacesuffix
+--~ local test = file.nameonly
+
+--~ print(1,test("./a/b/c/abd.def.xxx","!!!"))
+--~ print(2,test("./../b/c/abd.def.xxx","!!!"))
+--~ print(3,test("a/b/c/abd.def.xxx","!!!"))
+--~ print(4,test("a/b/c/def.xxx","!!!"))
+--~ print(5,test("a/b/c/def","!!!"))
+--~ print(6,test("def","!!!"))
+--~ print(7,test("def.xxx","!!!"))
+
+--~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim)
+
+-- also rewrite previous
+
+local letter    = lpeg.R("az","AZ") + lpeg.S("_-+")
+local separator = lpeg.P("://")
+
+local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/")
+local rootbased = lpeg.P("/") + letter*lpeg.P(":")
+
+-- ./name ../name  /name c: :// name/name
+
+function file.is_qualified_path(filename)
+    return qualified:match(filename)
+end
+
+function file.is_rootbased_path(filename)
+    return rootbased:match(filename)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-md5'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This also provides file checksums and checkers.
+
+local gsub, format, byte = string.gsub, string.format, string.byte
+
+local function convert(str,fmt)
+    return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end))
+end
+
+if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end
+if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end
+if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end
+
+--~ if not md5.HEX then
+--~     local function remap(chr) return format("%02X",byte(chr)) end
+--~     function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.hex then
+--~     local function remap(chr) return format("%02x",byte(chr)) end
+--~     function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+--~ if not md5.dec then
+--~     local function remap(chr) return format("%03i",byte(chr)) end
+--~     function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end
+--~ end
+
+file.needs_updating_threshold = 1
+
+function file.needs_updating(oldname,newname) -- size modification access change
+    local oldtime = lfs.attributes(oldname, modification)
+    local newtime = lfs.attributes(newname, modification)
+    if newtime >= oldtime then
+        return false
+    elseif oldtime - newtime < file.needs_updating_threshold then
+        return false
+    else
+        return true
+    end
+end
+
+function file.checksum(name)
+    if md5 then
+        local data = io.loaddata(name)
+        if data then
+            return md5.HEX(data)
+        end
+    end
+    return nil
+end
+
+function file.loadchecksum(name)
+    if md5 then
+        local data = io.loaddata(name .. ".md5")
+        return data and data:gsub("%s","")
+    end
+    return nil
+end
+
+function file.savechecksum(name, checksum)
+    if not checksum then checksum = file.checksum(name) end
+    if checksum then
+        io.savedata(name .. ".md5",checksum)
+        return checksum
+    end
+    return nil
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-dir'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type = type
+local find, gmatch = string.find, string.gmatch
+
+dir = dir or { }
+
+-- optimizing for no string.find (*) does not save time
+
+local attributes = lfs.attributes
+local walkdir    = lfs.dir
+
+local function glob_pattern(path,patt,recurse,action)
+    local ok, scanner
+    if path == "/" then
+        ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
+    else
+        ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
+    end
+    if ok and type(scanner) == "function" then
+        if not find(path,"/$") then path = path .. '/' end
+        for name in scanner do
+            local full = path .. name
+            local mode = attributes(full,'mode')
+            if mode == 'file' then
+                if find(full,patt) then
+                    action(full)
+                end
+            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
+                glob_pattern(full,patt,recurse,action)
+            end
+        end
+    end
+end
+
+dir.glob_pattern = glob_pattern
+
+local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V
+
+local pattern = Ct {
+    [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3),
+    [2] = C(((1-S("*?/"))^0 * P("/"))^0),
+    [3] = C(P(1)^0)
+}
+
+local filter = Cs ( (
+    P("**") / ".*" +
+    P("*")  / "[^/]*" +
+    P("?")  / "[^/]" +
+    P(".")  / "%%." +
+    P("+")  / "%%+" +
+    P("-")  / "%%-" +
+    P(1)
+)^0 )
+
+local function glob(str,t)
+    if type(str) == "table" then
+        local t = t or { }
+        for s=1,#str do
+            glob(str[s],t)
+        end
+        return t
+    elseif lfs.isfile(str) then
+        local t = t or { }
+        t[#t+1] = str
+        return t
+    else
+        local split = pattern:match(str)
+        if split then
+            local t = t or { }
+            local action = action or function(name) t[#t+1] = name end
+            local root, path, base = split[1], split[2], split[3]
+            local recurse = find(base,"%*%*")
+            local start = root .. path
+            local result = filter:match(start .. base)
+            glob_pattern(start,result,recurse,action)
+            return t
+        else
+            return { }
+        end
+    end
+end
+
+dir.glob = glob
+
+--~ list = dir.glob("**/*.tif")
+--~ list = dir.glob("/**/*.tif")
+--~ list = dir.glob("./**/*.tif")
+--~ list = dir.glob("oeps/**/*.tif")
+--~ list = dir.glob("/oeps/**/*.tif")
+
+local function globfiles(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func -- alas, we need this indirect way
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if recurse then
+                    globfiles(path .. "/" .. name,recurse,func,files)
+                end
+            elseif mode == "file" then
+                if func then
+                    if func(name) then
+                        files[#files+1] = path .. "/" .. name
+                    end
+                else
+                    files[#files+1] = path .. "/" .. name
+                end
+            end
+        end
+    end
+    return files
+end
+
+dir.globfiles = globfiles
+
+-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
+-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
+-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
+-- t = dir.glob("f:/minimal/tex/**/*")
+-- print(dir.ls("f:/minimal/tex/**/*"))
+-- print(dir.ls("*.tex"))
+
+function dir.ls(pattern)
+    return table.concat(glob(pattern),"\n")
+end
+
+--~ mkdirs("temp")
+--~ mkdirs("a/b/c")
+--~ mkdirs(".","/a/b/c")
+--~ mkdirs("a","b","c")
+
+local make_indeed = true -- false
+
+if string.find(os.getenv("PATH"),";") then
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        local first, middle, last
+        local drive = false
+        first, middle, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            -- empty network path == local path
+        else
+            first, last = str:match("^(//)/*(.-)$")
+            if first then
+                middle, last = str:match("([^/]+)/+(.-)$")
+                if middle then
+                    pth = "//" .. middle
+                else
+                    pth = "//" .. last
+                    last = ""
+                end
+            else
+                first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$")
+                if first then
+                    pth, drive = first .. middle, true
+                else
+                    middle, last = str:match("^(/*)(.-)$")
+                    if not middle then
+                        last = str
+                    end
+                end
+            end
+        end
+        for s in gmatch(last,"[^/]+") do
+            if pth == "" then
+                pth = s
+            elseif drive then
+                pth, drive = pth .. s, false
+            else
+                pth = pth .. "/" .. s
+            end
+            if make_indeed and not lfs.isdir(pth) then
+                lfs.mkdir(pth)
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("a:"))
+--~         print(dir.mkdirs("a:/b/c"))
+--~         print(dir.mkdirs("a:b/c"))
+--~         print(dir.mkdirs("a:/bbb/c"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        local first, nothing, last = str:match("^(//)(//*)(.*)$")
+        if first then
+            first = lfs.currentdir() .. "/"
+            first = first:gsub("\\","/")
+        end
+        if not first then
+            first, last = str:match("^(//)/*(.*)$")
+        end
+        if not first then
+            first, last = str:match("^([a-zA-Z]:)(.*)$")
+            if first and not find(last,"^/") then
+                local d = lfs.currentdir()
+                if lfs.chdir(first) then
+                    first = lfs.currentdir()
+                    first = first:gsub("\\","/")
+                end
+                lfs.chdir(d)
+            end
+        end
+        if not first then
+            first, last = lfs.currentdir(), str
+            first = first:gsub("\\","/")
+        end
+        last = last:gsub("//","/")
+        last = last:gsub("/%./","/")
+        last = last:gsub("^/*","")
+        first = first:gsub("/*$","")
+        if last == "" then
+            return first
+        else
+            return first .. "/" .. last
+        end
+    end
+
+else
+
+    function dir.mkdirs(...)
+        local str, pth = "", ""
+        for _, s in ipairs({...}) do
+            if s ~= "" then
+                if str ~= "" then
+                    str = str .. "/" .. s
+                else
+                    str = s
+                end
+            end
+        end
+        str = str:gsub("/+","/")
+        if find(str,"^/") then
+            pth = "/"
+            for s in gmatch(str,"[^/]+") do
+                local first = (pth == "/")
+                if first then
+                    pth = pth .. s
+                else
+                    pth = pth .. "/" .. s
+                end
+                if make_indeed and not first and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        else
+            pth = "."
+            for s in gmatch(str,"[^/]+") do
+                pth = pth .. "/" .. s
+                if make_indeed and not lfs.isdir(pth) then
+                    lfs.mkdir(pth)
+                end
+            end
+        end
+        return pth, (lfs.isdir(pth) == true)
+    end
+
+--~         print(dir.mkdirs("","","a","c"))
+--~         print(dir.mkdirs("a"))
+--~         print(dir.mkdirs("/a/b/c"))
+--~         print(dir.mkdirs("/aaa/b/c"))
+--~         print(dir.mkdirs("//a/b/c"))
+--~         print(dir.mkdirs("///a/b/c"))
+--~         print(dir.mkdirs("a/bbb//ccc/"))
+
+    function dir.expand_name(str)
+        if not find(str,"^/") then
+            str = lfs.currentdir() .. "/" .. str
+        end
+        str = str:gsub("//","/")
+        str = str:gsub("/%./","/")
+        return str
+    end
+
+end
+
+dir.makedirs = dir.mkdirs
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-boolean'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+boolean = boolean or { }
+
+local type, tonumber = type, tonumber
+
+function boolean.tonumber(b)
+    if b then return 1 else return 0 end
+end
+
+function toboolean(str,tolerant)
+    if tolerant then
+        local tstr = type(str)
+        if tstr == "string" then
+            return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t"
+        elseif tstr == "number" then
+            return tonumber(str) ~= 0
+        elseif tstr == "nil" then
+            return false
+        else
+            return str
+        end
+    elseif str == "true" then
+        return true
+    elseif str == "false" then
+        return false
+    else
+        return str
+    end
+end
+
+function string.is_boolean(str)
+    if type(str) == "string" then
+        if str == "true" or str == "yes" or str == "on" or str == "t" then
+            return true
+        elseif str == "false" or str == "no" or str == "off" or str == "f" then
+            return false
+        end
+    end
+    return nil
+end
+
+function boolean.alwaystrue()
+    return true
+end
+
+function boolean.falsetrue()
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-math'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan
+
+if not math.round then
+    function math.round(x)
+        return floor(x + 0.5)
+    end
+end
+
+if not math.div then
+    function math.div(n,m)
+        return floor(n/m)
+    end
+end
+
+if not math.mod then
+    function math.mod(n,m)
+        return n % m
+    end
+end
+
+local pipi = 2*math.pi/360
+
+function math.sind(d)
+    return sin(d*pipi)
+end
+
+function math.cosd(d)
+    return cos(d*pipi)
+end
+
+function math.tand(d)
+    return tan(d*pipi)
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['l-utils'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- hm, quite unreadable
+
+if not utils        then utils        = { } end
+if not utils.merger then utils.merger = { } end
+if not utils.lua    then utils.lua    = { } end
+
+utils.merger.m_begin = "begin library merge"
+utils.merger.m_end   = "end library merge"
+utils.merger.pattern =
+    "%c+" ..
+    "%-%-%s+" .. utils.merger.m_begin ..
+    "%c+(.-)%c+" ..
+    "%-%-%s+" .. utils.merger.m_end ..
+    "%c+"
+
+function utils.merger._self_fake_()
+    return
+        "-- " .. "created merged file" .. "\n\n" ..
+        "-- " .. utils.merger.m_begin  .. "\n\n" ..
+        "-- " .. utils.merger.m_end    .. "\n\n"
+end
+
+function utils.report(...)
+    print(...)
+end
+
+utils.merger.strip_comment = true
+
+function utils.merger._self_load_(name)
+    local f, data = io.open(name), ""
+    if f then
+        utils.report("reading merge from %s",name)
+        data = f:read("*all")
+        f:close()
+    else
+        utils.report("unknown file to merge %s",name)
+    end
+    if data and utils.merger.strip_comment then
+        -- saves some 20K
+        data = data:gsub("%-%-~[^\n\r]*[\r\n]", "")
+    end
+    return data or ""
+end
+
+function utils.merger._self_save_(name, data)
+    if data ~= "" then
+        local f = io.open(name,'w')
+        if f then
+            utils.report("saving merge from %s",name)
+            f:write(data)
+            f:close()
+        end
+    end
+end
+
+function utils.merger._self_swap_(data,code)
+    if data ~= "" then
+        return (data:gsub(utils.merger.pattern, function(s)
+            return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n"
+        end, 1))
+    else
+        return ""
+    end
+end
+
+--~ stripper:
+--~
+--~ data = string.gsub(data,"%-%-~[^\n]*\n","")
+--~ data = string.gsub(data,"\n\n+","\n")
+
+function utils.merger._self_libs_(libs,list)
+    local result, f, frozen = { }, nil, false
+    result[#result+1] = "\n"
+    if type(libs) == 'string' then libs = { libs } end
+    if type(list) == 'string' then list = { list } end
+    local foundpath = nil
+    for _, lib in ipairs(libs) do
+        for _, pth in ipairs(list) do
+            pth = string.gsub(pth,"\\","/") -- file.clean_path
+            utils.report("checking library path %s",pth)
+            local name = pth .. "/" .. lib
+            if lfs.isfile(name) then
+                foundpath = pth
+            end
+        end
+        if foundpath then break end
+    end
+    if foundpath then
+        utils.report("using library path %s",foundpath)
+        local right, wrong = { }, { }
+        for _, lib in ipairs(libs) do
+            local fullname = foundpath .. "/" .. lib
+            if lfs.isfile(fullname) then
+            --  right[#right+1] = lib
+                utils.report("merging library %s",fullname)
+                result[#result+1] = "do -- create closure to overcome 200 locals limit"
+                result[#result+1] = io.loaddata(fullname,true)
+                result[#result+1] = "end -- of closure"
+            else
+            --  wrong[#wrong+1] = lib
+                utils.report("no library %s",fullname)
+            end
+        end
+        if #right > 0 then
+            utils.report("merged libraries: %s",table.concat(right," "))
+        end
+        if #wrong > 0 then
+            utils.report("skipped libraries: %s",table.concat(wrong," "))
+        end
+    else
+        utils.report("no valid library path found")
+    end
+    return table.concat(result, "\n\n")
+end
+
+function utils.merger.selfcreate(libs,list,target)
+    if target then
+        utils.merger._self_save_(
+            target,
+            utils.merger._self_swap_(
+                utils.merger._self_fake_(),
+                utils.merger._self_libs_(libs,list)
+            )
+        )
+    end
+end
+
+function utils.merger.selfmerge(name,libs,list,target)
+    utils.merger._self_save_(
+        target or name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            utils.merger._self_libs_(libs,list)
+        )
+    )
+end
+
+function utils.merger.selfclean(name)
+    utils.merger._self_save_(
+        name,
+        utils.merger._self_swap_(
+            utils.merger._self_load_(name),
+            ""
+        )
+    )
+end
+
+function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true
+ -- utils.report("compiling",luafile,"into",lucfile)
+    os.remove(lucfile)
+    local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile)
+    if strip ~= false then
+        command = "-s " .. command
+    end
+    local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0)
+    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+     -- utils.report("removing",luafile)
+        os.remove(luafile)
+    end
+    return done
+end
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-tab'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
The parser used here is inspired by the variant discussed in the lua book, but
+handles comment and processing instructions, has a different structure, provides
+parent access; a first version used different trickery but was less optimized to we
+went this route. First we had a find based parser, now we have an  based one.
+The find based parser can be found in l-xml-edu.lua along with other older code.

+
+
Expecially the lpath code is experimental, we will support some of xpath, but
+only things that make sense for us; as compensation it is possible to hook in your
+own functions. Apart from preprocessing content for  we also need
+this module for process management, like handling  and 
+files.

+
+
+a/b/c /*/c
+a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n)
+a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n)
+
+
+
Beware, the interface may change. For instance at, ns, tg, dt may get more
+verbose names. Once the code is stable we will also remove some tracing and
+optimize the code.

+--ldx]]--
+
+xml = xml or { }
+
+--~ local xml = xml
+
+local concat, remove, insert = table.concat, table.remove, table.insert
+local type, next, setmetatable = type, next, setmetatable
+local format, lower, find = string.format, string.lower, string.find
+
+--[[ldx--
+
This module can be used stand alone but also inside  in
+which case it hooks into the tracker code. Therefore we provide a few
+functions that set the tracers.

+--ldx]]--
+
+local trace_remap = false
+
+if trackers then
+    trackers.register("xml.remap", function(v) trace_remap = v end)
+end
+
+function xml.settrace(str,value)
+    if str == "remap" then
+        trace_remap = value or false
+    end
+end
+
+--[[ldx--
+
First a hack to enable namespace resolving. A namespace is characterized by
+a . The following function associates a namespace prefix with a
+pattern. We use , which in this case is more than twice as fast as a
+find based solution where we loop over an array of patterns. Less code and
+much cleaner.

+--ldx]]--
+
+xml.xmlns = xml.xmlns or { }
+
+local check = lpeg.P(false)
+local parse = check
+
+--[[ldx--
+
The next function associates a namespace prefix with an . This
+normally happens independent of parsing.

+
+
+xml.registerns("mml","mathml")
+
+--ldx]]--
+
+function xml.registerns(namespace, pattern) -- pattern can be an lpeg
+    check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace
+    parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) }
+end
+
+--[[ldx--
+
The next function also registers a namespace, but this time we map a
+given namespace prefix onto a registered one, using the given
+. This used for attributes like xmlns:m.

+
+
+xml.checkns("m","http://www.w3.org/mathml")
+
+--ldx]]--
+
+function xml.checkns(namespace,url)
+    local ns = parse:match(lower(url))
+    if ns and namespace ~= ns then
+        xml.xmlns[namespace] = ns
+    end
+end
+
+--[[ldx--
+
Next we provide a way to turn an  into a registered
+namespace. This used for the xmlns attribute.

+
+
+resolvedns = xml.resolvens("http://www.w3.org/mathml")
+
+
+This returns mml.
+--ldx]]--
+
+function xml.resolvens(url)
+     return parse:match(lower(url)) or ""
+end
+
+--[[ldx--
+
A namespace in an element can be remapped onto the registered
+one efficiently by using the xml.xmlns table.

+--ldx]]--
+
+--[[ldx--
+
This version uses . We follow the same approach as before, stack and top and
+such. This version is about twice as fast which is mostly due to the fact that
+we don't have to prepare the stream for cdata, doctype etc etc. This variant is
+is dedicated to Luigi Scarso, who challenged me with 40 megabyte  files that
+took 12.5 seconds to load (1.5 for file io and the rest for tree building). With
+the  implementation we got that down to less 7.3 seconds. Loading the 14
+ interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.

+
+
Next comes the parser. The rather messy doctype definition comes in many
+disguises so it is no surprice that later on have to dedicate quite some
+ code to it.

+
+
+
+
+
+
+
+
+
+
+
The code may look a bit complex but this is mostly due to the fact that we
+resolve namespaces and attach metatables. There is only one public function:

+
+
+local x = xml.convert(somestring)
+
+
+
An optional second boolean argument tells this function not to create a root
+element.

+--ldx]]--
+
+xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes
+
+-- not just one big nested table capture (lpeg overflow)
+
+local nsremap, resolvens = xml.xmlns, xml.resolvens
+
+local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {}
+
+local mt = { __tostring = xml.text }
+
+function xml.check_error(top,toclose)
+    return ""
+end
+
+local strip   = false
+local cleanup = false
+
+function xml.set_text_cleanup(fnc)
+    cleanup = fnc
+end
+
+local function add_attribute(namespace,tag,value)
+    if cleanup and #value > 0 then
+        value = cleanup(value) -- new
+    end
+    if tag == "xmlns" then
+        xmlns[#xmlns+1] = resolvens(value)
+        at[tag] = value
+    elseif namespace == "xmlns" then
+        xml.checkns(tag,value)
+        at["xmlns:" .. tag] = value
+    else
+        at[tag] = value
+    end
+end
+
+local function add_begin(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
+    top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] }
+    setmetatable(top, mt)
+    dt = top.dt
+    stack[#stack+1] = top
+    at = { }
+end
+
+local function add_end(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local toclose = remove(stack)
+    top = stack[#stack]
+    if #stack < 1 then
+        errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "")
+    elseif toclose.tg ~= tag then -- no namespace check
+        errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "")
+    end
+    dt = top.dt
+    dt[#dt+1] = toclose
+    dt[0] = top
+    if toclose.at.xmlns then
+        remove(xmlns)
+    end
+end
+
+local function add_empty(spacing, namespace, tag)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace
+    top = stack[#stack]
+    dt = top.dt
+    local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top }
+    dt[#dt+1] = t
+    setmetatable(t, mt)
+    if at.xmlns then
+        remove(xmlns)
+    end
+    at = { }
+end
+
+local function add_text(text)
+    if cleanup and #text > 0 then
+        dt[#dt+1] = cleanup(text)
+    else
+        dt[#dt+1] = text
+    end
+end
+
+local function add_special(what, spacing, text)
+    if #spacing > 0 then
+        dt[#dt+1] = spacing
+    end
+    if strip and (what == "@cm@" or what == "@dt@") then
+        -- forget it
+    else
+        dt[#dt+1] = { special=true, ns="", tg=what, dt={text} }
+    end
+end
+
+local function set_message(txt)
+    errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","")
+end
+
+local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V
+
+local space            = S(' \r\n\t')
+local open             = P('<')
+local close            = P('>')
+local squote           = S("'")
+local dquote           = S('"')
+local equal            = P('=')
+local slash            = P('/')
+local colon            = P(':')
+local valid            = R('az', 'AZ', '09') + S('_-.')
+local name_yes         = C(valid^1) * colon * C(valid^1)
+local name_nop         = C(P(true)) * C(valid^1)
+local name             = name_yes + name_nop
+
+local utfbom           = P('\000\000\254\255') + P('\255\254\000\000') +
+                         P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture
+
+local spacing          = C(space^0)
+local justtext         = C((1-open)^1)
+local somespace        = space^1
+local optionalspace    = space^0
+
+local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
+local attribute        = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute
+local attributes       = attribute^0
+
+local text             = justtext / add_text
+local balanced         = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example
+
+local emptyelement     = (spacing * open         * name * attributes * optionalspace * slash * close) / add_empty
+local beginelement     = (spacing * open         * name * attributes * optionalspace         * close) / add_begin
+local endelement       = (spacing * open * slash * name              * optionalspace         * close) / add_end
+
+local begincomment     = open * P("!--")
+local endcomment       = P("--") * close
+local begininstruction = open * P("?")
+local endinstruction   = P("?") * close
+local begincdata       = open * P("![CDATA[")
+local endcdata         = P("]]") * close
+
+local someinstruction  = C((1 - endinstruction)^0)
+local somecomment      = C((1 - endcomment    )^0)
+local somecdata        = C((1 - endcdata      )^0)
+
+local function entity(k,v) entities[k] = v end
+
+local begindoctype     = open * P("!DOCTYPE")
+local enddoctype       = close
+local beginset         = P("[")
+local endset           = P("]")
+local doctypename      = C((1-somespace)^0)
+local elementdoctype   = optionalspace * P("Packaging data in an xml like table is done with the following
+function. Maybe it will go away (when not used).

+--ldx]]--
+
+function xml.is_valid(root)
+    return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er
+end
+
+function xml.package(tag,attributes,data)
+    local ns, tg = tag:match("^(.-):?([^:]+)$")
+    local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} }
+    setmetatable(t, mt)
+    return t
+end
+
+function xml.is_valid(root)
+    return root and not root.error
+end
+
+xml.error_handler = (logs and logs.report) or (input and logs.report) or print
+
+--[[ldx--
+
We cannot load an  from a filehandle so we need to load
+the whole file first. The function accepts a string representing
+a filename or a file handle.

+--ldx]]--
+
+function xml.load(filename)
+    if type(filename) == "string" then
+        local f = io.open(filename,'r')
+        if f then
+            local root = xml.convert(f:read("*all"))
+            f:close()
+            return root
+        else
+            return xml.convert("")
+        end
+    elseif filename then -- filehandle
+        return xml.convert(filename:read("*all"))
+    else
+        return xml.convert("")
+    end
+end
+
+--[[ldx--
+
When we inject new elements, we need to convert strings to
+valid trees, which is what the next function does.

+--ldx]]--
+
+function xml.toxml(data)
+    if type(data) == "string" then
+        local root = { xml.convert(data,true) }
+        return (#root > 1 and root) or root[1]
+    else
+        return data
+    end
+end
+
+--[[ldx--
+
For copying a tree we use a dedicated function instead of the
+generic table copier. Since we know what we're dealing with we
+can speed up things a bit. The second argument is not to be used!

+--ldx]]--
+
+function copy(old,tables)
+    if old then
+        tables = tables or { }
+        local new = { }
+        if not tables[old] then
+            tables[old] = new
+        end
+        for k,v in pairs(old) do
+            new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v
+        end
+        local mt = getmetatable(old)
+        if mt then
+            setmetatable(new,mt)
+        end
+        return new
+    else
+        return { }
+    end
+end
+
+xml.copy = copy
+
+--[[ldx--
+
In  serializing the tree or parts of the tree is a major
+actitivity which is why the following function is pretty optimized resulting
+in a few more lines of code than needed. The variant that uses the formatting
+function for all components is about 15% slower than the concatinating
+alternative.

+--ldx]]--
+
+-- todo: add  when not present
+
+local fallbackhandle = (tex and tex.sprint) or io.write
+
+local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands)
+    if not e then
+        return
+    elseif not nocommands then
+        local ec = e.command
+        if ec ~= nil then -- we can have all kind of types
+            if e.special then
+                local etg, edt = e.tg, e.dt
+                local spc = specialconverter and specialconverter[etg]
+                if spc then
+                    local result = spc(edt[1])
+                    if result then
+                        handle(result)
+                        return
+                    else
+                        -- no need to handle any further
+                    end
+                end
+            end
+            local xc = xml.command
+            if xc then
+                xc(e,ec)
+                return
+            end
+        end
+    end
+    handle = handle or fallbackhandle
+    local etg = e.tg
+    if etg then
+        if e.special then
+            local edt = e.dt
+            local spc = specialconverter and specialconverter[etg]
+            if spc then
+                local result = spc(edt[1])
+                if result then
+                    handle(result)
+                else
+                    -- no need to handle any further
+                end
+            elseif etg == "@pi@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@cm@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@cd@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@dt@" then
+            --  handle(format("",edt[1]))
+                handle("")
+            elseif etg == "@rt@" then
+                serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands)
+            end
+        else
+            local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn
+            local ats = eat and next(eat) and { } -- type test maybe faster
+            if ats then
+                if attributeconverter then
+                    for k,v in next, eat do
+                        ats[#ats+1] = format('%s=%q',k,attributeconverter(v))
+                    end
+                else
+                    for k,v in next, eat do
+                        ats[#ats+1] = format('%s=%q',k,v)
+                    end
+                end
+            end
+            if ern and trace_remap and ern ~= ens then
+                ens = ern
+            end
+            if ens ~= "" then
+                if edt and #edt > 0 then
+                    if ats then
+                    --  handle(format("<%s:%s %s>",ens,etg,concat(ats," ")))
+                        handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">")
+                    else
+                    --  handle(format("<%s:%s>",ens,etg))
+                        handle("<" .. ens .. ":" .. etg .. ">")
+                    end
+                    for i=1,#edt do
+                        local e = edt[i]
+                        if type(e) == "string" then
+                            if textconverter then
+                                handle(textconverter(e))
+                            else
+                                handle(e)
+                            end
+                        else
+                            serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                        end
+                    end
+                --  handle(format("",ens,etg))
+                    handle("")
+                else
+                    if ats then
+                    --  handle(format("<%s:%s %s/>",ens,etg,concat(ats," ")))
+                        handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>")
+                    else
+                    --  handle(format("<%s:%s/>",ens,etg))
+                        handle("<" .. ens .. ":" .. etg .. "/>")
+                    end
+                end
+            else
+                if edt and #edt > 0 then
+                    if ats then
+                    --  handle(format("<%s %s>",etg,concat(ats," ")))
+                        handle("<" .. etg .. " " .. concat(ats," ") .. ">")
+                    else
+                    --  handle(format("<%s>",etg))
+                        handle("<" .. etg .. ">")
+                    end
+                    for i=1,#edt do
+                        local ei = edt[i]
+                        if type(ei) == "string" then
+                            if textconverter then
+                                handle(textconverter(ei))
+                            else
+                                handle(ei)
+                            end
+                        else
+                            serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
+                        end
+                    end
+                --  handle(format("",etg))
+                    handle("")
+                else
+                    if ats then
+                    --  handle(format("<%s %s/>",etg,concat(ats," ")))
+                        handle("<" .. etg .. " " .. concat(ats," ") .. "/>")
+                    else
+                    --  handle(format("<%s/>",etg))
+                        handle("<" .. etg .. "/>")
+                    end
+                end
+            end
+        end
+    elseif type(e) == "string" then
+        if textconverter then
+            handle(textconverter(e))
+        else
+            handle(e)
+        end
+    else
+        for i=1,#e do
+            local ei = e[i]
+            if type(ei) == "string" then
+                if textconverter then
+                    handle(textconverter(ei))
+                else
+                    handle(ei)
+                end
+            else
+                serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands)
+            end
+        end
+    end
+end
+
+xml.serialize = serialize
+
+function xml.checkbom(root) -- can be made faster
+    if root.ri then
+        local dt, found = root.dt, false
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then
+                found = true
+                break
+            end
+        end
+        if not found then
+            insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } )
+            insert(dt, 2, "\n" )
+        end
+    end
+end
+
+--[[ldx--
+
At the cost of some 25% runtime overhead you can first convert the tree to a string
+and then handle the lot.

+--ldx]]--
+
+function xml.tostring(root) -- 25% overhead due to collecting
+    if root then
+        if type(root) == 'string' then
+            return root
+        elseif next(root) then -- next is faster than type (and >0 test)
+            local result = { }
+            serialize(root,function(s) result[#result+1] = s end)
+            return concat(result,"")
+        end
+    end
+    return ""
+end
+
+--[[ldx--
+
The next function operated on the content only and needs a handle function
+that accepts a string.

+--ldx]]--
+
+function xml.string(e,handle)
+    if not handle or (e.special and e.tg ~= "@rt@") then
+        -- nothing
+    elseif e.tg then
+        local edt = e.dt
+        if edt then
+            for i=1,#edt do
+                xml.string(edt[i],handle)
+            end
+        end
+    else
+        handle(e)
+    end
+end
+
+--[[ldx--
+
How you deal with saving data depends on your preferences. For a 40 MB database
+file the timing on a 2.3 Core Duo are as follows (time in seconds):

+
+
+1.3 : load data from file to string
+6.1 : convert string into tree
+5.3 : saving in file using xmlsave
+6.8 : converting to string using xml.tostring
+3.6 : saving converted string in file
+
+
+
The save function is given below.

+--ldx]]--
+
+function xml.save(root,name)
+    local f = io.open(name,"w")
+    if f then
+        xml.serialize(root,function(s) f:write(s) end)
+        f:close()
+    end
+end
+
+--[[ldx--
+
A few helpers:

+--ldx]]--
+
+function xml.body(root)
+    return (root.ri and root.dt[root.ri]) or root
+end
+
+function xml.text(root)
+    return (root and xml.tostring(root)) or ""
+end
+
+function xml.content(root) -- bugged
+    return (root and root.dt and xml.tostring(root.dt)) or ""
+end
+
+function xml.isempty(root, pattern)
+    if pattern == "" or pattern == "*" then
+        pattern = nil
+    end
+    if pattern then
+        -- todo
+        return false
+    else
+        return not root or not root.dt or #root.dt == 0 or root.dt == ""
+    end
+end
+
+--[[ldx--
+
The next helper erases an element but keeps the table as it is,
+and since empty strings are not serialized (effectively) it does
+not harm. Copying the table would take more time. Usage:

+
+
+dt[k] = xml.empty() or xml.empty(dt,k)
+
+--ldx]]--
+
+function xml.empty(dt,k)
+    if dt and k then
+        dt[k] = ""
+        return dt[k]
+    else
+        return ""
+    end
+end
+
+--[[ldx--
+
The next helper assigns a tree (or string). Usage:

+
+
+dt[k] = xml.assign(root) or xml.assign(dt,k,root)
+
+--ldx]]--
+
+function xml.assign(dt,k,root)
+    if dt and k then
+        dt[k] = (type(root) == "table" and xml.body(root)) or root
+        return dt[k]
+    else
+        return xml.body(root)
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-pth'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local concat, remove, insert = table.concat, table.remove, table.insert
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find
+
+--[[ldx--
+
This module can be used stand alone but also inside  in
+which case it hooks into the tracker code. Therefore we provide a few
+functions that set the tracers. Here we overload a previously defined
+function.

+--ldx]]--
+
+local trace_lpath = false
+
+if trackers then
+    trackers.register("xml.lpath", function(v) trace_lpath = v end)
+end
+
+local settrace = xml.settrace -- lxml-tab
+
+function xml.settrace(str,value)
+    if str == "lpath" then
+        trace_lpath = value or false
+    else
+        settrace(str,value) -- lxml-tab
+    end
+end
+
+--[[ldx--
+
We've now arrived at an intersting part: accessing the tree using a subset
+of  and since we're not compatible we call it . We
+will explain more about its usage in other documents.

+--ldx]]--
+
+local lpathcalls  = 0 -- statistics
+local lpathcached = 0 -- statistics
+
+xml.functions   = xml.functions   or { }
+xml.expressions = xml.expressions or { }
+
+local functions   = xml.functions
+local expressions = xml.expressions
+
+local actions = {
+    [10] = "stay",
+    [11] = "parent",
+    [12] = "subtree root",
+    [13] = "document root",
+    [14] = "any",
+    [15] = "many",
+    [16] = "initial",
+    [20] = "match",
+    [21] = "match one of",
+    [22] = "match and attribute eq",
+    [23] = "match and attribute ne",
+    [24] = "match one of and attribute eq",
+    [25] = "match one of and attribute ne",
+    [27] = "has attribute",
+    [28] = "has value",
+    [29] = "fast match",
+    [30] = "select",
+    [31] = "expression",
+    [40] = "processing instruction",
+}
+
+-- a rather dumb lpeg
+
+local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
+
+-- instead of using functions we just parse a few names which saves a call
+-- later on
+
+local lp_position  = P("position()") / "ps"
+local lp_index     = P("index()")    / "id"
+local lp_text      = P("text()")     / "tx"
+local lp_name      = P("name()")     / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')"
+local lp_tag       = P("tag()")      / "tg" -- (rt.tg or '')
+local lp_ns        = P("ns()")       / "ns" -- (rt.ns or '')
+local lp_noequal   = P("!=")         / "~=" + P("<=") + P(">=") + P("==")
+local lp_doequal   = P("=")          / "=="
+local lp_attribute = P("@")          / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')")
+
+local lp_lua_function  = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling
+    return t .. "("
+end
+
+local lp_function  = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling
+    if expressions[t] then
+        return "expressions." .. t .. "("
+    else
+        return "expressions.error("
+    end
+end
+
+local lparent  = lpeg.P("(")
+local rparent  = lpeg.P(")")
+local noparent = 1 - (lparent+rparent)
+local nested   = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent}
+local value    = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"}
+
+-- if we use a dedicated namespace then we don't need to pass rt and k
+
+local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s)
+    if expressions[t] then
+        if s then
+            return "expressions." .. t .. "(r,k," .. s ..")"
+        else
+            return "expressions." .. t .. "(r,k)"
+        end
+    else
+        return "expressions.error(" .. t .. ")"
+    end
+end
+
+local converter = lpeg.Cs ( (
+    lp_position +
+    lp_index +
+    lp_text + lp_name + -- fast one
+    lp_special +
+    lp_noequal + lp_doequal +
+    lp_attribute +
+    lp_lua_function +
+    lp_function +
+1 )^1 )
+
+-- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1
+
+local template = [[
+    return function(expressions,r,d,k,e,dt,ns,tg,id,ps)
+        local at, tx = e.at or { }, dt[1] or ""
+        return %s
+    end
+]]
+
+local function make_expression(str)
+    str = converter:match(str)
+    return str, loadstring(format(template,str))()
+end
+
+local map = { }
+
+local space                    = S(' \r\n\t')
+local squote                   = S("'")
+local dquote                   = S('"')
+local lparent                  = P('(')
+local rparent                  = P(')')
+local atsign                   = P('@')
+local lbracket                 = P('[')
+local rbracket                 = P(']')
+local exclam                   = P('!')
+local period                   = P('.')
+local eq                       = P('==') + P('=')
+local ne                       = P('<>') + P('!=')
+local star                     = P('*')
+local slash                    = P('/')
+local colon                    = P(':')
+local bar                      = P('|')
+local hat                      = P('^')
+local valid                    = R('az', 'AZ', '09') + S('_-')
+local name_yes                 = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:*
+local name_nop                 = Cc("*") * C(valid^1)
+local name                     = name_yes + name_nop
+local number                   = C((S('+-')^0 * R('09')^1)) / tonumber
+local names                    = (bar^0 * name)^1
+local morenames                = name * (bar^0 * name)^1
+local instructiontag           = P('pi::')
+local spacing                  = C(space^0)
+local somespace                = space^1
+local optionalspace            = space^0
+local text                     = C(valid^0)
+local value                    = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote)
+local empty                    = 1-slash
+
+local is_eq                    = lbracket * atsign * name * eq * value * rbracket
+local is_ne                    = lbracket * atsign * name * ne * value * rbracket
+local is_attribute             = lbracket * atsign * name              * rbracket
+local is_value                 = lbracket *          value             * rbracket
+local is_number                = lbracket *          number            * rbracket
+
+local nobracket                = 1-(lbracket+rbracket)  -- must be improved
+local is_expression            = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket
+
+local is_expression            = lbracket * (C(nobracket^1))/make_expression * rbracket
+
+local is_one                   =          name
+local is_none                  = exclam * name
+local is_one_of                =          ((lparent * names * rparent) + morenames)
+local is_none_of               = exclam * ((lparent * names * rparent) + morenames)
+
+local stay                     = (period                )
+local parent                   = (period * period       ) / function(   ) map[#map+1] = { 11             } end
+local subtreeroot              = (slash + hat           ) / function(   ) map[#map+1] = { 12             } end
+local documentroot             = (hat * hat             ) / function(   ) map[#map+1] = { 13             } end
+local any                      = (star                  ) / function(   ) map[#map+1] = { 14             } end
+local many                     = (star * star           ) / function(   ) map[#map+1] = { 15             } end
+local initial                  = (hat * hat * hat       ) / function(   ) map[#map+1] = { 16             } end
+
+local match                    = (is_one                ) / function(...) map[#map+1] = { 20, true , ... } end
+local match_one_of             = (is_one_of             ) / function(...) map[#map+1] = { 21, true , ... } end
+local dont_match               = (is_none               ) / function(...) map[#map+1] = { 20, false, ... } end
+local dont_match_one_of        = (is_none_of            ) / function(...) map[#map+1] = { 21, false, ... } end
+
+local match_and_eq             = (is_one     * is_eq    ) / function(...) map[#map+1] = { 22, true , ... } end
+local match_and_ne             = (is_one     * is_ne    ) / function(...) map[#map+1] = { 23, true , ... } end
+local dont_match_and_eq        = (is_none    * is_eq    ) / function(...) map[#map+1] = { 22, false, ... } end
+local dont_match_and_ne        = (is_none    * is_ne    ) / function(...) map[#map+1] = { 23, false, ... } end
+
+local match_one_of_and_eq      = (is_one_of  * is_eq    ) / function(...) map[#map+1] = { 24, true , ... } end
+local match_one_of_and_ne      = (is_one_of  * is_ne    ) / function(...) map[#map+1] = { 25, true , ... } end
+local dont_match_one_of_and_eq = (is_none_of * is_eq    ) / function(...) map[#map+1] = { 24, false, ... } end
+local dont_match_one_of_and_ne = (is_none_of * is_ne    ) / function(...) map[#map+1] = { 25, false, ... } end
+
+local has_attribute            = (is_one  * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end
+local has_value                = (is_one  * is_value    ) / function(...) map[#map+1] = { 28, true , ... } end
+local dont_has_attribute       = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end
+local dont_has_value           = (is_none * is_value    ) / function(...) map[#map+1] = { 28, false, ... } end
+local position                 = (is_one  * is_number   ) / function(...) map[#map+1] = { 30, true,  ... } end
+local dont_position            = (is_none * is_number   ) / function(...) map[#map+1] = { 30, false, ... } end
+
+local expression               = (is_one  * is_expression)/ function(...) map[#map+1] = { 31, true,  ... } end
+local dont_expression          = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end
+
+local self_expression          = (         is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
+                                                                          map[#map+1] = { 31, true,  "*", "*", ... } end
+local dont_self_expression     = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end
+                                                                          map[#map+1] = { 31, false, "*", "*", ... } end
+
+local instruction              = (instructiontag * text ) / function(...) map[#map+1] = { 40,        ... } end
+local nothing                  = (empty                 ) / function(   ) map[#map+1] = { 15             } end -- 15 ?
+local crap                     = (1-slash)^1
+
+-- a few ugly goodies:
+
+local docroottag               = P('^^')             / function(   ) map[#map+1] = { 12             } end
+local subroottag               = P('^')              / function(   ) map[#map+1] = { 13             } end
+local roottag                  = P('root::')         / function(   ) map[#map+1] = { 12             } end
+local parenttag                = P('parent::')       / function(   ) map[#map+1] = { 11             } end
+local childtag                 = P('child::')
+local selftag                  = P('self::')
+
+-- there will be more and order will be optimized
+
+local selector = (
+    instruction +
+--  many + any + -- brrr, not here !
+    parent + stay +
+    dont_position + position +
+    dont_match_one_of_and_eq + dont_match_one_of_and_ne +
+    match_one_of_and_eq + match_one_of_and_ne +
+    dont_match_and_eq + dont_match_and_ne +
+    match_and_eq + match_and_ne +
+    dont_expression + expression +
+    dont_self_expression + self_expression +
+    has_attribute + has_value +
+    dont_match_one_of + match_one_of +
+    dont_match + match +
+    many + any +
+    crap + empty
+)
+
+local grammar = P { "startup",
+    startup  = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"),
+    followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1,
+}
+
+local function compose(str)
+    if not str or str == "" then
+        -- wildcard
+        return true
+    elseif str == '/' then
+        -- root
+        return false
+    else
+        map = { }
+        grammar:match(str)
+        if #map == 0 then
+            return true
+        else
+            local m = map[1][1]
+            if #map == 1 then
+                if m == 14 or m == 15 then
+                    -- wildcard
+                    return true
+                elseif m == 12 then
+                    -- root
+                    return false
+                end
+            elseif #map == 2 and m == 12 and map[2][1] == 20 then
+            --  return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } }
+                map[2][1] = 29
+                return { map[2] }
+            end
+            if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then
+                insert(map, 1, { 16 })
+            end
+        --  print(gsub(table.serialize(map),"[ \n]+"," "))
+            return map
+        end
+    end
+end
+
+local cache = { }
+
+function xml.lpath(pattern,trace)
+    lpathcalls = lpathcalls + 1
+    if type(pattern) == "string" then
+        local result = cache[pattern]
+        if result == nil then -- can be false which is valid -)
+            result = compose(pattern)
+            cache[pattern] = result
+            lpathcached = lpathcached + 1
+        end
+        if trace or trace_lpath then
+            xml.lshow(result)
+        end
+        return result
+    else
+        return pattern
+    end
+end
+
+function xml.cached_patterns()
+    return cache
+end
+
+--  we run out of locals (limited to 200)
+--
+--  local fallbackreport = (texio and texio.write) or io.write
+
+function xml.lshow(pattern,report)
+--      report = report or fallbackreport
+    report = report or (texio and texio.write) or io.write
+    local lp = xml.lpath(pattern)
+    if lp == false then
+        report(" -: root\n")
+    elseif lp == true then
+        report(" -: wildcard\n")
+    else
+        if type(pattern) == "string" then
+            report(format("pattern: %s\n",pattern))
+        end
+        for k=1,#lp do
+            local v = lp[k]
+            if #v > 1 then
+                local t = { }
+                for i=2,#v do
+                    local vv = v[i]
+                    if type(vv) == "string" then
+                        t[#t+1] = (vv ~= "" and vv) or "#"
+                    elseif type(vv) == "boolean" then
+                        t[#t+1] = (vv and "==") or "<>"
+                    end
+                end
+                report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," ")))
+            else
+                report(format("%2i: %s %s\n", k,v[1],actions[v[1]]))
+            end
+        end
+    end
+end
+
+function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e
+    local t = { ... }
+--      local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport
+    local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write
+    if e == nil then
+        report("\n")
+    elseif type(e) ~= "table" then
+        report(tostring(e))
+    elseif e.tg then
+        report(tostring(e) .. "\n")
+    else
+        for i=1,#e do
+            report(tostring(e[i]) .. "\n")
+        end
+    end
+end
+
+--[[ldx--
+
An  is converted to a table with instructions for traversing the
+tree. Hoever, simple cases are signaled by booleans. Because we don't know in
+advance what we want to do with the found element the handle gets three arguments:

+
+
+r : the root element of the data table
+d : the data table of the result
+t : the index in the data table of the result
+
+
+
 Access to the root and data table makes it possible to construct insert and delete
+functions.

+--ldx]]--
+
+local functions   = xml.functions
+local expressions = xml.expressions
+
+expressions.contains = string.find
+expressions.find     = string.find
+expressions.upper    = string.upper
+expressions.lower    = string.lower
+expressions.number   = tonumber
+expressions.boolean  = toboolean
+
+expressions.oneof = function(s,...) -- slow
+    local t = {...} for i=1,#t do if s == t[i] then return true end end return false
+end
+
+expressions.error = function(str)
+    xml.error_handler("unknown function in lpath expression",str or "?")
+    return false
+end
+
+functions.text = function(root,k,n) -- unchecked, maybe one deeper
+    local t = type(t)
+    if t == "string" then
+        return t
+    else -- todo n
+        local rdt = root.dt
+        return (rdt and rdt[k]) or root[k] or ""
+    end
+end
+
+functions.name = function(d,k,n) -- ns + tg
+    local found = false
+    n = n or 0
+    if not k then
+        -- not found
+    elseif n == 0 then
+        local dk = d[k]
+        found = dk and (type(dk) == "table") and dk
+    elseif n < 0 then
+        for i=k-1,1,-1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == -1 then
+                    found = di
+                    break
+                else
+                    n = n + 1
+                end
+            end
+        end
+    else
+        for i=k+1,#d,1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == 1 then
+                    found = di
+                    break
+                else
+                    n = n - 1
+                end
+            end
+        end
+    end
+    if found then
+        local ns, tg = found.rn or found.ns or "", found.tg
+        if ns ~= "" then
+            return ns .. ":" .. tg
+        else
+            return tg
+        end
+    else
+        return ""
+    end
+end
+
+functions.tag = function(d,k,n) -- only tg
+    local found = false
+    n = n or 0
+    if not k then
+        -- not found
+    elseif n == 0 then
+        local dk = d[k]
+        found = dk and (type(dk) == "table") and dk
+    elseif n < 0 then
+        for i=k-1,1,-1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == -1 then
+                    found = di
+                    break
+                else
+                    n = n + 1
+                end
+            end
+        end
+    else
+        for i=k+1,#d,1 do
+            local di = d[i]
+            if type(di) == "table" then
+                if n == 1 then
+                    found = di
+                    break
+                else
+                    n = n - 1
+                end
+            end
+        end
+    end
+    return (found and found.tg) or ""
+end
+
+expressions.text = functions.text
+expressions.name = functions.name
+expressions.tag  = functions.tag
+
+local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces
+    if not root then -- error
+        return false
+    elseif pattern == false then -- root
+        handle(root,root.dt,root.ri)
+        return false
+    elseif pattern == true then -- wildcard
+        local rootdt = root.dt
+        if rootdt then
+            local start, stop, step = 1, #rootdt, 1
+            if reverse then
+                start, stop, step = stop, start, -1
+            end
+            for k=start,stop,step do
+                if handle(root,rootdt,root.ri or k)            then return false end
+                if not traverse(rootdt[k],true,handle,reverse) then return false end
+            end
+        end
+        return false
+    elseif root.dt then
+        index = index or 1
+        local action = pattern[index]
+        local command = action[1]
+        if command == 29 then -- fast case /oeps
+            local rootdt = root.dt
+            for k=1,#rootdt do
+                local e = rootdt[k]
+                local tg = e.tg
+                if e.tg then
+                    local ns = e.rn or e.ns
+                    local ns_a, tg_a = action[3], action[4]
+                    local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a)
+                    if not action[2] then matched = not matched end
+                    if matched then
+                        if handle(root,rootdt,k) then return false end
+                    end
+                end
+            end
+        elseif command == 11 then -- parent
+            local ep = root.__p__ or parent
+            if index < #pattern then
+                if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+            elseif handle(root,rootdt,k) then
+                return false
+            end
+        else
+            if (command == 16 or command == 12) and index == 1 then -- initial
+            --  wildcard = true
+                wildcard = command == 16 -- ok?
+                index = index + 1
+                action = pattern[index]
+                command = action and action[1] or 0 -- something is wrong
+            end
+            if command == 11 then -- parent
+                local ep = root.__p__ or parent
+                if index < #pattern then
+                    if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+                elseif handle(root,rootdt,k) then
+                    return false
+                end
+            else
+                local rootdt = root.dt
+                local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1
+                if command == 30 then
+                    if action[5] < 0 then
+                        start, stop, step = stop, start, -1
+                        dn = -1
+                    end
+                elseif reverse and index == #pattern then
+                    start, stop, step = stop, start, -1
+                end
+                local idx = 0
+                local hsh = { } -- this will slooow down the lot
+                for k=start,stop,step do -- we used to have functions for all but a case is faster
+                    local e = rootdt[k]
+                    local ns, tg = e.rn or e.ns, e.tg
+                    if tg then
+                     -- we can optimize this for simple searches, but it probably does not pay off
+                        hsh[tg] = (hsh[tg] or 0) + 1
+                        idx = idx + 1
+                        if command == 30 then
+                            local ns_a, tg_a = action[3], action[4]
+                            if tg == tg_a then
+                                matched = ns_a == "*" or ns == ns_a
+                            elseif tg_a == '*' then
+                                matched, multiple = ns_a == "*" or ns == ns_a, true
+                            else
+                                matched = false
+                            end
+                            if not action[2] then matched = not matched end
+                            if matched then
+                                n = n + dn
+                                if n == action[5] then
+                                    if index == #pattern then
+                                        if handle(root,rootdt,root.ri or k) then return false end
+                                    else
+                                        if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                    end
+                                    break
+                                end
+                            elseif wildcard then
+                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                            end
+                        else
+                            local matched, multiple = false, false
+                            if command == 20 then -- match
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                            elseif command == 21 then -- match one of
+                                multiple = true
+                                for i=3,#action,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
+                                    end
+                                end
+                                if not action[2] then matched = not matched end
+                            elseif command == 22 then -- eq
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                matched = matched and e.at[action[6]] == action[7]
+                            elseif command == 23 then -- ne
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = mached and e.at[action[6]] ~= action[7]
+                            elseif command == 24 then -- one of eq
+                                multiple = true
+                                for i=3,#action-2,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
+                                    end
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[#action-1]] == action[#action]
+                            elseif command == 25 then -- one of ne
+                                multiple = true
+                                for i=3,#action-2,2 do
+                                    local ns_a, tg_a = action[i], action[i+1]
+                                    if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then
+                                        matched = true
+                                        break
+                                    end
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[#action-1]] ~= action[#action]
+                            elseif command == 27 then -- has attribute
+                                local ns_a, tg_a = action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and e.at[action[5]]
+                            elseif command == 28 then -- has value
+                                local edt, ns_a, tg_a = e.dt, action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                matched = matched and edt and edt[1] == action[5]
+                            elseif command == 31 then
+                                local edt, ns_a, tg_a = e.dt, action[3], action[4]
+                                if tg == tg_a then
+                                    matched = ns_a == "*" or ns == ns_a
+                                elseif tg_a == '*' then
+                                    matched, multiple = ns_a == "*" or ns == ns_a, true
+                                else
+                                    matched = false
+                                end
+                                if not action[2] then matched = not matched end
+                                if matched then
+                                    matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1)
+                                end
+                            end
+                            if matched then -- combine tg test and at test
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                    if wildcard then
+                                        if multiple then
+                                            if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                                        else
+                                         -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml
+                                            if not traverse(e,pattern,handle,reverse,index,root) then return false end
+                                        end
+                                    end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                end
+                            elseif command == 14 then -- any
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root) then return false end
+                                end
+                            elseif command == 15 then -- many
+                                if index == #pattern then
+                                    if handle(root,rootdt,root.ri or k) then return false end
+                                else
+                                    if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end
+                                end
+                            -- not here : 11
+                            elseif command == 11 then -- parent
+                                local ep = e.__p__ or parent
+                                if index < #pattern then
+                                    if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end
+                                elseif handle(root,rootdt,k) then
+                                    return false
+                                end
+                            elseif command == 40 and e.special and tg == "@pi@" then -- pi
+                                local pi = action[2]
+                                if pi ~= "" then
+                                    local pt = e.dt[1]
+                                    if pt and pt:find(pi) then
+                                        if handle(root,rootdt,k) then
+                                            return false
+                                        end
+                                    end
+                                elseif handle(root,rootdt,k) then
+                                    return false
+                                end
+                            elseif wildcard then
+                                if not traverse(e,pattern,handle,reverse,index,root,true) then return false end
+                            end
+                        end
+                    else
+                        -- not here : 11
+                        if command == 11 then -- parent
+                            local ep = e.__p__ or parent
+                            if index < #pattern then
+                                if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end
+                            elseif handle(root,rootdt,k) then
+                                return false
+                            end
+                            break -- else loop
+                        end
+                    end
+                end
+            end
+        end
+    end
+    return true
+end
+
+xml.traverse = traverse
+
+--[[ldx--
+
Next come all kind of locators and manipulators. The most generic function here
+is xml.filter(root,pattern). All registers functions in the filters namespace
+can be path of a search path, as in:

+
+
+local r, d, k = xml.filter(root,"/a/b/c/position(4)"
+
+--ldx]]--
+
+local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert
+
+xml.filters = { }
+
+function xml.filters.default(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.attributes(root,pattern,arguments)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
+    local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
+    if ekat then
+        if arguments then
+            return ekat[arguments] or "", rt, dt, dk
+        else
+            return ekat, rt, dt, dk
+        end
+    else
+        return { }, rt, dt, dk
+    end
+end
+
+function xml.filters.reverse(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.count(root,pattern,everything)
+    local n = 0
+    traverse(root, lpath(pattern), function(r,d,t)
+        if everything or type(d[t]) == "table" then
+            n = n + 1
+        end
+    end)
+    return n
+end
+
+function xml.filters.elements(root, pattern) -- == all
+    local t = { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local e = d[k]
+        if e then
+            t[#t+1] = e
+        end
+    end)
+    return t
+end
+
+function xml.filters.texts(root, pattern)
+    local t = { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local e = d[k]
+        if e and e.dt then
+            t[#t+1] = e.dt
+        end
+    end)
+    return t
+end
+
+function xml.filters.first(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end)
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.last(root,pattern)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse')
+    return dt and dt[dk], rt, dt, dk
+end
+
+function xml.filters.index(root,pattern,arguments)
+    local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1
+    if i and i ~= 0 then
+        if i < 0 then
+            reverse, i = true, -i
+        end
+        traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse)
+        if i == 0 then
+            return dt and dt[dk], rt, dt, dk
+        end
+    end
+    return nil, nil, nil, nil
+end
+
+function xml.filters.attribute(root,pattern,arguments)
+    local rt, dt, dk
+    traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end)
+    local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at)
+    return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or ""
+end
+
+function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow
+    local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments)
+    if dtk then -- n
+        local dtkdt = dtk.dt
+        if not dtkdt then
+            return "", rt, dt, dk
+        elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then
+            return dtkdt[1], rt, dt, dk
+        else
+            return xml.tostring(dtkdt), rt, dt, dk
+        end
+    else
+        return "", rt, dt, dk
+    end
+end
+
+function xml.filters.tag(root,pattern,n)
+    local tag = ""
+    traverse(root, lpath(pattern), function(r,d,k)
+        tag = xml.functions.tag(d,k,n and tonumber(n))
+        return true
+    end)
+    return tag
+end
+
+function xml.filters.name(root,pattern,n)
+    local tag = ""
+    traverse(root, lpath(pattern), function(r,d,k)
+        tag = xml.functions.name(d,k,n and tonumber(n))
+        return true
+    end)
+    return tag
+end
+
+--[[ldx--
+
For splitting the filter function from the path specification, we can
+use string matching or lpeg matching. Here the difference in speed is
+neglectable but the lpeg variant is more robust.

+--ldx]]--
+
+--  not faster but hipper ... although ... i can't get rid of the trailing / in the path
+
+local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc
+
+local slash     = P('/')
+local name      = (R("az","AZ","--","__"))^1
+local path      = C(((1-slash)^0 * slash)^1)
+local argument  = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" }
+local action    = Cc(1) * path * C(name) * argument
+local attribute = Cc(2) * path * P('@') * C(name)
+local direct    = Cc(3) * Cc("../*") * slash^0 * C(name) * argument
+
+local parser    = direct + action + attribute
+
+local filters          = xml.filters
+local attribute_filter = xml.filters.attributes
+local default_filter   = xml.filters.default
+
+-- todo: also hash, could be gc'd
+
+function xml.filter(root,pattern)
+    local kind, a, b, c = parser:match(pattern)
+    if kind == 1 or kind == 3 then
+        return (filters[b] or default_filter)(root,a,c)
+    elseif kind == 2 then
+        return attribute_filter(root,a,b)
+    else
+        return default_filter(root,pattern)
+    end
+end
+
+--~     slightly faster, but first we need a proper test file
+--~
+--~     local hash = { }
+--~
+--~     function xml.filter(root,pattern)
+--~         local h = hash[pattern]
+--~         if not h then
+--~             local kind, a, b, c = parser:match(pattern)
+--~             if kind == 1 then
+--~                 h = { kind, filters[b] or default_filter, a, b, c }
+--~             elseif kind == 2 then
+--~                 h = { kind, attribute_filter, a, b, c }
+--~             else
+--~                 h = { kind, default_filter, a, b, c }
+--~             end
+--~             hash[pattern] = h
+--~         end
+--~         local kind = h[1]
+--~         if kind == 1 then
+--~             return h[2](root,h[2],h[4])
+--~         elseif kind == 2 then
+--~             return h[2](root,h[2],h[3])
+--~         else
+--~             return h[2](root,pattern)
+--~         end
+--~     end
+
+--[[ldx--
+
The following functions collect elements and texts.

+--ldx]]--
+
+-- still somewhat bugged
+
+function xml.collect_elements(root, pattern, ignorespaces)
+    local rr, dd = { }, { }
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d and d[k]
+        if dk then
+            if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then
+                -- ignore
+            else
+                local n = #rr+1
+                rr[n], dd[n] = r, dk
+            end
+        end
+    end)
+    return dd, rr
+end
+
+function xml.collect_texts(root, pattern, flatten)
+    local t = { } -- no r collector
+    traverse(root, lpath(pattern), function(r,d,k)
+        if d then
+            local ek = d[k]
+            local tx = ek and ek.dt
+            if flatten then
+                if tx then
+                    t[#t+1] = xml.tostring(tx) or ""
+                else
+                    t[#t+1] = ""
+                end
+            else
+                t[#t+1] = tx or ""
+            end
+        else
+            t[#t+1] = ""
+        end
+    end)
+    return t
+end
+
+function xml.collect_tags(root, pattern, nonamespace)
+    local t = { }
+    xml.traverse(root, xml.lpath(pattern), function(r,d,k)
+        local dk = d and d[k]
+        if dk and type(dk) == "table" then
+            local ns, tg = e.ns, e.tg
+            if nonamespace then
+                t[#t+1] = tg -- if needed we can return an extra table
+            elseif ns == "" then
+                t[#t+1] = tg
+            else
+                t[#t+1] = ns .. ":" .. tg
+            end
+        end
+    end)
+    return #t > 0 and {}
+end
+
+--[[ldx--
+
Often using an iterators looks nicer in the code than passing handler
+functions. The  book describes how to use coroutines for that
+purpose (). This permits
+code like:

+
+
+for r, d, k in xml.elements(xml.load('text.xml'),"title") do
+    print(d[k])
+end
+
+
+
Which will print all the titles in the document. The iterator variant takes
+1.5 times the runtime of the function variant which is due to the overhead in
+creating the wrapper. So, instead of:

+
+
+function xml.filters.first(root,pattern)
+    for rt,dt,dk in xml.elements(root,pattern)
+        return dt and dt[dk], rt, dt, dk
+    end
+    return nil, nil, nil, nil
+end
+
+
+
We use the function variants in the filters.

+--ldx]]--
+
+local wrap, yield = coroutine.wrap, coroutine.yield
+
+function xml.elements(root,pattern,reverse)
+    return wrap(function() traverse(root, lpath(pattern), yield, reverse) end)
+end
+
+function xml.elements_only(root,pattern,reverse)
+    return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end)
+end
+
+function xml.each_element(root, pattern, handle, reverse)
+    local ok
+    traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse)
+    return ok
+end
+
+function xml.process_elements(root, pattern, handle)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dkdt = d[k].dt
+        if dkdt then
+            for i=1,#dkdt do
+                local v = dkdt[i]
+                if v.tg then handle(v) end
+            end
+        end
+    end)
+end
+
+function xml.process_attributes(root, pattern, handle)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local ek = d[k]
+        local a = ek.at or { }
+        handle(a)
+        if next(a) then -- next is faster than type (and >0 test)
+            ek.at = a
+        else
+            ek.at = nil
+        end
+    end)
+end
+
+--[[ldx--
+
We've now arrives at the functions that manipulate the tree.

+--ldx]]--
+
+function xml.inject_element(root, pattern, element, prepend)
+    if root and element then
+        local matches, collect = { }, nil
+        if type(element) == "string" then
+            element = convert(element,true)
+        end
+        if element then
+            collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
+            traverse(root, lpath(pattern), collect)
+            for i=1,#matches do
+                local m = matches[i]
+                local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil
+                if element.ri then
+                    element = element.dt[element.ri].dt
+                else
+                    element = element.dt
+                end
+                if r.ri then
+                    edt = r.dt[r.ri].dt
+                else
+                    edt = d and d[k] and d[k].dt
+                end
+                if edt then
+                    local be, af
+                    if prepend then
+                        be, af = xml.copy(element), edt
+                    else
+                        be, af = edt, xml.copy(element)
+                    end
+                    for i=1,#af do
+                        be[#be+1] = af[i]
+                    end
+                    if r.ri then
+                        r.dt[r.ri].dt = be
+                    else
+                        d[k].dt = be
+                    end
+                else
+                 -- r.dt = element.dt -- todo
+                end
+            end
+        end
+    end
+end
+
+-- todo: copy !
+
+function xml.insert_element(root, pattern, element, before) -- todo: element als functie
+    if root and element then
+        if pattern == "/" then
+            xml.inject_element(root, pattern, element, before)
+        else
+            local matches, collect = { }, nil
+            if type(element) == "string" then
+                element = convert(element,true)
+            end
+            if element and element.ri then
+                element = element.dt[element.ri]
+            end
+            if element then
+                collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end
+                traverse(root, lpath(pattern), collect)
+                for i=#matches,1,-1 do
+                    local m = matches[i]
+                    local r, d, k, element = m[1], m[2], m[3], m[4]
+                    if not before then k = k + 1 end
+                    if element.tg then
+                        insert(d,k,element) -- untested
+--~                         elseif element.dt then
+--~                             for _,v in ipairs(element.dt) do -- i added
+--~                                 insert(d,k,v)
+--~                                 k = k + 1
+--~                             end
+--~                         end
+                    else
+                        local edt = element.dt
+                        if edt then
+                            for i=1,#edt do
+                                insert(d,k,edt[i])
+                                k = k + 1
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+xml.insert_element_after  =                 xml.insert_element
+xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end
+xml.inject_element_after  =                 xml.inject_element
+xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end
+
+function xml.delete_element(root, pattern)
+    local matches, deleted = { }, { }
+    local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end
+    traverse(root, lpath(pattern), collect)
+    for i=#matches,1,-1 do
+        local m = matches[i]
+        deleted[#deleted+1] = remove(m[2],m[3])
+    end
+    return deleted
+end
+
+function xml.replace_element(root, pattern, element)
+    if type(element) == "string" then
+        element = convert(element,true)
+    end
+    if element and element.ri then
+        element = element.dt[element.ri]
+    end
+    if element then
+        traverse(root, lpath(pattern), function(rm, d, k)
+            d[k] = element.dt -- maybe not clever enough
+        end)
+    end
+end
+
+local function load_data(name) -- == io.loaddata
+    local f, data = io.open(name), ""
+    if f then
+        data = f:read("*all",'b') -- 'b' ?
+        f:close()
+    end
+    return data
+end
+
+function xml.include(xmldata,pattern,attribute,recursive,loaddata)
+    -- parse="text" (default: xml), encoding="" (todo)
+    -- attribute = attribute or 'href'
+    pattern = pattern or 'include'
+    loaddata = loaddata or load_data
+    local function include(r,d,k)
+        local ek, name = d[k], nil
+        if not attribute or attribute == "" then
+            local ekdt = ek.dt
+            name = (type(ekdt) == "table" and ekdt[1]) or ekdt
+        end
+        if not name then
+            if ek.at then
+                for a in gmatch(attribute or "href","([^|]+)") do
+                    name = ek.at[a]
+                    if name then break end
+                end
+            end
+        end
+        local data = (name and name ~= "" and loaddata(name)) or ""
+        if data == "" then
+            xml.empty(d,k)
+        elseif ek.at["parse"] == "text" then -- for the moment hard coded
+            d[k] = xml.escaped(data)
+        else
+            local xi = xml.convert(data)
+            if not xi then
+                xml.empty(d,k)
+            else
+                if recursive then
+                    xml.include(xi,pattern,attribute,recursive,loaddata)
+                end
+                xml.assign(d,k,xi)
+            end
+        end
+    end
+    xml.each_element(xmldata, pattern, include)
+end
+
+function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space !
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dkdt = d[k].dt
+        if dkdt then -- can be optimized
+            local t = { }
+            for i=1,#dkdt do
+                local str = dkdt[i]
+                if type(str) == "string" then
+
+                    if str == "" then
+                        -- stripped
+                    else
+                        if nolines then
+                            str = gsub(str,"[ \n\r\t]+"," ")
+                        end
+                        if str == "" then
+                            -- stripped
+                        else
+                            t[#t+1] = str
+                        end
+                    end
+                else
+                    t[#t+1] = str
+                end
+            end
+            d[k].dt = t
+        end
+    end)
+end
+
+local function rename_space(root, oldspace, newspace) -- fast variant
+    local ndt = #root.dt
+    for i=1,ndt or 0 do
+        local e = root[i]
+        if type(e) == "table" then
+            if e.ns == oldspace then
+                e.ns = newspace
+                if e.rn then
+                    e.rn = newspace
+                end
+            end
+            local edt = e.dt
+            if edt then
+                rename_space(edt, oldspace, newspace)
+            end
+        end
+    end
+end
+
+xml.rename_space = rename_space
+
+function xml.remap_tag(root, pattern, newtg)
+    traverse(root, lpath(pattern), function(r,d,k)
+        d[k].tg = newtg
+    end)
+end
+function xml.remap_namespace(root, pattern, newns)
+    traverse(root, lpath(pattern), function(r,d,k)
+        d[k].ns = newns
+    end)
+end
+function xml.check_namespace(root, pattern, newns)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d[k]
+        if (not dk.rn or dk.rn == "") and dk.ns == "" then
+            dk.rn = newns
+        end
+    end)
+end
+function xml.remap_name(root, pattern, newtg, newns, newrn)
+    traverse(root, lpath(pattern), function(r,d,k)
+        local dk = d[k]
+        dk.tg = newtg
+        dk.ns = newns
+        dk.rn = newrn
+    end)
+end
+
+function xml.filters.found(root,pattern,check_content)
+    local found = false
+    traverse(root, lpath(pattern), function(r,d,k)
+        if check_content then
+            local dk = d and d[k]
+            found = dk and dk.dt and next(dk.dt) and true
+        else
+            found = true
+        end
+        return true
+    end)
+    return found
+end
+
+--[[ldx--
+
Here are a few synonyms.

+--ldx]]--
+
+xml.filters.position = xml.filters.index
+
+xml.count    = xml.filters.count
+xml.index    = xml.filters.index
+xml.position = xml.filters.index
+xml.first    = xml.filters.first
+xml.last     = xml.filters.last
+xml.found    = xml.filters.found
+
+xml.each     = xml.each_element
+xml.process  = xml.process_element
+xml.strip    = xml.strip_whitespace
+xml.collect  = xml.collect_elements
+xml.all      = xml.collect_elements
+
+xml.insert   = xml.insert_element_after
+xml.inject   = xml.inject_element_after
+xml.after    = xml.insert_element_after
+xml.before   = xml.insert_element_before
+xml.delete   = xml.delete_element
+xml.replace  = xml.replace_element
+
+--[[ldx--
+
The following helper functions best belong to the lmxl-ini
+module. Some are here because we need then in the mk
+document and other manuals, others came up when playing with
+this module. Since this module is also used in  we've
+put them here instead of loading mode modules there then needed.

+--ldx]]--
+
+function xml.gsub(t,old,new)
+    local dt = t.dt
+    if dt then
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "string" then
+                dt[k] = gsub(v,old,new)
+            else
+                xml.gsub(v,old,new)
+            end
+        end
+    end
+end
+
+function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual
+    if d and k and d[k-1] and type(d[k-1]) == "string" then
+        local s = d[k-1]:match("\n(%s+)")
+        xml.gsub(dk,"\n"..string.rep(" ",#s),"\n")
+    end
+end
+
+function xml.serialize_path(root,lpath,handle)
+    local dk, r, d, k = xml.first(root,lpath)
+    dk = xml.copy(dk)
+    xml.strip_leading_spaces(dk,d,k)
+    xml.serialize(dk,handle)
+end
+
+--~ xml.escapes   = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' }
+--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end
+
+--~ function xml.escaped  (str) return (gsub(str,"(.)"   , xml.escapes  )) end
+--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end
+--~ function xml.cleansed (str) return (gsub(str,"<.->"  , ''           )) end -- "%b<>"
+
+local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
+
+-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
+--
+-- 1021:0335:0287:0247
+
+-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
+--
+-- 1559:0257:0288:0190 (last one suggested by roberto)
+
+--    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
+--    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+local normal  = (1 - S("<&>"))^0
+local special = P("<")/"<" + P(">")/">" + P("&")/"&"
+local escaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
+
+--    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+--    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
+local normal    = (1 - S"&")^0
+local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
+local unescaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
+
+local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
+
+function xml.escaped  (str) return escaped  :match(str) end
+function xml.unescaped(str) return unescaped:match(str) end
+function xml.cleansed (str) return cleansed :match(str) end
+
+function xml.join(t,separator,lastseparator)
+    if #t > 0 then
+        local result = { }
+        for k,v in pairs(t) do
+            result[k] = xml.tostring(v)
+        end
+        if lastseparator then
+            return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result]
+        else
+            return concat(result,separator)
+        end
+    else
+        return ""
+    end
+end
+
+function xml.statistics()
+    return {
+        lpathcalls = lpathcalls,
+        lpathcached = lpathcached,
+    }
+end
+
+--  xml.set_text_cleanup(xml.show_text_entities)
+--  xml.set_text_cleanup(xml.resolve_text_entities)
+
+--~ xml.lshow("/../../../a/(b|c)[@d='e']/f")
+--~ xml.lshow("/../../../a/!(b|c)[@d='e']/f")
+--~ xml.lshow("/../../../a/!b[@d!='e']/f")
+
+--~ x = xml.convert([[
+--~     
+--~         01
+--~         02
+--~         03
+--~         OK
+--~         05
+--~         06
+--~         ALSO OK
+--~     
+--~ ]])
+
+--~ xml.settrace("lpath",true)
+
+--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']"))
+--~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]"))
+--~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']"))
+--~ xml.xshow(xml.all  (x,"b[number(@n)>2 and number(@n)<6]"))
+--~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]"))
+
+--~ str = [[
+--~ 
+--~ 
+--~     my secret
+--~ 
+--~ ]]
+
+--~ x = xml.convert([[
+--~     0102xx03OK
+--~ ]])
+--~ xml.xshow(xml.first(x,"b[tag(2) == 'x']"))
+--~ xml.xshow(xml.first(x,"b[tag(1) == 'x']"))
+--~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']"))
+--~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']"))
+
+--~ print(xml.filter(x,"b/tag(2)"))
+--~ print(xml.filter(x,"b/tag(1)"))
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-ent'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, gsub, find = string.format, string.gsub, string.find
+local utfchar = unicode.utf8.char
+
+--[[ldx--
+
We provide (at least here) two entity handlers. The more extensive
+resolver consults a hash first, tries to convert to  next,
+and finaly calls a handler when defines. When this all fails, the
+original entity is returned.

+--ldx]]--
+
+xml.entities = xml.entities or { } -- xml.entity_handler == function
+
+function xml.entity_handler(e)
+    return format("[%s]",e)
+end
+
+local function toutf(s)
+    return utfchar(tonumber(s,16))
+end
+
+local function utfize(root)
+    local d = root.dt
+    for k=1,#d do
+        local dk = d[k]
+        if type(dk) == "string" then
+        --  test prevents copying if no match
+            if find(dk,"&#x.-;") then
+                d[k] = gsub(dk,"&#x(.-);",toutf)
+            end
+        else
+            utfize(dk)
+        end
+    end
+end
+
+xml.utfize = utfize
+
+local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks
+    if find(e,"^#x") then
+        return utfchar(tonumber(e:sub(3),16))
+    elseif find(e,"^#") then
+        return utfchar(tonumber(e:sub(2)))
+    else
+        local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded)
+        if ee then
+            return ee
+        else
+            local h = xml.entity_handler
+            return (h and h(e)) or "&" .. e .. ";"
+        end
+    end
+end
+
+local function resolve_entities(root)
+    if not root.special or root.tg == "@rt@" then
+        local d = root.dt
+        for k=1,#d do
+            local dk = d[k]
+            if type(dk) == "string" then
+                if find(dk,"&.-;") then
+                    d[k] = gsub(dk,"&(.-);",resolve)
+                end
+            else
+                resolve_entities(dk)
+            end
+        end
+    end
+end
+
+xml.resolve_entities = resolve_entities
+
+function xml.utfize_text(str)
+    if find(str,"&#") then
+        return (gsub(str,"&#x(.-);",toutf))
+    else
+        return str
+    end
+end
+
+function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline
+    if find(str,"&") then
+        return (gsub(str,"&(.-);",resolve))
+    else
+        return str
+    end
+end
+
+function xml.show_text_entities(str)
+    if find(str,"&") then
+        return (gsub(str,"&(.-);","[%1]"))
+    else
+        return str
+    end
+end
+
+-- experimental, this will be done differently
+
+function xml.merge_entities(root)
+    local documententities = root.entities
+    local allentities = xml.entities
+    if documententities then
+        for k, v in next, documententities do
+            allentities[k] = v
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['lxml-mis'] = {
+    version   = 1.001,
+    comment   = "this module is the basis for the lxml-* ones",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local concat = table.concat
+local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring
+local format, gsub = string.format, string.gsub
+
+--[[ldx--
+
The following helper functions best belong to the lmxl-ini
+module. Some are here because we need then in the mk
+document and other manuals, others came up when playing with
+this module. Since this module is also used in  we've
+put them here instead of loading mode modules there then needed.

+--ldx]]--
+
+function xml.gsub(t,old,new)
+    local dt = t.dt
+    if dt then
+        for k=1,#dt do
+            local v = dt[k]
+            if type(v) == "string" then
+                dt[k] = gsub(v,old,new)
+            else
+                xml.gsub(v,old,new)
+            end
+        end
+    end
+end
+
+function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual
+    if d and k and d[k-1] and type(d[k-1]) == "string" then
+        local s = d[k-1]:match("\n(%s+)")
+        xml.gsub(dk,"\n"..string.rep(" ",#s),"\n")
+    end
+end
+
+function xml.serialize_path(root,lpath,handle)
+    local dk, r, d, k = xml.first(root,lpath)
+    dk = xml.copy(dk)
+    xml.strip_leading_spaces(dk,d,k)
+    xml.serialize(dk,handle)
+end
+
+--~ xml.escapes   = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' }
+--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end
+
+--~ function xml.escaped  (str) return (gsub(str,"(.)"   , xml.escapes  )) end
+--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end
+--~ function xml.cleansed (str) return (gsub(str,"<.->"  , ''           )) end -- "%b<>"
+
+local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs
+
+-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg
+--
+-- 1021:0335:0287:0247
+
+-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ"
+--
+-- 1559:0257:0288:0190 (last one suggested by roberto)
+
+--    escaped = Cs((S("<&>") / xml.escapes + 1)^0)
+--    escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+local normal  = (1 - S("<&>"))^0
+local special = P("<")/"<" + P(">")/">" + P("&")/"&"
+local escaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto)
+
+--    unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0)
+--    unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0)
+local normal    = (1 - S"&")^0
+local special   = P("<")/"<" + P(">")/">" + P("&")/"&"
+local unescaped = Cs(normal * (special * normal)^0)
+
+-- 100 * 5000 * "oeps  oeps  oeps " : gsub:lpeg == 623:501 msec (short tags, less difference)
+
+local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0)
+
+xml.escaped_pattern   = escaped
+xml.unescaped_pattern = unescaped
+xml.cleansed_pattern  = cleansed
+
+function xml.escaped  (str) return escaped  :match(str) end
+function xml.unescaped(str) return unescaped:match(str) end
+function xml.cleansed (str) return cleansed :match(str) end
+
+function xml.join(t,separator,lastseparator)
+    if #t > 0 then
+        local result = { }
+        for k,v in pairs(t) do
+            result[k] = xml.tostring(v)
+        end
+        if lastseparator then
+            return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result]
+        else
+            return concat(result,separator)
+        end
+    else
+        return ""
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-tra'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- the  tag is kind of generic and used for functions that are not
+-- bound to a variable, like node.new, node.copy etc (contrary to for instance
+-- node.has_attribute which is bound to a has_attribute local variable in mkiv)
+
+debugger = debugger or { }
+
+local counters = { }
+local names = { }
+local getinfo = debug.getinfo
+local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch
+
+-- one
+
+local function hook()
+    local f = getinfo(2,"f").func
+    local n = getinfo(2,"Sn")
+--  if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end
+    if f then
+        local cf = counters[f]
+        if cf == nil then
+            counters[f] = 1
+            names[f] = n
+        else
+            counters[f] = cf + 1
+        end
+    end
+end
+local function getname(func)
+    local n = names[func]
+    if n then
+        if n.what == "C" then
+            return n.name or ''
+        else
+            -- source short_src linedefined what name namewhat nups func
+            local name = n.name or n.namewhat or n.what
+            if not name or name == "" then name = "?" end
+            return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name)
+        end
+    else
+        return "unknown"
+    end
+end
+function debugger.showstats(printer,threshold)
+    printer   = printer or texio.write or print
+    threshold = threshold or 0
+    local total, grandtotal, functions = 0, 0, 0
+    printer("\n") -- ugly but ok
+ -- table.sort(counters)
+    for func, count in pairs(counters) do
+        if count > threshold then
+            local name = getname(func)
+            if not name:find("for generator") then
+                printer(format("%8i  %s", count, name))
+                total = total + count
+            end
+        end
+        grandtotal = grandtotal + count
+        functions = functions + 1
+    end
+    printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+end
+
+-- two
+
+--~ local function hook()
+--~     local n = getinfo(2)
+--~     if n.what=="C" and not n.name then
+--~         local f = tostring(debug.traceback())
+--~         local cf = counters[f]
+--~         if cf == nil then
+--~             counters[f] = 1
+--~             names[f] = n
+--~         else
+--~             counters[f] = cf + 1
+--~         end
+--~     end
+--~ end
+--~ function debugger.showstats(printer,threshold)
+--~     printer   = printer or texio.write or print
+--~     threshold = threshold or 0
+--~     local total, grandtotal, functions = 0, 0, 0
+--~     printer("\n") -- ugly but ok
+--~  -- table.sort(counters)
+--~     for func, count in pairs(counters) do
+--~         if count > threshold then
+--~             printer(format("%8i  %s", count, func))
+--~             total = total + count
+--~         end
+--~         grandtotal = grandtotal + count
+--~         functions = functions + 1
+--~     end
+--~     printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold))
+--~ end
+
+-- rest
+
+function debugger.savestats(filename,threshold)
+    local f = io.open(filename,'w')
+    if f then
+        debugger.showstats(function(str) f:write(str) end,threshold)
+        f:close()
+    end
+end
+
+function debugger.enable()
+    debug.sethook(hook,"c")
+end
+
+function debugger.disable()
+    debug.sethook()
+--~ counters[debug.getinfo(2,"f").func] = nil
+end
+
+function debugger.tracing()
+    local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0
+    if n > 0 then
+        function debugger.tracing() return true  end ; return true
+    else
+        function debugger.tracing() return false end ; return false
+    end
+end
+
+--~ debugger.enable()
+
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+--~ print(math.sin(1*.5))
+
+--~ debugger.disable()
+
+--~ print("")
+--~ debugger.showstats()
+--~ print("")
+--~ debugger.showstats(print,3)
+
+trackers = trackers or { }
+
+local data, done = { }, { }
+
+local function set(what,value)
+    for w in gmatch(lower(what),"[^, ]+") do
+        for d, f in next, data do
+            if done[d] then
+                -- prevent recursion due to wildcards
+            elseif find(d,w) then
+                done[d] = true
+                for i=1,#f do
+                    f[i](value)
+                end
+            end
+        end
+    end
+end
+
+local function reset()
+    for d, f in next, data do
+        for i=1,#f do
+            f[i](false)
+        end
+    end
+end
+
+function trackers.register(what,...)
+    what = lower(what)
+    local w = data[what]
+    if not w then
+        w = { }
+        data[what] = w
+    end
+    for _, fnc in next, { ... } do
+        local typ = type(fnc)
+        if typ == "function" then
+            w[#w+1] = fnc
+        elseif typ == "string" then
+            w[#w+1] = function(value) set(fnc,value,nesting) end
+        end
+    end
+end
+
+function trackers.enable(what)
+    done = { }
+    set(what,true)
+end
+
+function trackers.disable(what)
+    done = { }
+    if not what or what == "" then
+        trackers.reset(what)
+    else
+        set(what,false)
+    end
+end
+
+function trackers.reset(what)
+    done = { }
+    reset()
+end
+
+function trackers.list() -- pattern
+    local list = table.sortedkeys(data)
+    local user, system = { }, { }
+    for l=1,#list do
+        local what = list[l]
+        if find(what,"^%*") then
+            system[#system+1] = what
+        else
+            user[#user+1] = what
+        end
+    end
+    return user, system
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-env'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- A former version provided functionality for non embeded core
+-- scripts i.e. runtime library loading. Given the amount of
+-- Lua code we use now, this no longer makes sense. Much of this
+-- evolved before bytecode arrays were available and so a lot of
+-- code has disappeared already.
+
+local trace_verbose  = false  trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+
+local format = string.format
+
+-- precautions
+
+os.setlocale(nil,nil) -- useless feature and even dangerous in luatex
+
+function os.setlocale()
+    -- no way you can mess with it
+end
+
+-- dirty tricks
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then
+    profiler.start("luatex-profile.log")
+end
+
+-- environment
+
+environment             = environment or { }
+environment.arguments   = { }
+environment.files       = { }
+environment.sortedflags = nil
+
+if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end
+if not environment.version or environment.version == "" then             environment.version = "unknown"   end
+if not environment.jobname                              then             environment.jobname = "unknown"   end
+
+function environment.initialize_arguments(arg)
+    local arguments, files = { }, { }
+    environment.arguments, environment.files, environment.sortedflags = arguments, files, nil
+    for index, argument in pairs(arg) do
+        if index > 0 then
+            local flag, value = argument:match("^%-+(.+)=(.-)$")
+            if flag then
+                arguments[flag] = string.unquote(value or "")
+            else
+                flag = argument:match("^%-+(.+)")
+                if flag then
+                    arguments[flag] = true
+                else
+                    files[#files+1] = argument
+                end
+            end
+        end
+    end
+    environment.ownname = environment.ownname or arg[0] or 'unknown.lua'
+end
+
+function environment.setargument(name,value)
+    environment.arguments[name] = value
+end
+
+-- todo: defaults, better checks e.g on type (boolean versus string)
+--
+-- tricky: too many hits when we support partials unless we add
+-- a registration of arguments so from now on we have 'partial'
+
+function environment.argument(name,partial)
+    local arguments, sortedflags = environment.arguments, environment.sortedflags
+    if arguments[name] then
+        return arguments[name]
+    elseif partial then
+        if not sortedflags then
+            sortedflags = { }
+            for _,v in pairs(table.sortedkeys(arguments)) do
+                sortedflags[#sortedflags+1] = "^" .. v
+            end
+            environment.sortedflags = sortedflags
+        end
+        -- example of potential clash: ^mode ^modefile
+        for _,v in ipairs(sortedflags) do
+            if name:find(v) then
+                return arguments[v:sub(2,#v)]
+            end
+        end
+    end
+    return nil
+end
+
+function environment.split_arguments(separator) -- rather special, cut-off before separator
+    local done, before, after = false, { }, { }
+    for _,v in ipairs(environment.original_arguments) do
+        if not done and v == separator then
+            done = true
+        elseif done then
+            after[#after+1] = v
+        else
+            before[#before+1] = v
+        end
+    end
+    return before, after
+end
+
+function environment.reconstruct_commandline(arg,noquote)
+    arg = arg or environment.original_arguments
+    if noquote and #arg == 1 then
+        local a = arg[1]
+        a = resolvers.resolve(a)
+        a = a:unquote()
+        return a
+    elseif next(arg) then
+        local result = { }
+        for _,a in ipairs(arg) do -- ipairs 1 .. #n
+            a = resolvers.resolve(a)
+            a = a:unquote()
+            a = a:gsub('"','\\"') -- tricky
+            if a:find(" ") then
+                result[#result+1] = a:quote()
+            else
+                result[#result+1] = a
+            end
+        end
+        return table.join(result," ")
+    else
+        return ""
+    end
+end
+
+if arg then
+
+    -- new, reconstruct quoted snippets (maybe better just remnove the " then and add them later)
+    local newarg, instring = { }, false
+
+    for index, argument in ipairs(arg) do
+        if argument:find("^\"") then
+            newarg[#newarg+1] = argument:gsub("^\"","")
+            if not argument:find("\"$") then
+                instring = true
+            end
+        elseif argument:find("\"$") then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument:gsub("\"$","")
+            instring = false
+        elseif instring then
+            newarg[#newarg] = newarg[#newarg] .. " " .. argument
+        else
+            newarg[#newarg+1] = argument
+        end
+    end
+    for i=1,-5,-1 do
+        newarg[i] = arg[i]
+    end
+
+    environment.initialize_arguments(newarg)
+    environment.original_arguments = newarg
+    environment.raw_arguments = arg
+
+    arg = { } -- prevent duplicate handling
+
+end
+
+-- weird place ... depends on a not yet loaded module
+
+function environment.texfile(filename)
+    return resolvers.find_file(filename,'tex')
+end
+
+function environment.luafile(filename)
+    local resolved = resolvers.find_file(filename,'tex') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    resolved = resolvers.find_file(filename,'texmfscripts') or ""
+    if resolved ~= "" then
+        return resolved
+    end
+    return resolvers.find_file(filename,'luatexlibs') or ""
+end
+
+environment.loadedluacode = loadfile -- can be overloaded
+
+--~ function environment.loadedluacode(name)
+--~     if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then
+--~         local chunk = loadstring(io.loaddata("texluac.luc"))
+--~         os.remove("texluac.luc")
+--~         return chunk
+--~     else
+--~         environment.loadedluacode = loadfile -- can be overloaded
+--~         return loadfile(name)
+--~     end
+--~ end
+
+function environment.luafilechunk(filename) -- used for loading lua bytecode in the format
+    filename = file.replacesuffix(filename, "lua")
+    local fullname = environment.luafile(filename)
+    if fullname and fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading file %s", fullname)
+        end
+        return environment.loadedluacode(fullname)
+    else
+        if trace_verbose then
+            logs.report("fileio","unknown file %s", filename)
+        end
+        return nil
+    end
+end
+
+-- the next ones can use the previous ones / combine
+
+function environment.loadluafile(filename, version)
+    local lucname, luaname, chunk
+    local basename = file.removesuffix(filename)
+    if basename == filename then
+        lucname, luaname = basename .. ".luc",  basename .. ".lua"
+    else
+        lucname, luaname = nil, basename -- forced suffix
+    end
+    -- when not overloaded by explicit suffix we look for a luc file first
+    local fullname = (lucname and environment.luafile(lucname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+    end
+    if chunk then
+        assert(chunk)()
+        if version then
+            -- we check of the version number of this chunk matches
+            local v = version -- can be nil
+            if modules and modules[filename] then
+                v = modules[filename].version -- new method
+            elseif versions and versions[filename] then
+                v = versions[filename]        -- old method
+            end
+            if v == version then
+                return true
+            else
+                if trace_verbose then
+                    logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version)
+                end
+                environment.loadluafile(filename)
+            end
+        else
+            return true
+        end
+    end
+    fullname = (luaname and environment.luafile(luaname)) or ""
+    if fullname ~= "" then
+        if trace_verbose then
+            logs.report("fileio","loading %s", fullname)
+        end
+        chunk = loadfile(fullname) -- this way we don't need a file exists check
+        if not chunk then
+            if verbose then
+                logs.report("fileio","unknown file %s", filename)
+            end
+        else
+            assert(chunk)()
+            return true
+        end
+    end
+    return false
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['trac-inf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format = string.format
+
+local statusinfo, n, registered = { }, 0, { }
+
+statistics = statistics or { }
+
+statistics.enable    = true
+statistics.threshold = 0.05
+
+-- timing functions
+
+local clock = os.gettimeofday or os.clock
+
+function statistics.hastimer(instance)
+    return instance and instance.starttime
+end
+
+function statistics.starttiming(instance)
+    if instance then
+        local it = instance.timing
+        if not it then
+            it = 0
+        end
+        if it == 0 then
+            instance.starttime = clock()
+            if not instance.loadtime then
+                instance.loadtime = 0
+            end
+        end
+        instance.timing = it + 1
+    end
+end
+
+function statistics.stoptiming(instance, report)
+    if instance then
+        local it = instance.timing
+        if it > 1 then
+            instance.timing = it - 1
+        else
+            local starttime = instance.starttime
+            if starttime then
+                local stoptime = clock()
+                local loadtime = stoptime - starttime
+                instance.stoptime = stoptime
+                instance.loadtime = instance.loadtime + loadtime
+                if report then
+                    statistics.report("load time %0.3f",loadtime)
+                end
+                instance.timing = 0
+                return loadtime
+            end
+        end
+    end
+    return 0
+end
+
+function statistics.elapsedtime(instance)
+    return format("%0.3f",(instance and instance.loadtime) or 0)
+end
+
+function statistics.elapsedindeed(instance)
+    local t = (instance and instance.loadtime) or 0
+    return t > statistics.threshold
+end
+
+-- general function
+
+function statistics.register(tag,fnc)
+    if statistics.enable and type(fnc) == "function" then
+        local rt = registered[tag] or (#statusinfo + 1)
+        statusinfo[rt] = { tag, fnc }
+        registered[tag] = rt
+        if #tag > n then n = #tag end
+    end
+end
+
+function statistics.show(reporter)
+    if statistics.enable then
+        if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end
+        -- this code will move
+        local register = statistics.register
+        register("luatex banner", function()
+            return string.lower(status.banner)
+        end)
+        register("control sequences", function()
+            return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra)
+        end)
+        register("callbacks", function()
+            local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0
+            return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total)
+        end)
+        register("current memory usage", statistics.memused)
+        register("runtime",statistics.runtime)
+--         --
+        for i=1,#statusinfo do
+            local s = statusinfo[i]
+            local r = s[2]()
+            if r then
+                reporter(s[1],r,n)
+            end
+        end
+        statistics.enable = false
+    end
+end
+
+function statistics.show_job_stat(tag,data,n)
+    texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data))
+end
+
+function statistics.memused() -- no math.round yet -)
+    local round = math.round or math.floor
+    return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000))
+end
+
+if statistics.runtime then
+    -- already loaded and set
+elseif luatex and luatex.starttime then
+    statistics.starttime = luatex.starttime
+    statistics.loadtime = 0
+    statistics.timing = 0
+else
+    statistics.starttiming(statistics)
+end
+
+function statistics.runtime()
+    statistics.stoptiming(statistics)
+    return statistics.formatruntime(statistics.elapsedtime(statistics))
+end
+
+function statistics.formatruntime(runtime)
+    return format("%s seconds", statistics.elapsedtime(statistics))
+end
+
+function statistics.timed(action,report)
+    local timer = { }
+    report = report or logs.simple
+    statistics.starttiming(timer)
+    action()
+    statistics.stoptiming(timer)
+    report("total runtime: %s",statistics.elapsedtime(timer))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-log'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this is old code that needs an overhaul
+
+local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format
+
+if texlua then
+    write_nl = print
+    write    = io.write
+end
+
+--[[ldx--
+
This is a prelude to a more extensive logging module. For the sake
+of parsing log files, in addition to the standard logging we will
+provide an  structured file. Actually, any logging that
+is hooked into callbacks will be \XML\ by default.

+--ldx]]--
+
+logs     = logs     or { }
+logs.xml = logs.xml or { }
+logs.tex = logs.tex or { }
+
+--[[ldx--
+
This looks pretty ugly but we need to speed things up a bit.

+--ldx]]--
+
+logs.moreinfo = [[
+more information about ConTeXt and the tools that come with it can be found at:
+
+maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context
+webpage  : http://www.pragma-ade.nl / http://tex.aanhet.net
+wiki     : http://contextgarden.net
+]]
+
+logs.levels = {
+    ['error']   = 1,
+    ['warning'] = 2,
+    ['info']    = 3,
+    ['debug']   = 4,
+}
+
+logs.functions = {
+    'report', 'start', 'stop', 'push', 'pop', 'line', 'direct',
+    'start_run', 'stop_run',
+    'start_page_number', 'stop_page_number',
+    'report_output_pages', 'report_output_log',
+    'report_tex_stat', 'report_job_stat',
+    'show_open', 'show_close', 'show_load',
+}
+
+logs.tracers = {
+}
+
+logs.level = 0
+logs.mode  = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"))
+
+function logs.set_level(level)
+    logs.level = logs.levels[level] or level
+end
+
+function logs.set_method(method)
+    for _, v in next, logs.functions do
+        logs[v] = logs[method][v] or function() end
+    end
+end
+
+-- tex logging
+
+function logs.tex.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(category .. " | " .. format(fmt,...))
+    else
+        write_nl(category .. " |")
+    end
+end
+
+function logs.tex.line(fmt,...) -- new
+    if fmt then
+        write_nl(format(fmt,...))
+    else
+        write_nl("")
+    end
+end
+
+local texcount = tex and tex.count
+
+function logs.tex.start_page_number()
+    local real, user, sub = texcount[0], texcount[1], texcount[2]
+    if real > 0 then
+        if user > 0 then
+            if sub > 0 then
+                write(format("[%s.%s.%s",real,user,sub))
+            else
+                write(format("[%s.%s",real,user))
+            end
+        else
+            write(format("[%s",real))
+        end
+    else
+        write("[-")
+    end
+end
+
+function logs.tex.stop_page_number()
+    write("]")
+end
+
+logs.tex.report_job_stat = statistics.show_job_stat
+
+-- xml logging
+
+function logs.xml.report(category,fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",category,format(fmt,...)))
+    else
+        write_nl(format("",category))
+    end
+end
+function logs.xml.line(fmt,...) -- new
+    if fmt then
+        write_nl(format("%s",format(fmt,...)))
+    else
+        write_nl("")
+    end
+end
+
+function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end
+function logs.xml.stop () if logs.level > 0 then tw("") end end
+function logs.xml.push () if logs.level > 0 then tw("" ) end end
+
+function logs.xml.start_run()
+    write_nl("")
+    write_nl("") --  xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng'
+    write_nl("")
+end
+
+function logs.xml.stop_run()
+    write_nl("")
+end
+
+function logs.xml.start_page_number()
+    write_nl(format("
")
+    write_nl("")
+end
+
+function logs.xml.report_output_pages(p,b)
+    write_nl(format("", p))
+    write_nl(format("", b))
+    write_nl("")
+end
+
+function logs.xml.report_output_log()
+end
+
+function logs.xml.report_tex_stat(k,v)
+    texiowrite_nl("log",""..tostring(v).."")
+end
+
+local level = 0
+
+function logs.xml.show_open(name)
+    level = level + 1
+    texiowrite_nl(format("",level,name))
+end
+
+function logs.xml.show_close(name)
+    texiowrite(" ")
+    level = level - 1
+end
+
+function logs.xml.show_load(name)
+    texiowrite_nl(format("",level+1,name))
+end
+
+--
+
+local name, banner = 'report', 'context'
+
+local function report(category,fmt,...)
+    if fmt then
+        write_nl(format("%s | %s: %s",name,category,format(fmt,...)))
+    elseif category then
+        write_nl(format("%s | %s",name,category))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+local function simple(fmt,...)
+    if fmt then
+        write_nl(format("%s | %s",name,format(fmt,...)))
+    else
+        write_nl(format("%s |",name))
+    end
+end
+
+function logs.setprogram(_name_,_banner_,_verbose_)
+    name, banner = _name_, _banner_
+    if _verbose_ then
+        trackers.enable("resolvers.verbose")
+    end
+    logs.set_method("tex")
+    logs.report = report -- also used in libraries
+    logs.simple = simple -- only used in scripts !
+    if utils then
+        utils.report = simple
+    end
+    logs.verbose = _verbose_
+end
+
+function logs.setverbose(what)
+    if what then
+        trackers.enable("resolvers.verbose")
+    else
+        trackers.disable("resolvers.verbose")
+    end
+    logs.verbose = what or false
+end
+
+function logs.extendbanner(_banner_,_verbose_)
+    banner = banner .. " | ".. _banner_
+    if _verbose_ ~= nil then
+        logs.setverbose(what)
+    end
+end
+
+logs.verbose = false
+logs.report  = logs.tex.report
+logs.simple  = logs.tex.report
+
+function logs.reportlines(str) -- todo: 
+    for line in str:gmatch("(.-)[\n\r]") do
+        logs.report(line)
+    end
+end
+
+function logs.reportline() -- for scripts too
+    logs.report()
+end
+
+logs.simpleline = logs.reportline
+
+function logs.help(message,option)
+    logs.report(banner)
+    logs.reportline()
+    logs.reportlines(message)
+    local moreinfo = logs.moreinfo or ""
+    if moreinfo ~= "" and option ~= "nomoreinfo" then
+        logs.reportline()
+        logs.reportlines(moreinfo)
+    end
+end
+
+logs.set_level('error')
+logs.set_method('tex')
+
+function logs.system(whereto,process,jobname,category,...)
+    for i=1,10 do
+        local f = io.open(whereto,"a")
+        if f then
+            f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)))
+            f:close()
+            break
+        else
+            sleep(0.1)
+        end
+    end
+end
+
+--~ local syslogname = "oeps.xxx"
+--~
+--~ for i=1,10 do
+--~     logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123")
+--~ end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+    comment   = "companion to luat-lib.tex",
+}
+
+-- After a few years using the code the large luat-inp.lua file
+-- has been split up a bit. In the process some functionality was
+-- dropped:
+--
+-- * support for reading lsr files
+-- * selective scanning (subtrees)
+-- * some public auxiliary functions were made private
+--
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
+-- This lib is multi-purpose and can be loaded again later on so that
+-- additional functionality becomes available. We will split thislogs.report("fileio",
+-- module in components once we're done with prototyping. This is the
+-- first code I wrote for LuaTeX, so it needs some cleanup. Before changing
+-- something in this module one can best check with Taco or Hans first; there
+-- is some nasty trickery going on that relates to traditional kpse support.
+
+-- To be considered: hash key lowercase, first entry in table filename
+-- (any case), rest paths (so no need for optimization). Or maybe a
+-- separate table that matches lowercase names to mixed case when
+-- present. In that case the lower() cases can go away. I will do that
+-- only when we run into problems with names ... well ... Iwona-Regular.
+
+-- Beware, loading and saving is overloaded in luat-tmp!
+
+local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
+local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
+local next, type = next, type
+
+local trace_locating, trace_detail, trace_verbose  = false, false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end)
+trackers.register("resolvers.detail",   function(v) trace_detail   = v trackers.enable("resolvers.verbose,resolvers.detail") end)
+
+if not resolvers then
+    resolvers = {
+        suffixes     = { },
+        formats      = { },
+        dangerous    = { },
+        suffixmap    = { },
+        alternatives = { },
+        locators     = { },  -- locate databases
+        hashers      = { },  -- load databases
+        generators   = { },  -- generate databases
+    }
+end
+
+local resolvers = resolvers
+
+resolvers.locators  .notfound = { nil }
+resolvers.hashers   .notfound = { nil }
+resolvers.generators.notfound = { nil }
+
+resolvers.cacheversion = '1.0.1'
+resolvers.cnfname      = 'texmf.cnf'
+resolvers.luaname      = 'texmfcnf.lua'
+resolvers.homedir      = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~'
+resolvers.cnfdefault   = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}'
+
+local dummy_path_expr = "^!*unset/*$"
+
+local formats      = resolvers.formats
+local suffixes     = resolvers.suffixes
+local dangerous    = resolvers.dangerous
+local suffixmap    = resolvers.suffixmap
+local alternatives = resolvers.alternatives
+
+formats['afm'] = 'AFMFONTS'       suffixes['afm'] = { 'afm' }
+formats['enc'] = 'ENCFONTS'       suffixes['enc'] = { 'enc' }
+formats['fmt'] = 'TEXFORMATS'     suffixes['fmt'] = { 'fmt' }
+formats['map'] = 'TEXFONTMAPS'    suffixes['map'] = { 'map' }
+formats['mp']  = 'MPINPUTS'       suffixes['mp']  = { 'mp' }
+formats['ocp'] = 'OCPINPUTS'      suffixes['ocp'] = { 'ocp' }
+formats['ofm'] = 'OFMFONTS'       suffixes['ofm'] = { 'ofm', 'tfm' }
+formats['otf'] = 'OPENTYPEFONTS'  suffixes['otf'] = { 'otf' } -- 'ttf'
+formats['opl'] = 'OPLFONTS'       suffixes['opl'] = { 'opl' }
+formats['otp'] = 'OTPINPUTS'      suffixes['otp'] = { 'otp' }
+formats['ovf'] = 'OVFFONTS'       suffixes['ovf'] = { 'ovf', 'vf' }
+formats['ovp'] = 'OVPFONTS'       suffixes['ovp'] = { 'ovp' }
+formats['tex'] = 'TEXINPUTS'      suffixes['tex'] = { 'tex' }
+formats['tfm'] = 'TFMFONTS'       suffixes['tfm'] = { 'tfm' }
+formats['ttf'] = 'TTFONTS'        suffixes['ttf'] = { 'ttf', 'ttc' }
+formats['pfb'] = 'T1FONTS'        suffixes['pfb'] = { 'pfb', 'pfa' }
+formats['vf']  = 'VFFONTS'        suffixes['vf']  = { 'vf' }
+
+formats['fea'] = 'FONTFEATURES'   suffixes['fea'] = { 'fea' }
+formats['cid'] = 'FONTCIDMAPS'    suffixes['cid'] = { 'cid', 'cidmap' }
+
+formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new
+suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua'
+
+formats ['lua'] = 'LUAINPUTS' -- new
+suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' }
+
+-- backward compatible ones
+
+alternatives['map files']            = 'map'
+alternatives['enc files']            = 'enc'
+alternatives['cid files']            = 'cid'
+alternatives['fea files']            = 'fea'
+alternatives['opentype fonts']       = 'otf'
+alternatives['truetype fonts']       = 'ttf'
+alternatives['truetype collections'] = 'ttc'
+alternatives['type1 fonts']          = 'pfb'
+
+-- obscure ones
+
+formats ['misc fonts'] = ''
+suffixes['misc fonts'] = { }
+
+formats     ['sfd']                      = 'SFDFONTS'
+suffixes    ['sfd']                      = { 'sfd' }
+alternatives['subfont definition files'] = 'sfd'
+
+-- In practice we will work within one tds tree, but i want to keep
+-- the option open to build tools that look at multiple trees, which is
+-- why we keep the tree specific data in a table. We used to pass the
+-- instance but for practical pusposes we now avoid this and use a
+-- instance variable.
+
+-- here we catch a few new thingies (todo: add these paths to context.tmf)
+--
+-- FONTFEATURES  = .;$TEXMF/fonts/fea//
+-- FONTCIDMAPS   = .;$TEXMF/fonts/cid//
+
+-- we always have one instance active
+
+resolvers.instance = resolvers.instance or nil -- the current one (slow access)
+local instance = resolvers.instance or nil -- the current one (fast access)
+
+function resolvers.newinstance()
+
+    -- store once, freeze and faster (once reset we can best use
+    -- instance.environment) maybe better have a register suffix
+    -- function
+
+    for k, v in next, suffixes do
+        for i=1,#v do
+            local vi = v[i]
+            if vi then
+                suffixmap[vi] = k
+            end
+        end
+    end
+
+    -- because vf searching is somewhat dangerous, we want to prevent
+    -- too liberal searching esp because we do a lookup on the current
+    -- path anyway; only tex (or any) is safe
+
+    for k, v in next, formats do
+        dangerous[k] = true
+    end
+    dangerous.tex = nil
+
+    -- the instance
+
+    local newinstance = {
+        rootpath        = '',
+        treepath        = '',
+        progname        = 'context',
+        engine          = 'luatex',
+        format          = '',
+        environment     = { },
+        variables       = { },
+        expansions      = { },
+        files           = { },
+        remap           = { },
+        configuration   = { },
+        setup           = { },
+        order           = { },
+        found           = { },
+        foundintrees    = { },
+        kpsevars        = { },
+        hashes          = { },
+        cnffiles        = { },
+        luafiles        = { },
+        lists           = { },
+        remember        = true,
+        diskcache       = true,
+        renewcache      = false,
+        scandisk        = true,
+        cachepath       = nil,
+        loaderror       = false,
+        sortdata        = false,
+        savelists       = true,
+        cleanuppaths    = true,
+        allresults      = false,
+        pattern         = nil, -- lists
+        data            = { }, -- only for loading
+        force_suffixes  = true,
+        fakepaths       = { },
+    }
+
+    local ne = newinstance.environment
+
+    for k,v in next, os.env do
+        ne[k] = resolvers.bare_variable(v)
+    end
+
+    return newinstance
+
+end
+
+function resolvers.setinstance(someinstance)
+    instance = someinstance
+    resolvers.instance = someinstance
+    return someinstance
+end
+
+function resolvers.reset()
+    return resolvers.setinstance(resolvers.newinstance())
+end
+
+local function reset_hashes()
+    instance.lists = { }
+    instance.found = { }
+end
+
+local function check_configuration() -- not yet ok, no time for debugging now
+    local ie = instance.environment
+    local function fix(varname,default)
+        local proname = varname .. "." .. instance.progname or "crap"
+        local p, v = ie[proname], ie[varname]
+        if not ((p and p ~= "") or (v and v ~= "")) then
+            instance.variables[varname] = default -- or environment?
+        end
+    end
+    local name = os.name
+    if name == "windows" then
+        fix("OSFONTDIR", "c:/windows/fonts//")
+    elseif name == "macosx" then
+        fix("OSFONTDIR", "$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//")
+    else
+        -- bad luck
+    end
+    fix("LUAINPUTS"   , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm
+    fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS")
+    fix("LUATEXLIBS"  , ".;$TEXMF/luatex/lua//")
+end
+
+function resolvers.bare_variable(str) -- assumes str is a string
+    return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2"))
+end
+
+function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail'
+    if n then
+        trackers.disable("resolvers.*")
+        trackers.enable("resolvers."..n)
+    end
+end
+
+resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE"))
+
+function resolvers.osenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
+        if e == nil then
+         -- value = "" -- false
+        else
+            value = resolvers.bare_variable(e)
+        end
+        ie[key] = value
+    end
+    return value or ""
+end
+
+function resolvers.env(key)
+    return instance.environment[key] or resolvers.osenv(key)
+end
+
+--
+
+local function expand_vars(lst) -- simple vars
+    local variables, env = instance.variables, resolvers.env
+    local function resolve(a)
+        return variables[a] or env(a)
+    end
+    for k=1,#lst do
+        lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve)
+    end
+end
+
+local function expanded_var(var) -- simple vars
+    local function resolve(a)
+        return instance.variables[a] or resolvers.env(a)
+    end
+    return (gsub(var,"%$([%a%d%_%-]+)",resolve))
+end
+
+local function entry(entries,name)
+    if name and (name ~= "") then
+        name = gsub(name,'%$','')
+        local result = entries[name..'.'..instance.progname] or entries[name]
+        if result then
+            return result
+        else
+            result = resolvers.env(name)
+            if result then
+                instance.variables[name] = result
+                resolvers.expand_variables()
+                return instance.expansions[name] or ""
+            end
+        end
+    end
+    return ""
+end
+
+local function is_entry(entries,name)
+    if name and name ~= "" then
+        name = gsub(name,'%$','')
+        return (entries[name..'.'..instance.progname] or entries[name]) ~= nil
+    else
+        return false
+    end
+end
+
+-- {a,b,c,d}
+-- a,b,c/{p,q,r},d
+-- a,b,c/{p,q,r}/d/{x,y,z}//
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a{b,c}{d,e}f
+-- {a,b,c,d}
+-- {a,b,c/{p,q,r},d}
+-- {a,b,c/{p,q,r}/d/{x,y,z}//}
+-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
+-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
+-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
+
+-- this one is better and faster, but it took me a while to realize
+-- that this kind of replacement is cleaner than messy parsing and
+-- fuzzy concatenating we can probably gain a bit with selectively
+-- applying lpeg, but experiments with lpeg parsing this proved not to
+-- work that well; the parsing is ok, but dealing with the resulting
+-- table is a pain because we need to work inside-out recursively
+
+local function splitpathexpr(str, t, validate)
+    -- no need for further optimization as it is only called a
+    -- few times, we can use lpeg for the sub; we could move
+    -- the local functions outside the body
+    t = t or { }
+    str = gsub(str,",}",",@}")
+    str = gsub(str,"{,","{@,")
+ -- str = "@" .. str .. "@"
+    local ok, done
+    local function do_first(a,b)
+        local t = { }
+        for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_second(a,b)
+        local t = { }
+        for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_both(a,b)
+        local t = { }
+        for sa in gmatch(a,"[^,]+") do
+            for sb in gmatch(b,"[^,]+") do
+                t[#t+1] = sa .. sb
+            end
+        end
+        return "{" .. concat(t,",") .. "}"
+    end
+    local function do_three(a,b,c)
+        return a .. b.. c
+    end
+    while true do
+        done = false
+        while true do
+            str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second)
+            if ok > 0 then done = true else break end
+        end
+        while true do
+            str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both)
+            if ok > 0 then done = true else break end
+        end
+        str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three)
+        if ok > 0 then done = true end
+        if not done then break end
+    end
+    str = gsub(str,"[{}]", "")
+    str = gsub(str,"@","")
+    if validate then
+        for s in gmatch(str,"[^,]+") do
+            s = validate(s)
+            if s then t[#t+1] = s end
+        end
+    else
+        for s in gmatch(str,"[^,]+") do
+            t[#t+1] = s
+        end
+    end
+    return t
+end
+
+local function expanded_path_from_list(pathlist) -- maybe not a list, just a path
+    -- a previous version fed back into pathlist
+    local newlist, ok = { }, false
+    for k=1,#pathlist do
+        if find(pathlist[k],"[{}]") then
+            ok = true
+            break
+        end
+    end
+    if ok then
+        local function validate(s)
+            s = file.collapse_path(s)
+            return s ~= "" and not find(s,dummy_path_expr) and s
+        end
+        for k=1,#pathlist do
+            splitpathexpr(pathlist[k],newlist,validate)
+        end
+    else
+        for k=1,#pathlist do
+            for p in gmatch(pathlist[k],"([^,]+)") do
+                p = file.collapse_path(p)
+                if p ~= "" then newlist[#newlist+1] = p end
+            end
+        end
+    end
+    return newlist
+end
+
+-- we follow a rather traditional approach:
+--
+-- (1) texmf.cnf given in TEXMFCNF
+-- (2) texmf.cnf searched in default variable
+--
+-- also we now follow the stupid route: if not set then just assume *one*
+-- cnf file under texmf (i.e. distribution)
+
+resolvers.ownpath     = resolvers.ownpath or nil
+resolvers.ownbin      = resolvers.ownbin  or arg[-2] or arg[-1] or arg[0] or "luatex"
+resolvers.autoselfdir = true -- false may be handy for debugging
+
+function resolvers.getownpath()
+    if not resolvers.ownpath then
+        if resolvers.autoselfdir and os.selfdir then
+            resolvers.ownpath = os.selfdir
+        else
+            local binary = resolvers.ownbin
+            if os.platform == "windows" then
+                binary = file.replacesuffix(binary,"exe")
+            end
+            for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+                local b = file.join(p,binary)
+                if lfs.isfile(b) then
+                    -- we assume that after changing to the path the currentdir function
+                    -- resolves to the real location and use this side effect here; this
+                    -- trick is needed because on the mac installations use symlinks in the
+                    -- path instead of real locations
+                    local olddir = lfs.currentdir()
+                    if lfs.chdir(p) then
+                        local pp = lfs.currentdir()
+                        if trace_verbose and p ~= pp then
+                            logs.report("fileio","following symlink %s to %s",p,pp)
+                        end
+                        resolvers.ownpath = pp
+                        lfs.chdir(olddir)
+                    else
+                        if trace_verbose then
+                            logs.report("fileio","unable to check path %s",p)
+                        end
+                        resolvers.ownpath =  p
+                    end
+                    break
+                end
+            end
+        end
+        if not resolvers.ownpath then resolvers.ownpath = '.' end
+    end
+    return resolvers.ownpath
+end
+
+local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" }
+
+local function identify_own()
+    local ownpath = resolvers.getownpath() or lfs.currentdir()
+    local ie = instance.environment
+    if ownpath then
+        if resolvers.env('SELFAUTOLOC')    == "" then os.env['SELFAUTOLOC']    = file.collapse_path(ownpath) end
+        if resolvers.env('SELFAUTODIR')    == "" then os.env['SELFAUTODIR']    = file.collapse_path(ownpath .. "/..") end
+        if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end
+    else
+        logs.report("fileio","error: unable to locate ownpath")
+        os.exit()
+    end
+    if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end
+    if resolvers.env('TEXOS')    == "" then os.env['TEXOS']    = resolvers.env('SELFAUTODIR') end
+    if resolvers.env('TEXROOT')  == "" then os.env['TEXROOT']  = resolvers.env('SELFAUTOPARENT') end
+    if trace_verbose then
+        for i=1,#own_places do
+            local v = own_places[i]
+            logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown")
+        end
+    end
+    identify_own = function() end
+end
+
+function resolvers.identify_cnf()
+    if #instance.cnffiles == 0 then
+        -- fallback
+        identify_own()
+        -- the real search
+        resolvers.expand_variables()
+        local t = resolvers.split_path(resolvers.env('TEXMFCNF'))
+        t = expanded_path_from_list(t)
+        expand_vars(t) -- redundant
+        local function locate(filename,list)
+            for i=1,#t do
+                local ti = t[i]
+                local texmfcnf = file.collapse_path(file.join(ti,filename))
+                if lfs.isfile(texmfcnf) then
+                    list[#list+1] = texmfcnf
+                end
+            end
+        end
+        locate(resolvers.luaname,instance.luafiles)
+        locate(resolvers.cnfname,instance.cnffiles)
+    end
+end
+
+local function load_cnf_file(fname)
+    fname = resolvers.clean_path(fname)
+    local lname = file.replacesuffix(fname,'lua')
+    local f = io.open(lname)
+    if f then -- this will go
+        f:close()
+        local dname = file.dirname(fname)
+        if not instance.configuration[dname] then
+            resolvers.load_data(dname,'configuration',lname and file.basename(lname))
+            instance.order[#instance.order+1] = instance.configuration[dname]
+        end
+    else
+        f = io.open(fname)
+        if f then
+            if trace_verbose then
+                logs.report("fileio","loading %s", fname)
+            end
+            local line, data, n, k, v
+            local dname = file.dirname(fname)
+            if not instance.configuration[dname] then
+                instance.configuration[dname] = { }
+                instance.order[#instance.order+1] = instance.configuration[dname]
+            end
+            local data = instance.configuration[dname]
+            while true do
+                local line, n = f:read(), 0
+                if line then
+                    while true do -- join lines
+                        line, n = gsub(line,"\\%s*$", "")
+                        if n > 0 then
+                            line = line .. f:read()
+                        else
+                            break
+                        end
+                    end
+                    if not find(line,"^[%%#]") then
+                        local l = gsub(line,"%s*%%.*$","")
+                        local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$")
+                        if k and v and not data[k] then
+                            v = gsub(v,"[%%#].*",'')
+                            data[k] = gsub(v,"~","$HOME")
+                            instance.kpsevars[k] = true
+                        end
+                    end
+                else
+                    break
+                end
+            end
+            f:close()
+        elseif trace_verbose then
+            logs.report("fileio","skipping %s", fname)
+        end
+    end
+end
+
+local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in next, c do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = resolvers.bare_variable(v)
+                end
+            end
+        end
+    end
+end
+
+function resolvers.load_cnf()
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            load_cnf_file(fname)
+        end
+    end
+    -- instance.cnffiles contain complete names now !
+    if #instance.cnffiles == 0 then
+        if trace_verbose then
+            logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)")
+        end
+    else
+        instance.rootpath = instance.cnffiles[1]
+        for k,fname in ipairs(instance.cnffiles) do
+            instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        if instance.diskcache and not instance.renewcache then
+            resolvers.loadoldconfig(instance.cnffiles)
+            if instance.loaderror then
+                loadoldconfigdata()
+                resolvers.saveoldconfig()
+            end
+        else
+            loadoldconfigdata()
+            if instance.renewcache then
+                resolvers.saveoldconfig()
+            end
+        end
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+function resolvers.load_lua()
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = file.collapse_path(instance.rootpath)
+        resolvers.loadnewconfig()
+        collapse_cnf_data()
+    end
+    check_configuration()
+end
+
+-- database loading
+
+function resolvers.load_hash()
+    resolvers.locatelists()
+    if instance.diskcache and not instance.renewcache then
+        resolvers.loadfiles()
+        if instance.loaderror then
+            resolvers.loadlists()
+            resolvers.savefiles()
+        end
+    else
+        resolvers.loadlists()
+        if instance.renewcache then
+            resolvers.savefiles()
+        end
+    end
+end
+
+function resolvers.append_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash append: %s",tag)
+    end
+    insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.prepend_hash(type,tag,name)
+    if trace_locating then
+        logs.report("fileio","= hash prepend: %s",tag)
+    end
+    insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } )
+end
+
+function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash
+--  local t = resolvers.expanded_path_list('TEXMF') -- full expansion
+    local t = resolvers.split_path(resolvers.env('TEXMF'))
+    insert(t,1,specification)
+    local newspec = concat(t,";")
+    if instance.environment["TEXMF"] then
+        instance.environment["TEXMF"] = newspec
+    elseif instance.variables["TEXMF"] then
+        instance.variables["TEXMF"] = newspec
+    else
+        -- weird
+    end
+    resolvers.expand_variables()
+    reset_hashes()
+end
+
+-- locators
+
+function resolvers.locatelists()
+    for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do
+        if trace_verbose then
+            logs.report("fileio","locating list of %s",path)
+        end
+        resolvers.locatedatabase(file.collapse_path(path))
+    end
+end
+
+function resolvers.locatedatabase(specification)
+    return resolvers.methodhandler('locators', specification)
+end
+
+function resolvers.locators.tex(specification)
+    if specification and specification ~= '' and lfs.isdir(specification) then
+        if trace_locating then
+            logs.report("fileio",'! tex locator found: %s',specification)
+        end
+        resolvers.append_hash('file',specification,filename)
+    elseif trace_locating then
+        logs.report("fileio",'? tex locator not found: %s',specification)
+    end
+end
+
+-- hashers
+
+function resolvers.hashdatabase(tag,name)
+    return resolvers.methodhandler('hashers',tag,name)
+end
+
+function resolvers.loadfiles()
+    instance.loaderror = false
+    instance.files = { }
+    if not instance.renewcache then
+        for _, hash in ipairs(instance.hashes) do
+            resolvers.hashdatabase(hash.tag,hash.name)
+            if instance.loaderror then break end
+        end
+    end
+end
+
+function resolvers.hashers.tex(tag,name)
+    resolvers.load_data(tag,'files')
+end
+
+-- generators:
+
+function resolvers.loadlists()
+    for _, hash in ipairs(instance.hashes) do
+        resolvers.generatedatabase(hash.tag)
+    end
+end
+
+function resolvers.generatedatabase(specification)
+    return resolvers.methodhandler('generators', specification)
+end
+
+-- starting with . or .. etc or funny char
+
+local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t"))
+
+function resolvers.generators.tex(specification)
+    local tag = specification
+    if trace_verbose then
+        logs.report("fileio","scanning path %s",specification)
+    end
+    instance.files[tag] = { }
+    local files = instance.files[tag]
+    local n, m, r = 0, 0, 0
+    local spec = specification .. '/'
+    local attributes = lfs.attributes
+    local directory = lfs.dir
+    local function action(path)
+        local full
+        if path then
+            full = spec .. path .. '/'
+        else
+            full = spec
+        end
+        for name in directory(full) do
+            if not weird:match(name) then
+                local mode = attributes(full..name,'mode')
+                if mode == 'file' then
+                    if path then
+                        n = n + 1
+                        local f = files[name]
+                        if f then
+                            if type(f) == 'string' then
+                                files[name] = { f, path }
+                            else
+                                f[#f+1] = path
+                            end
+                        else -- probably unique anyway
+                            files[name] = path
+                            local lower = lower(name)
+                            if name ~= lower then
+                                files["remap:"..lower] = name
+                                r = r + 1
+                            end
+                        end
+                    end
+                elseif mode == 'directory' then
+                    m = m + 1
+                    if path then
+                        action(path..'/'..name)
+                    else
+                        action(name)
+                    end
+                end
+            end
+        end
+    end
+    action()
+    if trace_verbose then
+        logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r)
+    end
+end
+
+-- savers, todo
+
+function resolvers.savefiles()
+    resolvers.save_data('files')
+end
+
+-- A config (optionally) has the paths split in tables. Internally
+-- we join them and split them after the expansion has taken place. This
+-- is more convenient.
+
+function resolvers.splitconfig()
+    for i,c in ipairs(instance) do
+        for k,v in pairs(c) do
+            if type(v) == 'string' then
+                local t = file.split_path(v)
+                if #t > 1 then
+                    c[k] = t
+                end
+            end
+        end
+    end
+end
+
+function resolvers.joinconfig()
+    for i,c in ipairs(instance.order) do
+        for k,v in pairs(c) do -- ipairs?
+            if type(v) == 'table' then
+                c[k] = file.join_path(v)
+            end
+        end
+    end
+end
+function resolvers.split_path(str)
+    if type(str) == 'table' then
+        return str
+    else
+        return file.split_path(str)
+    end
+end
+function resolvers.join_path(str)
+    if type(str) == 'table' then
+        return file.join_path(str)
+    else
+        return str
+    end
+end
+
+function resolvers.splitexpansions()
+    local ie = instance.expansions
+    for k,v in next, ie do
+        local t, h = { }, { }
+        for _,vv in ipairs(file.split_path(v)) do
+            if vv ~= "" and not h[vv] then
+                t[#t+1] = vv
+                h[vv] = true
+            end
+        end
+        if #t > 1 then
+            ie[k] = t
+        else
+            ie[k] = t[1]
+        end
+    end
+end
+
+-- end of split/join code
+
+function resolvers.saveoldconfig()
+    resolvers.splitconfig()
+    resolvers.save_data('configuration')
+    resolvers.joinconfig()
+end
+
+resolvers.configbanner = [[
+-- This is a Luatex configuration file created by 'luatools.lua' or
+-- 'luatex.exe' directly. For comment, suggestions and questions you can
+-- contact the ConTeXt Development Team. This configuration file is
+-- not copyrighted. [HH & TH]
+]]
+
+function resolvers.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local function dump(k,v,m) -- could be moved inline
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+        end
+    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sortedkeys(files)) do -- ipairs
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sortedkeys(fk)) do -- ipairs
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
+            end
+        end
+    else
+        for k, v in next, files do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in next, v do
+                    t[#t+1] = dump(kk,vv,"\t\t")
+                end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,v,"\t")
+            end
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+function resolvers.save_data(dataname, makename) -- untested without cache overload
+    for cachename, files in next, instance[dataname] do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. ".lua", name .. ".luc"
+        if trace_verbose then
+            logs.report("fileio","preparing %s for %s",dataname,cachename)
+        end
+        for k, v in next, files do
+            if type(v) == "table" and #v == 1 then
+                files[k] = v[1]
+            end
+        end
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = resolvers.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local ok = io.savedata(luaname,resolvers.serialize(data))
+        if ok then
+            if trace_verbose then
+                logs.report("fileio","%s saved in %s",dataname,luaname)
+            end
+            if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip
+                if trace_verbose then
+                    logs.report("fileio","%s compiled to %s",dataname,lucname)
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname)
+                end
+                os.remove(lucname)
+            end
+        elseif trace_verbose then
+            logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname)
+        end
+    end
+end
+
+function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua")
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then
+            if trace_verbose then
+                logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = data.content
+        else
+            if trace_verbose then
+                logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+            end
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
+        end
+    elseif trace_verbose then
+        logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename)
+    end
+end
+
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
+
+function resolvers.resetconfig()
+    identify_own()
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function resolvers.loadnewconfig()
+    for _, cnf in ipairs(instance.luafiles) do
+        local pathname = file.dirname(cnf)
+        local filename = file.join(pathname,resolvers.luaname)
+        local blob = loadfile(filename)
+        if blob then
+            local data = blob()
+            if data then
+                if trace_verbose then
+                    logs.report("fileio","loading configuration file %s",filename)
+                end
+                if true then
+                    -- flatten to variable.progname
+                    local t = { }
+                    for k, v in next, data do -- v = progname
+                        if type(v) == "string" then
+                            t[k] = v
+                        else
+                            for kk, vv in next, v do -- vv = variable
+                                if type(vv) == "string" then
+                                    t[vv.."."..v] = kk
+                                end
+                            end
+                        end
+                    end
+                    instance['setup'][pathname] = t
+                else
+                    instance['setup'][pathname] = data
+                end
+            else
+                if trace_verbose then
+                    logs.report("fileio","skipping configuration file %s",filename)
+                end
+                instance['setup'][pathname] = { }
+                instance.loaderror = true
+            end
+        elseif trace_verbose then
+            logs.report("fileio","skipping configuration file %s",filename)
+        end
+        instance.order[#instance.order+1] = instance.setup[pathname]
+        if instance.loaderror then break end
+    end
+end
+
+function resolvers.loadoldconfig()
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            resolvers.load_data(dname,'configuration')
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
+        end
+    end
+    resolvers.joinconfig()
+end
+
+function resolvers.expand_variables()
+    local expansions, environment, variables = { }, instance.environment, instance.variables
+    local env = resolvers.env
+    instance.expansions = expansions
+    if instance.engine   ~= "" then environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then environment['progname'] = instance.progname end
+    for k,v in next, environment do
+        local a, b = match(k,"^(%a+)%_(.*)%s*$")
+        if a and b then
+            expansions[a..'.'..b] = v
+        else
+            expansions[k] = v
+        end
+    end
+    for k,v in next, environment do -- move environment to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    for k,v in next, variables do -- move variables to expansions
+        if not expansions[k] then expansions[k] = v end
+    end
+    local busy = false
+    local function resolve(a)
+        busy = true
+        return expansions[a] or env(a)
+    end
+    while true do
+        busy = false
+        for k,v in next, expansions do
+            local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve)
+            local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve)
+            if n > 0 or m > 0 then
+                expansions[k]= s
+            end
+        end
+        if not busy then break end
+    end
+    for k,v in next, expansions do
+        expansions[k] = gsub(v,"\\", '/')
+    end
+end
+
+function resolvers.variable(name)
+    return entry(instance.variables,name)
+end
+
+function resolvers.expansion(name)
+    return entry(instance.expansions,name)
+end
+
+function resolvers.is_variable(name)
+    return is_entry(instance.variables,name)
+end
+
+function resolvers.is_expansion(name)
+    return is_entry(instance.expansions,name)
+end
+
+function resolvers.unexpanded_path_list(str)
+    local pth = resolvers.variable(str)
+    local lst = resolvers.split_path(pth)
+    return expanded_path_from_list(lst)
+end
+
+function resolvers.unexpanded_path(str)
+    return file.join_path(resolvers.unexpanded_path_list(str))
+end
+
+do -- no longer needed
+
+    local done = { }
+
+    function resolvers.reset_extra_path()
+        local ep = instance.extra_paths
+        if not ep then
+            ep, done = { }, { }
+            instance.extra_paths = ep
+        elseif #ep > 0 then
+            instance.lists, done = { }, { }
+        end
+    end
+
+    function resolvers.register_extra_path(paths,subpaths)
+        local ep = instance.extra_paths or { }
+        local n = #ep
+        if paths and paths ~= "" then
+            if subpaths and subpaths ~= "" then
+                for p in gmatch(paths,"[^,]+") do
+                    -- we gmatch each step again, not that fast, but used seldom
+                    for s in gmatch(subpaths,"[^,]+") do
+                        local ps = p .. "/" .. s
+                        if not done[ps] then
+                            ep[#ep+1] = resolvers.clean_path(ps)
+                            done[ps] = true
+                        end
+                    end
+                end
+            else
+                for p in gmatch(paths,"[^,]+") do
+                    if not done[p] then
+                        ep[#ep+1] = resolvers.clean_path(p)
+                        done[p] = true
+                    end
+                end
+            end
+        elseif subpaths and subpaths ~= "" then
+            for i=1,n do
+                -- we gmatch each step again, not that fast, but used seldom
+                for s in gmatch(subpaths,"[^,]+") do
+                    local ps = ep[i] .. "/" .. s
+                    if not done[ps] then
+                        ep[#ep+1] = resolvers.clean_path(ps)
+                        done[ps] = true
+                    end
+                end
+            end
+        end
+        if #ep > 0 then
+            instance.extra_paths = ep -- register paths
+        end
+        if #ep > n then
+            instance.lists = { } -- erase the cache
+        end
+    end
+
+end
+
+local function made_list(instance,list)
+    local ep = instance.extra_paths
+    if not ep or #ep == 0 then
+        return list
+    else
+        local done, new = { }, { }
+        -- honour . .. ../.. but only when at the start
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                if find(v,"^[%.%/]$") then
+                    done[v] = true
+                    new[#new+1] = v
+                else
+                    break
+                end
+            end
+        end
+        -- first the extra paths
+        for k=1,#ep do
+            local v = ep[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        -- next the formal paths
+        for k=1,#list do
+            local v = list[k]
+            if not done[v] then
+                done[v] = true
+                new[#new+1] = v
+            end
+        end
+        return new
+    end
+end
+
+function resolvers.clean_path_list(str)
+    local t = resolvers.expanded_path_list(str)
+    if t then
+        for i=1,#t do
+            t[i] = file.collapse_path(resolvers.clean_path(t[i]))
+        end
+    end
+    return t
+end
+
+function resolvers.expand_path(str)
+    return file.join_path(resolvers.expanded_path_list(str))
+end
+
+function resolvers.expanded_path_list(str)
+    if not str then
+        return ep or { }
+    elseif instance.savelists then
+        -- engine+progname hash
+        str = gsub(str,"%$","")
+        if not instance.lists[str] then -- cached
+            local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str)))
+            instance.lists[str] = expanded_path_from_list(lst)
+        end
+        return instance.lists[str]
+    else
+        local lst = resolvers.split_path(resolvers.expansion(str))
+        return made_list(instance,expanded_path_from_list(lst))
+    end
+end
+
+function resolvers.expanded_path_list_from_var(str) -- brrr
+    local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$",""))
+    if tmp ~= "" then
+        return resolvers.expanded_path_list(str)
+    else
+        return resolvers.expanded_path_list(tmp)
+    end
+end
+
+function resolvers.expand_path_from_var(str)
+    return file.join_path(resolvers.expanded_path_list_from_var(str))
+end
+
+function resolvers.format_of_var(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+function resolvers.format_of_suffix(str)
+    return suffixmap[file.extname(str)] or 'tex'
+end
+
+function resolvers.variable_of_format(str)
+    return formats[str] or formats[alternatives[str]] or ''
+end
+
+function resolvers.var_of_format_or_suffix(str)
+    local v = formats[str]
+    if v then
+        return v
+    end
+    v = formats[alternatives[str]]
+    if v then
+        return v
+    end
+    v = suffixmap[file.extname(str)]
+    if v then
+        return formats[isf]
+    end
+    return ''
+end
+
+function resolvers.expand_braces(str) -- output variable and brace expansion of STRING
+    local ori = resolvers.variable(str)
+    local pth = expanded_path_from_list(resolvers.split_path(ori))
+    return file.join_path(pth)
+end
+
+resolvers.isreadable = { }
+
+function resolvers.isreadable.file(name)
+    local readable = lfs.isfile(name) -- brrr
+    if trace_detail then
+        if readable then
+            logs.report("fileio","+ readable: %s",name)
+        else
+            logs.report("fileio","- readable: %s", name)
+        end
+    end
+    return readable
+end
+
+resolvers.isreadable.tex = resolvers.isreadable.file
+
+-- name
+-- name/name
+
+local function collect_files(names)
+    local filelist = { }
+    for k=1,#names do
+        local fname = names[k]
+        if trace_detail then
+            logs.report("fileio","? blobpath asked: %s",fname)
+        end
+        local bname = file.basename(fname)
+        local dname = file.dirname(fname)
+        if dname == "" or find(dname,"^%.") then
+            dname = false
+        else
+            dname = "/" .. dname .. "$"
+        end
+        local hashes = instance.hashes
+        for h=1,#hashes do
+            local hash = hashes[h]
+            local blobpath = hash.tag
+            local files = blobpath and instance.files[blobpath]
+            if files then
+                if trace_detail then
+                    logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname)
+                end
+                local blobfile = files[bname]
+                if not blobfile then
+                    local rname = "remap:"..bname
+                    blobfile = files[rname]
+                    if blobfile then
+                        bname = files[rname]
+                        blobfile = files[bname]
+                    end
+                end
+                if blobfile then
+                    if type(blobfile) == 'string' then
+                        if not dname or find(blobfile,dname) then
+                            filelist[#filelist+1] = {
+                                hash.type,
+                                file.join(blobpath,blobfile,bname), -- search
+                                resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result
+                            }
+                        end
+                    else
+                        for kk=1,#blobfile do
+                            local vv = blobfile[kk]
+                            if not dname or find(vv,dname) then
+                                filelist[#filelist+1] = {
+                                    hash.type,
+                                    file.join(blobpath,vv,bname), -- search
+                                    resolvers.concatinators[hash.type](blobpath,vv,bname) -- result
+                                }
+                            end
+                        end
+                    end
+                end
+            elseif trace_locating then
+                logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname)
+            end
+        end
+    end
+    if #filelist > 0 then
+        return filelist
+    else
+        return nil
+    end
+end
+
+function resolvers.suffix_of_format(str)
+    if suffixes[str] then
+        return suffixes[str][1]
+    else
+        return ""
+    end
+end
+
+function resolvers.suffixes_of_format(str)
+    if suffixes[str] then
+        return suffixes[str]
+    else
+        return {}
+    end
+end
+
+function resolvers.register_in_trees(name)
+    if not find(name,"^%.") then
+        instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one
+    end
+end
+
+-- split the next one up for readability (bu this module needs a cleanup anyway)
+
+local function can_be_dir(name) -- can become local
+    local fakepaths = instance.fakepaths
+    if not fakepaths[name] then
+        if lfs.isdir(name) then
+            fakepaths[name] = 1 -- directory
+        else
+            fakepaths[name] = 2 -- no directory
+        end
+    end
+    return (fakepaths[name] == 1)
+end
+
+local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc)
+    local result = collected or { }
+    local stamp  = nil
+    filename = file.collapse_path(filename)  -- elsewhere
+    filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere
+    -- speed up / beware: format problem
+    if instance.remember then
+        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format
+        if instance.found[stamp] then
+            if trace_locating then
+                logs.report("fileio",'! remembered: %s',filename)
+            end
+            return instance.found[stamp]
+        end
+    end
+    if not dangerous[instance.format or "?"] then
+        if resolvers.isreadable.file(filename) then
+            if trace_detail then
+                logs.report("fileio",'= found directly: %s',filename)
+            end
+            instance.found[stamp] = { filename }
+            return { filename }
+        end
+    end
+    if find(filename,'%*') then
+        if trace_locating then
+            logs.report("fileio",'! wildcard: %s', filename)
+        end
+        result = resolvers.find_wildcard_files(filename)
+    elseif file.is_qualified_path(filename) then
+        if resolvers.isreadable.file(filename) then
+            if trace_locating then
+                logs.report("fileio",'! qualified: %s', filename)
+            end
+            result = { filename }
+        else
+            local forcedname, ok, suffix = "", false, file.extname(filename)
+            if suffix == "" then -- why
+                if instance.format == "" then
+                    forcedname = filename .. ".tex"
+                    if resolvers.isreadable.file(forcedname) then
+                        if trace_locating then
+                            logs.report("fileio",'! no suffix, forcing standard filetype: tex')
+                        end
+                        result, ok = { forcedname }, true
+                    end
+                else
+                    local suffixes = resolvers.suffixes_of_format(instance.format)
+                    for _, s in next, suffixes do
+                        forcedname = filename .. "." .. s
+                        if resolvers.isreadable.file(forcedname) then
+                            if trace_locating then
+                                logs.report("fileio",'! no suffix, forcing format filetype: %s', s)
+                            end
+                            result, ok = { forcedname }, true
+                            break
+                        end
+                    end
+                end
+            end
+            if not ok and suffix ~= "" then
+                -- try to find in tree (no suffix manipulation), here we search for the
+                -- matching last part of the name
+                local basename = file.basename(filename)
+                local pattern = (filename .. "$"):gsub("([%.%-])","%%%1")
+                local savedformat = instance.format
+                local format = savedformat or ""
+                if format == "" then
+                    instance.format = resolvers.format_of_suffix(suffix)
+                end
+                if not format then
+                    instance.format = "othertextfiles" -- kind of everything, maybe texinput is better
+                end
+                --
+                local resolved = collect_instance_files(basename)
+                if #result == 0 then
+                    local lowered = lower(basename)
+                    if filename ~= lowered then
+                        resolved = collect_instance_files(lowered)
+                    end
+                end
+                resolvers.format = savedformat
+                --
+                for r=1,#resolved do
+                    local rr = resolved[r]
+                    if rr:find(pattern) then
+                        result[#result+1], ok = rr, true
+                    end
+                end
+                -- a real wildcard:
+                --
+                -- if not ok then
+                --     local filelist = collect_files({basename})
+                --     for f=1,#filelist do
+                --         local ff = filelist[f][3] or ""
+                --         if ff:find(pattern) then
+                --             result[#result+1], ok = ff, true
+                --         end
+                --     end
+                -- end
+            end
+            if not ok and trace_locating then
+                logs.report("fileio",'? qualified: %s', filename)
+            end
+        end
+    else
+        -- search spec
+        local filetype, extra, done, wantedfiles, ext = '', nil, false, { }, file.extname(filename)
+        if ext == "" then
+            if not instance.force_suffixes then
+                wantedfiles[#wantedfiles+1] = filename
+            end
+        else
+            wantedfiles[#wantedfiles+1] = filename
+        end
+        if instance.format == "" then
+            if ext == "" then
+                local forcedname = filename .. '.tex'
+                wantedfiles[#wantedfiles+1] = forcedname
+                filetype = resolvers.format_of_suffix(forcedname)
+                if trace_locating then
+                    logs.report("fileio",'! forcing filetype: %s',filetype)
+                end
+            else
+                filetype = resolvers.format_of_suffix(filename)
+                if trace_locating then
+                    logs.report("fileio",'! using suffix based filetype: %s',filetype)
+                end
+            end
+        else
+            if ext == "" then
+                local suffixes = resolvers.suffixes_of_format(instance.format)
+                for _, s in next, suffixes do
+                    wantedfiles[#wantedfiles+1] = filename .. "." .. s
+                end
+            end
+            filetype = instance.format
+            if trace_locating then
+                logs.report("fileio",'! using given filetype: %s',filetype)
+            end
+        end
+        local typespec = resolvers.variable_of_format(filetype)
+        local pathlist = resolvers.expanded_path_list(typespec)
+        if not pathlist or #pathlist == 0 then
+            -- no pathlist, access check only / todo == wildcard
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+                logs.report("fileio",'? filetype: %s',filetype or '?')
+                logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | "))
+            end
+            for k=1,#wantedfiles do
+                local fname = wantedfiles[k]
+                if fname and resolvers.isreadable.file(fname) then
+                    filename, done = fname, true
+                    result[#result+1] = file.join('.',fname)
+                    break
+                end
+            end
+            -- this is actually 'other text files' or 'any' or 'whatever'
+            local filelist = collect_files(wantedfiles)
+            local fl = filelist and filelist[1]
+            if fl then
+                filename = fl[3]
+                result[#result+1] = filename
+                done = true
+            end
+        else
+            -- list search
+            local filelist = collect_files(wantedfiles)
+            local doscan, recurse
+            if trace_detail then
+                logs.report("fileio",'? filename: %s',filename)
+            end
+            -- a bit messy ... esp the doscan setting here
+            for k=1,#pathlist do
+                local path = pathlist[k]
+                if find(path,"^!!") then doscan  = false else doscan  = true  end
+                if find(path,"//$") then recurse = true  else recurse = false end
+                local pathname = gsub(path,"^!+", '')
+                done = false
+                -- using file list
+                if filelist and not (done and not instance.allresults) and recurse then
+                    -- compare list entries with permitted pattern
+                    pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences
+                    pathname = gsub(pathname,"/+$", '/.*')      -- later usage of pathname
+                    pathname = gsub(pathname,"//", '/.-/')      -- not ok for /// but harmless
+                    local expr = "^" .. pathname
+                    for k=1,#filelist do
+                        local fl = filelist[k]
+                        local f = fl[2]
+                        if find(f,expr) then
+                            if trace_detail then
+                                logs.report("fileio",'= found in hash: %s',f)
+                            end
+                            --- todo, test for readable
+                            result[#result+1] = fl[3]
+                            resolvers.register_in_trees(f) -- for tracing used files
+                            done = true
+                            if not instance.allresults then break end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- check if on disk / unchecked / does not work at all / also zips
+                    if resolvers.splitmethod(pathname).scheme == 'file' then -- ?
+                        local pname = gsub(pathname,"%.%*$",'')
+                        if not find(pname,"%*") then
+                            local ppname = gsub(pname,"/+$","")
+                            if can_be_dir(ppname) then
+                                for k=1,#wantedfiles do
+                                    local w = wantedfiles[k]
+                                    local fname = file.join(ppname,w)
+                                    if resolvers.isreadable.file(fname) then
+                                        if trace_detail then
+                                            logs.report("fileio",'= found by scanning: %s',fname)
+                                        end
+                                        result[#result+1] = fname
+                                        done = true
+                                        if not instance.allresults then break end
+                                    end
+                                end
+                            else
+                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
+                            end
+                        end
+                    end
+                end
+                if not done and doscan then
+                    -- todo: slow path scanning
+                end
+                if done and not instance.allresults then break end
+            end
+        end
+    end
+    for k=1,#result do
+        result[k] = file.collapse_path(result[k])
+    end
+    if instance.remember then
+        instance.found[stamp] = result
+    end
+    return result
+end
+
+if not resolvers.concatinators  then resolvers.concatinators = { } end
+
+resolvers.concatinators.tex  = file.join
+resolvers.concatinators.file = resolvers.concatinators.tex
+
+function resolvers.find_files(filename,filetype,mustexist)
+    if type(mustexist) == boolean then
+        -- all set
+    elseif type(filetype) == 'boolean' then
+        filetype, mustexist = nil, false
+    elseif type(filetype) ~= 'string' then
+        filetype, mustexist = nil, false
+    end
+    instance.format = filetype or ''
+    local result = collect_instance_files(filename)
+    if #result == 0 then
+        local lowered = lower(filename)
+        if filename ~= lowered then
+            return collect_instance_files(lowered)
+        end
+    end
+    instance.format = ''
+    return result
+end
+
+function resolvers.find_file(filename,filetype,mustexist)
+    return (resolvers.find_files(filename,filetype,mustexist)[1] or "")
+end
+
+function resolvers.find_given_files(filename)
+    local bname, result = file.basename(filename), { }
+    local hashes = instance.hashes
+    for k=1,#hashes do
+        local hash = hashes[k]
+        local files = instance.files[hash.tag]
+        local blist = files[bname]
+        if not blist then
+            local rname = "remap:"..bname
+            blist = files[rname]
+            if blist then
+                bname = files[rname]
+                blist = files[bname]
+            end
+        end
+        if blist then
+            if type(blist) == 'string' then
+                result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or ""
+                if not instance.allresults then break end
+            else
+                for kk=1,#blist do
+                    local vv = blist[kk]
+                    result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or ""
+                    if not instance.allresults then break end
+                end
+            end
+        end
+    end
+    return result
+end
+
+function resolvers.find_given_file(filename)
+    return (resolvers.find_given_files(filename)[1] or "")
+end
+
+local function doit(path,blist,bname,tag,kind,result,allresults)
+    local done = false
+    if blist and kind then
+        if type(blist) == 'string' then
+            -- make function and share code
+            if find(lower(blist),path) then
+                result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or ""
+                done = true
+            end
+        else
+            for kk=1,#blist do
+                local vv = blist[kk]
+                if find(lower(vv),path) then
+                    result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or ""
+                    done = true
+                    if not allresults then break end
+                end
+            end
+        end
+    end
+    return done
+end
+
+function resolvers.find_wildcard_files(filename) -- todo: remap:
+    local result = { }
+    local bname, dname = file.basename(filename), file.dirname(filename)
+    local path = gsub(dname,"^*/","")
+    path = gsub(path,"*",".*")
+    path = gsub(path,"-","%%-")
+    if dname == "" then
+        path = ".*"
+    end
+    local name = bname
+    name = gsub(name,"*",".*")
+    name = gsub(name,"-","%%-")
+    path = lower(path)
+    name = lower(name)
+    local files, allresults, done = instance.files, instance.allresults, false
+    if find(name,"%*") then
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            for kk, hh in next, files[hash.tag] do
+                if not find(kk,"^remap:") then
+                    if find(lower(kk),name) then
+                        if doit(path,hh,kk,tag,kind,result,allresults) then done = true end
+                        if done and not allresults then break end
+                    end
+                end
+            end
+        end
+    else
+        local hashes = instance.hashes
+        for k=1,#hashes do
+            local hash = hashes[k]
+            local tag, kind = hash.tag, hash.type
+            if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end
+            if done and not allresults then break end
+        end
+    end
+    -- we can consider also searching the paths not in the database, but then
+    -- we end up with a messy search (all // in all path specs)
+    return result
+end
+
+function resolvers.find_wildcard_file(filename)
+    return (resolvers.find_wildcard_files(filename)[1] or "")
+end
+
+-- main user functions
+
+function resolvers.automount()
+    -- implemented later
+end
+
+function resolvers.load(option)
+    statistics.starttiming(instance)
+    resolvers.resetconfig()
+    resolvers.identify_cnf()
+    resolvers.load_lua()
+    resolvers.expand_variables()
+    resolvers.load_cnf()
+    resolvers.expand_variables()
+    if option ~= "nofiles" then
+        resolvers.load_hash()
+        resolvers.automount()
+    end
+    statistics.stoptiming(instance)
+end
+
+function resolvers.for_files(command, files, filetype, mustexist)
+    if files and #files > 0 then
+        local function report(str)
+            if trace_verbose then
+                logs.report("fileio",str) -- has already verbose
+            else
+                print(str)
+            end
+        end
+        if trace_verbose then
+            report('')
+        end
+        for _, file in ipairs(files) do
+            local result = command(file,filetype,mustexist)
+            if type(result) == 'string' then
+                report(result)
+            else
+                for _,v in ipairs(result) do
+                    report(v)
+                end
+            end
+        end
+    end
+end
+
+-- strtab
+
+resolvers.var_value  = resolvers.variable   -- output the value of variable $STRING.
+resolvers.expand_var = resolvers.expansion  -- output variable expansion of STRING.
+
+function resolvers.show_path(str)     -- output search path for file type NAME
+    return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str)))
+end
+
+-- resolvers.find_file(filename)
+-- resolvers.find_file(filename, filetype, mustexist)
+-- resolvers.find_file(filename, mustexist)
+-- resolvers.find_file(filename, filetype)
+
+function resolvers.register_file(files, name, path)
+    if files[name] then
+        if type(files[name]) == 'string' then
+            files[name] = { files[name], path }
+        else
+            files[name] = path
+        end
+    else
+        files[name] = path
+    end
+end
+
+function resolvers.splitmethod(filename)
+    if not filename then
+        return { } -- safeguard
+    elseif type(filename) == "table" then
+        return filename -- already split
+    elseif not find(filename,"://") then
+        return { scheme="file", path = filename, original=filename } -- quick hack
+    else
+        return url.hashed(filename)
+    end
+end
+
+function table.sequenced(t,sep) -- temp here
+    local s = { }
+    for k, v in pairs(t) do -- pairs?
+        s[#s+1] = k .. "=" .. v
+    end
+    return concat(s, sep or " | ")
+end
+
+function resolvers.methodhandler(what, filename, filetype) -- ...
+    local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb
+    local scheme = specification.scheme
+    if resolvers[what][scheme] then
+        if trace_locating then
+            logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification))
+        end
+        return resolvers[what][scheme](filename,filetype) -- todo: specification
+    else
+        return resolvers[what].tex(filename,filetype) -- todo: specification
+    end
+end
+
+function resolvers.clean_path(str)
+    if str then
+        str = gsub(str,"\\","/")
+        str = gsub(str,"^!+","")
+        str = gsub(str,"^~",resolvers.homedir)
+        return str
+    else
+        return nil
+    end
+end
+
+function resolvers.do_with_path(name,func)
+    for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs?
+        func("^"..resolvers.clean_path(v))
+    end
+end
+
+function resolvers.do_with_var(name,func)
+    func(expanded_var(name))
+end
+
+function resolvers.with_files(pattern,handle)
+    for _, hash in ipairs(instance.hashes) do
+        local blobpath = hash.tag
+        local blobtype = hash.type
+        if blobpath then
+            local files = instance.files[blobpath]
+            if files then
+                for k,v in next, files do
+                    if find(k,"^remap:") then
+                        k = files[k]
+                        v = files[k] -- chained
+                    end
+                    if find(k,pattern) then
+                        if type(v) == "string" then
+                            handle(blobtype,blobpath,v,k)
+                        else
+                            for _,vv in pairs(v) do -- ipairs?
+                                handle(blobtype,blobpath,vv,k)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+    end
+end
+
+function resolvers.locate_format(name)
+    local barename, fmtname = name:gsub("%.%a+$",""), ""
+    if resolvers.usecache then
+        local path = file.join(caches.setpath("formats")) -- maybe platform
+        fmtname = file.join(path,barename..".fmt") or ""
+    end
+    if fmtname == "" then
+        fmtname = resolvers.find_files(barename..".fmt")[1] or ""
+    end
+    fmtname = resolvers.clean_path(fmtname)
+    if fmtname ~= "" then
+        local barename = file.removesuffix(fmtname)
+        local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui"
+        if lfs.isfile(luiname) then
+            return barename, luiname
+        elseif lfs.isfile(lucname) then
+            return barename, lucname
+        elseif lfs.isfile(luaname) then
+            return barename, luaname
+        end
+    end
+    return nil, nil
+end
+
+function resolvers.boolean_variable(str,default)
+    local b = resolvers.expansion(str)
+    if b == "" then
+        return default
+    else
+        b = toboolean(b)
+        return (b == nil and default) or b
+    end
+end
+
+texconfig.kpse_init = false
+
+kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } )
+
+-- for a while
+
+input = resolvers
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This module deals with caching data. It sets up the paths and
+implements loaders and savers for tables. Best is to set the
+following variable. When not set, the usual paths will be
+checked. Personally I prefer the (users) temporary path.

+
+
+TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
+
+
+
Currently we do no locking when we write files. This is no real
+problem because most caching involves fonts and the chance of them
+being written at the same time is small. We also need to extend
+luatools with a recache feature.

+--ldx]]--
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache = false  trackers.register("resolvers.cache", function(v) trace_cache = v end)
+
+caches = caches or { }
+
+caches.path     = caches.path or nil
+caches.base     = caches.base or "luatex-cache"
+caches.more     = caches.more or "context"
+caches.direct   = false -- true is faster but may need huge amounts of memory
+caches.tree     = false
+caches.paths    = caches.paths or nil
+caches.force    = false
+caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" }
+
+function caches.temp()
+    local cachepath = nil
+    local function check(list,isenv)
+        if not cachepath then
+            for k=1,#list do
+                local v = list[k]
+                cachepath = (isenv and (os.env[v] or "")) or v or ""
+                if cachepath == "" then
+                    -- next
+                else
+                    cachepath = resolvers.clean_path(cachepath)
+                    if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory"
+                        break
+                    elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
+                        dir.mkdirs(cachepath)
+                        if lfs.isdir(cachepath) and file.iswritable(cachepath) then
+                            break
+                        end
+                    end
+                end
+                cachepath = nil
+            end
+        end
+    end
+    check(resolvers.clean_path_list("TEXMFCACHE") or { })
+    check(caches.defaults,true)
+    if not cachepath then
+        print("\nfatal error: there is no valid (writable) cache path defined\n")
+        os.exit()
+    elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory"
+        print(format("\nfatal error: cache path %s is not a directory\n",cachepath))
+        os.exit()
+    end
+    cachepath = file.collapse_path(cachepath)
+    function caches.temp()
+        return cachepath
+    end
+    return cachepath
+end
+
+function caches.configpath()
+    return table.concat(resolvers.instance.cnffiles,";")
+end
+
+function caches.hashed(tree)
+    return md5.hex(gsub(lower(tree),"[\\\/]+","/"))
+end
+
+function caches.treehash()
+    local tree = caches.configpath()
+    if not tree or tree == "" then
+        return false
+    else
+        return caches.hashed(tree)
+    end
+end
+
+function caches.setpath(...)
+    if not caches.path then
+        if not caches.path then
+            caches.path = caches.temp()
+        end
+        caches.path = resolvers.clean_path(caches.path) -- to be sure
+        caches.tree = caches.tree or caches.treehash()
+        if caches.tree then
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
+        else
+            caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
+        end
+    end
+    if not caches.path then
+        caches.path = '.'
+    end
+    caches.path = resolvers.clean_path(caches.path)
+    if not table.is_empty({...}) then
+        local pth = dir.mkdirs(caches.path,...)
+        return pth
+    end
+    caches.path = dir.expand_name(caches.path)
+    return caches.path
+end
+
+function caches.definepath(category,subcategory)
+    return function()
+        return caches.setpath(category,subcategory)
+    end
+end
+
+function caches.setluanames(path,name)
+    return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
+end
+
+function caches.loaddata(path,name)
+    local tmaname, tmcname = caches.setluanames(path,name)
+    local loader = loadfile(tmcname) or loadfile(tmaname)
+    if loader then
+        return loader()
+    else
+        return false
+    end
+end
+
+--~ function caches.loaddata(path,name)
+--~     local tmaname, tmcname = caches.setluanames(path,name)
+--~     return dofile(tmcname) or dofile(tmaname)
+--~ end
+
+function caches.iswritable(filepath,filename)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    return file.iswritable(tmaname)
+end
+
+function caches.savedata(filepath,filename,data,raw)
+    local tmaname, tmcname = caches.setluanames(filepath,filename)
+    local reduce, simplify = true, true
+    if raw then
+        reduce, simplify = false, false
+    end
+    if caches.direct then
+        file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex
+    else
+        table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true
+    end
+    local cleanup = resolvers.boolean_variable("PURGECACHE", false)
+    local strip = resolvers.boolean_variable("LUACSTRIP", true)
+    utils.lua.compile(tmaname, tmcname, cleanup, strip)
+end
+
+-- here we use the cache for format loading (texconfig.[formatname|jobname])
+
+--~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then
+if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then
+    if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc
+    texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt")
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-res'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex"))
+
+local upper, lower, gsub = string.upper, string.lower, string.gsub
+
+local prefixes = { }
+
+prefixes.environment = function(str)
+    return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "")
+end
+
+prefixes.relative = function(str,n)
+    if io.exists(str) then
+        -- nothing
+    elseif io.exists("./" .. str) then
+        str = "./" .. str
+    else
+        local p = "../"
+        for i=1,n or 2 do
+            if io.exists(p .. str) then
+                str = p .. str
+                break
+            else
+                p = p .. "../"
+            end
+        end
+    end
+    return resolvers.clean_path(str)
+end
+
+prefixes.locate = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path((fullname ~= "" and fullname) or str)
+end
+
+prefixes.filename = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str))
+end
+
+prefixes.pathname = function(str)
+    local fullname = resolvers.find_given_file(str) or ""
+    return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str))
+end
+
+prefixes.env  = prefixes.environment
+prefixes.rel  = prefixes.relative
+prefixes.loc  = prefixes.locate
+prefixes.kpse = prefixes.locate
+prefixes.full = prefixes.locate
+prefixes.file = prefixes.filename
+prefixes.path = prefixes.pathname
+
+local function _resolve_(method,target)
+    if prefixes[method] then
+        return prefixes[method](target)
+    else
+        return method .. ":" .. target
+    end
+end
+
+local function resolve(str)
+    if type(str) == "table" then
+        for k, v in pairs(str) do -- ipairs
+            str[k] = resolve(v) or v
+        end
+    elseif str and str ~= "" then
+        str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_)
+    end
+    return str
+end
+
+resolvers.resolve = resolve
+
+if os.uname then
+
+    for k, v in pairs(os.uname()) do
+        if not prefixes[k] then
+            prefixes[k] = function() return v end
+        end
+    end
+
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-inp'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+resolvers.finders = resolvers.finders or { }
+resolvers.openers = resolvers.openers or { }
+resolvers.loaders = resolvers.loaders or { }
+
+resolvers.finders.notfound  = { nil }
+resolvers.openers.notfound  = { nil }
+resolvers.loaders.notfound  = { false, nil, 0 }
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-out'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+outputs = outputs or { }
+
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-con'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_cache      = false  trackers.register("resolvers.cache",      function(v) trace_cache      = v end)
+local trace_containers = false  trackers.register("resolvers.containers", function(v) trace_containers = v end)
+local trace_storage    = false  trackers.register("resolvers.storage",    function(v) trace_storage    = v end)
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+--[[ldx--
+
Once we found ourselves defining similar cache constructs
+several times, containers were introduced. Containers are used
+to collect tables in memory and reuse them when possible based
+on (unique) hashes (to be provided by the calling function).

+
+
Caching to disk is disabled by default. Version numbers are
+stored in the saved table which makes it possible to change the
+table structures without bothering about the disk cache.

+
+
Examples of usage can be found in the font related code.

+--ldx]]--
+
+containers = containers or { }
+
+containers.usecache = true
+
+local function report(container,tag,name)
+    if trace_cache or trace_containers then
+        logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid')
+    end
+end
+
+local allocated = { }
+
+-- tracing
+
+function containers.define(category, subcategory, version, enabled)
+    return function()
+        if category and subcategory then
+            local c = allocated[category]
+            if not c then
+                c  = { }
+                allocated[category] = c
+            end
+            local s = c[subcategory]
+            if not s then
+                s = {
+                    category = category,
+                    subcategory = subcategory,
+                    storage = { },
+                    enabled = enabled,
+                    version = version or 1.000,
+                    trace = false,
+                    path = caches and caches.setpath and caches.setpath(category,subcategory),
+                }
+                c[subcategory] = s
+            end
+            return s
+        else
+            return nil
+        end
+    end
+end
+
+function containers.is_usable(container, name)
+    return container.enabled and caches and caches.iswritable(container.path, name)
+end
+
+function containers.is_valid(container, name)
+    if name and name ~= "" then
+        local storage = container.storage[name]
+        return storage and not table.is_empty(storage) and storage.cache_version == container.version
+    else
+        return false
+    end
+end
+
+function containers.read(container,name)
+    if container.enabled and caches and not container.storage[name] and containers.usecache then
+        container.storage[name] = caches.loaddata(container.path,name)
+        if containers.is_valid(container,name) then
+            report(container,"loaded",name)
+        else
+            container.storage[name] = nil
+        end
+    end
+    if container.storage[name] then
+        report(container,"reusing",name)
+    end
+    return container.storage[name]
+end
+
+function containers.write(container, name, data)
+    if data then
+        data.cache_version = container.version
+        if container.enabled and caches then
+            local unique, shared = data.unique, data.shared
+            data.unique, data.shared = nil, nil
+            caches.savedata(container.path, name, data)
+            report(container,"saved",name)
+            data.unique, data.shared = unique, shared
+        end
+        report(container,"stored",name)
+        container.storage[name] = data
+    end
+    return data
+end
+
+function containers.content(container,name)
+    return container.storage[name]
+end
+
+function containers.cleanname(name)
+    return (gsub(lower(name),"[^%w%d]+","-"))
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-use'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, lower, gsub = string.format, string.lower, string.gsub
+
+local trace_verbose    = false  trackers.register("resolvers.verbose",    function(v) trace_verbose    = v end)
+local trace_locating   = false  trackers.register("resolvers.locating",   function(v) trace_locating   = v trackers.enable("resolvers.verbose") end)
+
+-- since we want to use the cache instead of the tree, we will now
+-- reimplement the saver.
+
+local save_data = resolvers.save_data
+local load_data = resolvers.load_data
+
+resolvers.cachepath = nil  -- public, for tracing
+resolvers.usecache  = true -- public, for tracing
+
+function resolvers.save_data(dataname)
+    save_data(dataname, function(cachename,dataname)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(cachename))
+        else
+            return file.join(cachename,dataname)
+        end
+    end)
+end
+
+function resolvers.load_data(pathname,dataname,filename)
+    load_data(pathname,dataname,filename,function(dataname,filename)
+        resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true)
+        if resolvers.usecache then
+            resolvers.cachepath = resolvers.cachepath or caches.definepath("trees")
+            return file.join(resolvers.cachepath(),caches.hashed(pathname))
+        else
+            if not filename or (filename == "") then
+                filename = dataname
+            end
+            return file.join(pathname,filename)
+        end
+    end)
+end
+
+-- we will make a better format, maybe something xml or just text or lua
+
+resolvers.automounted = resolvers.automounted or { }
+
+function resolvers.automount(usecache)
+    local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT'))
+    if table.is_empty(mountpaths) and usecache then
+        mountpaths = { caches.setpath("mount") }
+    end
+    if not table.is_empty(mountpaths) then
+        statistics.starttiming(resolvers.instance)
+        for k, root in pairs(mountpaths) do
+            local f = io.open(root.."/url.tmi")
+            if f then
+                for line in f:lines() do
+                    if line then
+                        if line:find("^[%%#%-]") then -- or %W
+                            -- skip
+                        elseif line:find("^zip://") then
+                            if trace_locating then
+                                logs.report("fileio","mounting %s",line)
+                            end
+                            table.insert(resolvers.automounted,line)
+                            resolvers.usezipfile(line)
+                        end
+                    end
+                end
+                f:close()
+            end
+        end
+        statistics.stoptiming(resolvers.instance)
+    end
+end
+
+-- status info
+
+statistics.register("used config path", function() return caches.configpath()  end)
+statistics.register("used cache path",  function() return caches.temp() or "?" end)
+
+-- experiment (code will move)
+
+function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname
+    local enginebanner = status.list().banner
+    if formatbanner and enginebanner and sourcefile then
+        local luvname = file.replacesuffix(texname,"luv")
+        local luvdata = {
+            enginebanner = enginebanner,
+            formatbanner = formatbanner,
+            sourcehash   = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"),
+            sourcefile   = sourcefile,
+        }
+        io.savedata(luvname,table.serialize(luvdata,true))
+    end
+end
+
+function statistics.check_fmt_status(texname)
+    local enginebanner = status.list().banner
+    if enginebanner and texname then
+        local luvname = file.replacesuffix(texname,"luv")
+        if lfs.isfile(luvname) then
+            local luv = dofile(luvname)
+            if luv and luv.sourcefile then
+                local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown")
+                if luv.enginebanner and luv.enginebanner ~= enginebanner then
+                    return "engine mismatch"
+                end
+                if luv.sourcehash and luv.sourcehash ~= sourcehash then
+                    return "source mismatch"
+                end
+            else
+                return "invalid status file"
+            end
+        else
+            return "missing status file"
+        end
+    end
+    return true
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-zip'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local format, find = string.format, string.find
+
+local trace_locating, trace_verbose  = false, false
+
+trackers.register("resolvers.verbose",  function(v) trace_verbose  = v end)
+trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end)
+
+zip                 = zip or { }
+zip.archives        = zip.archives or { }
+zip.registeredfiles = zip.registeredfiles or { }
+
+local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
+local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators
+
+local archives = zip.archives
+
+-- zip:///oeps.zip?name=bla/bla.tex
+-- zip:///oeps.zip?tree=tex/texmf-local
+
+local function validzip(str) -- todo: use url splitter
+    if not find(str,"^zip://") then
+        return "zip:///" .. str
+    else
+        return str
+    end
+end
+
+function zip.openarchive(name)
+    if not name or name == "" then
+        return nil
+    else
+        local arch = archives[name]
+        if not arch then
+           local full = resolvers.find_file(name) or ""
+           arch = (full ~= "" and zip.open(full)) or false
+           archives[name] = arch
+        end
+       return arch
+    end
+end
+
+function zip.closearchive(name)
+    if not name or (name == "" and archives[name]) then
+        zip.close(archives[name])
+        archives[name] = nil
+    end
+end
+
+-- zip:///texmf.zip?tree=/tex/texmf
+-- zip:///texmf.zip?tree=/tex/texmf-local
+-- zip:///texmf-mine.zip?tree=/tex/texmf-projects
+
+function locators.zip(specification) -- where is this used? startup zips (untested)
+    specification = resolvers.splitmethod(specification)
+    local zipfile = specification.path
+    local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree
+    if trace_locating then
+        if zfile then
+            logs.report("fileio",'! zip locator, found: %s',specification.original)
+        else
+            logs.report("fileio",'? zip locator, not found: %s',specification.original)
+        end
+    end
+end
+
+function hashers.zip(tag,name)
+    if trace_verbose then
+        logs.report("fileio","loading zip file %s as %s",name,tag)
+    end
+    resolvers.usezipfile(format("%s?tree=%s",tag,name))
+end
+
+function concatinators.zip(tag,path,name)
+    if not path or path == "" then
+        return format('%s?name=%s',tag,name)
+    else
+        return format('%s?name=%s/%s',tag,path,name)
+    end
+end
+
+function resolvers.isreadable.zip(name)
+    return true
+end
+
+function finders.zip(specification,filetype)
+    specification = resolvers.splitmethod(specification)
+    if specification.path then
+        local q = url.query(specification.query)
+        if q.name then
+            local zfile = zip.openarchive(specification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'! zip finder, path: %s',specification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    dfile = zfile:close()
+                    if trace_locating then
+                        logs.report("fileio",'+ zip finder, name: %s',q.name)
+                    end
+                    return specification.original
+                end
+            elseif trace_locating then
+                logs.report("fileio",'? zip finder, path %s',specification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip finder, name: %s',filename)
+    end
+    return unpack(finders.notfound)
+end
+
+function openers.zip(specification)
+    local zipspecification = resolvers.splitmethod(specification)
+    if zipspecification.path then
+        local q = url.query(zipspecification.query)
+        if q.name then
+            local zfile = zip.openarchive(zipspecification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'+ zip starter, path: %s',zipspecification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    logs.show_open(specification)
+                    return openers.text_opener(specification,dfile,'zip')
+                end
+            elseif trace_locating then
+                logs.report("fileio",'- zip starter, path %s',zipspecification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip opener, name: %s',filename)
+    end
+    return unpack(openers.notfound)
+end
+
+function loaders.zip(specification)
+    specification = resolvers.splitmethod(specification)
+    if specification.path then
+        local q = url.query(specification.query)
+        if q.name then
+            local zfile = zip.openarchive(specification.path)
+            if zfile then
+                if trace_locating then
+                    logs.report("fileio",'+ zip starter, path: %s',specification.path)
+                end
+                local dfile = zfile:open(q.name)
+                if dfile then
+                    logs.show_load(filename)
+                    if trace_locating then
+                        logs.report("fileio",'+ zip loader, name: %s',filename)
+                    end
+                    local s = dfile:read("*all")
+                    dfile:close()
+                    return true, s, #s
+                end
+            elseif trace_locating then
+                logs.report("fileio",'- zip starter, path: %s',specification.path)
+            end
+        end
+    end
+    if trace_locating then
+        logs.report("fileio",'- zip loader, name: %s',filename)
+    end
+    return unpack(openers.notfound)
+end
+
+-- zip:///somefile.zip
+-- zip:///somefile.zip?tree=texmf-local -> mount
+
+function resolvers.usezipfile(zipname)
+    zipname = validzip(zipname)
+    if trace_locating then
+        logs.report("fileio",'! zip use, file: %s',zipname)
+    end
+    local specification = resolvers.splitmethod(zipname)
+    local zipfile = specification.path
+    if zipfile and not zip.registeredfiles[zipname] then
+        local tree = url.query(specification.query).tree or ""
+        if trace_locating then
+            logs.report("fileio",'! zip register, file: %s',zipname)
+        end
+        local z = zip.openarchive(zipfile)
+        if z then
+            local instance = resolvers.instance
+            if trace_locating then
+                logs.report("fileio","= zipfile, registering: %s",zipname)
+            end
+            statistics.starttiming(instance)
+            resolvers.prepend_hash('zip',zipname,zipfile)
+            resolvers.extend_texmf_var(zipname) -- resets hashes too
+            zip.registeredfiles[zipname] = z
+            instance.files[zipname] = resolvers.register_zip_file(z,tree or "")
+            statistics.stoptiming(instance)
+        elseif trace_locating then
+            logs.report("fileio","? zipfile, unknown: %s",zipname)
+        end
+    elseif trace_locating then
+        logs.report("fileio",'! zip register, no file: %s',zipname)
+    end
+end
+
+function resolvers.register_zip_file(z,tree)
+    local files, filter = { }, ""
+    if tree == "" then
+        filter = "^(.+)/(.-)$"
+    else
+        filter = format("^%s/(.+)/(.-)$",tree)
+    end
+    if trace_locating then
+        logs.report("fileio",'= zip filter: %s',filter)
+    end
+    local register, n = resolvers.register_file, 0
+    for i in z:files() do
+        local path, name = i.filename:match(filter)
+        if path then
+            if name and name ~= '' then
+                register(files, name, path)
+                n = n + 1
+            else
+                -- directory
+            end
+        else
+            register(files, i.filename, '')
+            n = n + 1
+        end
+    end
+    logs.report("fileio",'= zip entries: %s',n)
+    return files
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-crl'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+curl = curl or { }
+
+curl.cached    = { }
+curl.cachepath = caches.definepath("curl")
+
+local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
+
+function curl.fetch(protocol, name)
+    local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-")
+--  cachename = cachename:gsub("[\\/]", io.fileseparator)
+    cachename = cachename:gsub("[\\]", "/") -- cleanup
+    if not curl.cached[name] then
+        if not io.exists(cachename) then
+            curl.cached[name] = cachename
+            local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://"
+            os.spawn(command)
+        end
+        if io.exists(cachename) then
+            curl.cached[name] = cachename
+        else
+            curl.cached[name] = ""
+        end
+    end
+    return curl.cached[name]
+end
+
+function finders.curl(protocol,filename)
+    local foundname = curl.fetch(protocol, filename)
+    return finders.generic(protocol,foundname,filetype)
+end
+
+function openers.curl(protocol,filename)
+    return openers.generic(protocol,filename)
+end
+
+function loaders.curl(protocol,filename)
+    return loaders.generic(protocol,filename)
+end
+
+-- todo: metamethod
+
+function curl.install(protocol)
+    finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end
+    openers[protocol] = function (filename)          return openers.curl(protocol,filename) end
+    loaders[protocol] = function (filename)          return loaders.curl(protocol,filename) end
+end
+
+curl.install('http')
+curl.install('https')
+curl.install('ftp')
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-kps'] = {
+    version   = 1.001,
+    comment   = "companion to luatools.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+--[[ldx--
+
This file is used when we want the input handlers to behave like
+kpsewhich. What to do with the following:

+
+
+{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c}
+$SELFAUTOLOC    : /usr/tex/bin/platform
+$SELFAUTODIR    : /usr/tex/bin
+$SELFAUTOPARENT : /usr/tex
+
+
+
How about just forgetting about them?

+--ldx]]--
+
+local suffixes = resolvers.suffixes
+local formats  = resolvers.formats
+
+suffixes['gf']                       = { 'gf' }
+suffixes['pk']                       = { 'pk' }
+suffixes['base']                     = { 'base' }
+suffixes['bib']                      = { 'bib' }
+suffixes['bst']                      = { 'bst' }
+suffixes['cnf']                      = { 'cnf' }
+suffixes['mem']                      = { 'mem' }
+suffixes['mf']                       = { 'mf' }
+suffixes['mfpool']                   = { 'pool' }
+suffixes['mft']                      = { 'mft' }
+suffixes['mppool']                   = { 'pool' }
+suffixes['graphic/figure']           = { 'eps', 'epsi' }
+suffixes['texpool']                  = { 'pool' }
+suffixes['PostScript header']        = { 'pro' }
+suffixes['ist']                      = { 'ist' }
+suffixes['web']                      = { 'web', 'ch' }
+suffixes['cweb']                     = { 'w', 'web', 'ch' }
+suffixes['cmap files']               = { 'cmap' }
+suffixes['lig files']                = { 'lig' }
+suffixes['bitmap font']              = { }
+suffixes['MetaPost support']         = { }
+suffixes['TeX system documentation'] = { }
+suffixes['TeX system sources']       = { }
+suffixes['dvips config']             = { }
+suffixes['type42 fonts']             = { }
+suffixes['web2c files']              = { }
+suffixes['other text files']         = { }
+suffixes['other binary files']       = { }
+suffixes['opentype fonts']           = { 'otf' }
+
+suffixes['fmt']                      = { 'fmt' }
+suffixes['texmfscripts']             = { 'rb','lua','py','pl' }
+
+suffixes['pdftex config']            = { }
+suffixes['Troff fonts']              = { }
+
+suffixes['ls-R']                     = { }
+
+--[[ldx--
+
If you wondered abou tsome of the previous mappings, how about
+the next bunch:

+--ldx]]--
+
+formats['bib']                      = ''
+formats['bst']                      = ''
+formats['mft']                      = ''
+formats['ist']                      = ''
+formats['web']                      = ''
+formats['cweb']                     = ''
+formats['MetaPost support']         = ''
+formats['TeX system documentation'] = ''
+formats['TeX system sources']       = ''
+formats['Troff fonts']              = ''
+formats['dvips config']             = ''
+formats['graphic/figure']           = ''
+formats['ls-R']                     = ''
+formats['other text files']         = ''
+formats['other binary files']       = ''
+
+formats['gf']                       = ''
+formats['pk']                       = ''
+formats['base']                     = 'MFBASES'
+formats['cnf']                      = ''
+formats['mem']                      = 'MPMEMS'
+formats['mf']                       = 'MFINPUTS'
+formats['mfpool']                   = 'MFPOOL'
+formats['mppool']                   = 'MPPOOL'
+formats['texpool']                  = 'TEXPOOL'
+formats['PostScript header']        = 'TEXPSHEADERS'
+formats['cmap files']               = 'CMAPFONTS'
+formats['type42 fonts']             = 'T42FONTS'
+formats['web2c files']              = 'WEB2C'
+formats['pdftex config']            = 'PDFTEXCONFIG'
+formats['texmfscripts']             = 'TEXMFSCRIPTS'
+formats['bitmap font']              = ''
+formats['lig files']                = 'LIGFONTS'
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-aux'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local find = string.find
+
+local trace_verbose = false  trackers.register("resolvers.verbose", function(v) trace_verbose = v end)
+
+function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix
+    local scriptpath = "scripts/context/lua"
+    newname = file.addsuffix(newname,"lua")
+    local oldscript = resolvers.clean_path(oldname)
+    if trace_verbose then
+        logs.report("fileio","to be replaced old script %s", oldscript)
+    end
+    local newscripts = resolvers.find_files(newname) or { }
+    if #newscripts == 0 then
+        if trace_verbose then
+            logs.report("fileio","unable to locate new script")
+        end
+    else
+        for i=1,#newscripts do
+            local newscript = resolvers.clean_path(newscripts[i])
+            if trace_verbose then
+                logs.report("fileio","checking new script %s", newscript)
+            end
+            if oldscript == newscript then
+                if trace_verbose then
+                    logs.report("fileio","old and new script are the same")
+                end
+            elseif not find(newscript,scriptpath) then
+                if trace_verbose then
+                    logs.report("fileio","new script should come from %s",scriptpath)
+                end
+            elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then
+                if trace_verbose then
+                    logs.report("fileio","invalid new script name")
+                end
+            else
+                local newdata = io.loaddata(newscript)
+                if newdata then
+                    if trace_verbose then
+                        logs.report("fileio","old script content replaced by new content")
+                    end
+                    io.savedata(oldscript,newdata)
+                    break
+                elseif trace_verbose then
+                    logs.report("fileio","unable to load new script")
+                end
+            end
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['data-tmf'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.tex",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- loads *.tmf files in minimal tree roots (to be optimized and documented)
+
+function resolvers.check_environment(tree)
+    logs.simpleline()
+    os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME'))
+    os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform()))
+    os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",''))
+    os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS'))
+    logs.simpleline()
+    logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH'))
+    logs.simple("preset : TEXOS   => %s", os.getenv('TEXOS'))
+    logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS'))
+    logs.simple("preset : TMP     => %s", os.getenv('TMP'))
+    logs.simple('')
+end
+
+function resolvers.load_environment(name) -- todo: key=value as well as lua
+    local f = io.open(name)
+    if f then
+        for line in f:lines() do
+            if line:find("^[%%%#]") then
+                -- skip comment
+            else
+                local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$")
+                if how then
+                    value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end)
+                        if how == "=" or how == "<<" then
+                            os.setenv(key,value)
+                    elseif how == "?" or how == "??" then
+                            os.setenv(key,os.getenv(key) or value)
+                    elseif how == "<" or how == "+=" then
+                        if os.getenv(key) then
+                            os.setenv(key,os.getenv(key) .. io.fileseparator .. value)
+                        else
+                            os.setenv(key,value)
+                        end
+                    elseif how == ">" or how == "=+" then
+                        if os.getenv(key) then
+                            os.setenv(key,value .. io.pathseparator .. os.getenv(key))
+                        else
+                            os.setenv(key,value)
+                        end
+                    end
+                end
+            end
+        end
+        f:close()
+    end
+end
+
+function resolvers.load_tree(tree)
+    if tree and tree ~= "" then
+        local setuptex = 'setuptex.tmf'
+        if lfs.attributes(tree, "mode") == "directory" then -- check if not nil
+            setuptex = tree .. "/" .. setuptex
+        else
+            setuptex = tree
+        end
+        if io.exists(setuptex) then
+            resolvers.check_environment(tree)
+            resolvers.load_environment(setuptex)
+        end
+    end
+end
+
+
+end -- of closure
+
+do -- create closure to overcome 200 locals limit
+
+if not modules then modules = { } end modules ['luat-sta'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- this code is used in the updater
+
+states          = states          or { }
+states.data     = states.data     or { }
+states.hash     = states.hash     or { }
+states.tag      = states.tag      or ""
+states.filename = states.filename or ""
+
+function states.save(filename,tag)
+    tag = tag or states.tag
+    filename = file.addsuffix(filename or states.filename,'lus')
+    io.savedata(filename,
+        "-- generator : luat-sta.lua\n" ..
+        "-- state tag : " .. tag .. "\n\n" ..
+        table.serialize(states.data[tag or states.tag] or {},true)
+    )
+end
+
+function states.load(filename,tag)
+    states.filename = filename
+    states.tag = tag or "whatever"
+    states.filename = file.addsuffix(states.filename,'lus')
+    states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { }
+end
+
+function states.set_by_tag(tag,key,value,default,persistent)
+    local d, h = states.data[tag], states.hash[tag]
+    if d then
+        if type(d) == "table" then
+            local dkey, hkey = key, key
+            local pre, post = key:match("(.+)%.([^%.]+)$")
+            if pre and post then
+                for k in pre:gmatch("[^%.]+") do
+                    local dk = d[k]
+                    if not dk then
+                        dk = { }
+                        d[k] = dk
+                    end
+                    d = dk
+                end
+                dkey, hkey = post, key
+            end
+            if type(value) == nil then
+                value = value or default
+            elseif persistent then
+                value = value or d[dkey] or default
+            else
+                value = value or default
+            end
+            d[dkey], h[hkey] = value, value
+        elseif type(d) == "string" then
+            -- weird
+            states.data[tag], states.hash[tag] = value, value
+        end
+    end
+end
+
+function states.get_by_tag(tag,key,default)
+    local h = states.hash[tag]
+    if h and h[key] then
+        return h[key]
+    else
+        local d = states.data[tag]
+        if d then
+            for k in key:gmatch("[^%.]+") do
+                local dk = d[k]
+                if dk then
+                    d = dk
+                else
+                    return default
+                end
+            end
+            return d or default
+        end
+    end
+end
+
+function states.set(key,value,default,persistent)
+    states.set_by_tag(states.tag,key,value,default,persistent)
+end
+
+function states.get(key,default)
+    return states.get_by_tag(states.tag,key,default)
+end
+
+--~ states.data.update = {
+--~ 	["version"] = {
+--~ 		["major"] = 0,
+--~ 		["minor"] = 1,
+--~ 	},
+--~ 	["rsync"] = {
+--~ 		["server"]     = "contextgarden.net",
+--~ 		["module"]     = "minimals",
+--~ 		["repository"] = "current",
+--~ 		["flags"]      = "-rpztlv --stats",
+--~ 	},
+--~ 	["tasks"] = {
+--~ 		["update"] = true,
+--~ 		["make"]   = true,
+--~         ["delete"] = false,
+--~ 	},
+--~ 	["platform"] = {
+--~ 		["host"]  = true,
+--~ 		["other"] = {
+--~ 			["mswin"]     = false,
+--~ 			["linux"]     = false,
+--~ 			["linux-64"]  = false,
+--~ 			["osx-intel"] = false,
+--~ 			["osx-ppc"]   = false,
+--~ 			["sun"]       = false,
+--~ 		},
+--~ 	},
+--~ 	["context"] = {
+--~ 		["available"] = {"current", "beta", "alpha", "experimental"},
+--~ 		["selected"]  = "current",
+--~ 	},
+--~ 	["formats"] = {
+--~ 		["cont-en"] = true,
+--~ 		["cont-nl"] = true,
+--~ 		["cont-de"] = false,
+--~ 		["cont-cz"] = false,
+--~ 		["cont-fr"] = false,
+--~ 		["cont-ro"] = false,
+--~ 	},
+--~ 	["engine"] = {
+--~ 		["pdftex"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 				["pdftex"] = true,
+--~ 			},
+--~ 		},
+--~ 		["luatex"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 			},
+--~ 		},
+--~ 		["xetex"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 				["xetex"] = false,
+--~ 			},
+--~ 		},
+--~ 		["metapost"] = {
+--~ 			["install"] = true,
+--~ 			["formats"] = {
+--~ 				["mpost"] = true,
+--~ 				["metafun"] = true,
+--~ 			},
+--~ 		},
+--~ 	},
+--~ 	["fonts"] = {
+--~ 	},
+--~ 	["doc"] = {
+--~ 	},
+--~ 	["modules"] = {
+--~ 		["f-urwgaramond"] = false,
+--~ 		["f-urwgothic"] = false,
+--~ 		["t-bnf"] = false,
+--~ 		["t-chromato"] = false,
+--~ 		["t-cmscbf"] = false,
+--~ 		["t-cmttbf"] = false,
+--~ 		["t-construction-plan"] = false,
+--~ 		["t-degrade"] = false,
+--~ 		["t-french"] = false,
+--~ 		["t-lettrine"] = false,
+--~ 		["t-lilypond"] = false,
+--~ 		["t-mathsets"] = false,
+--~ 		["t-tikz"] = false,
+--~ 		["t-typearea"] = false,
+--~ 		["t-vim"] = false,
+--~ 	},
+--~ }
+
+--~ states.save("teststate", "update")
+--~ states.load("teststate", "update")
+
+--~ print(states.get_by_tag("update","rsync.server","unknown"))
+--~ states.set_by_tag("update","rsync.server","oeps")
+--~ print(states.get_by_tag("update","rsync.server","unknown"))
+--~ states.save("teststate", "update")
+--~ states.load("teststate", "update")
+--~ print(states.get_by_tag("update","rsync.server","unknown"))
+
+
+end -- of closure
+-- end library merge
+
+own = { } -- not local
+
+own.libs = { -- todo: check which ones are really needed
+    'l-string.lua',
+    'l-lpeg.lua',
+    'l-table.lua',
+    'l-io.lua',
+    'l-number.lua',
+    'l-set.lua',
+    'l-os.lua',
+    'l-file.lua',
+    'l-md5.lua',
+    'l-dir.lua',
+    'l-boolean.lua',
+    'l-math.lua',
+--  'l-unicode.lua',
+--  'l-tex.lua',
+    'l-utils.lua',
+--  'l-xml.lua',
+    'lxml-tab.lua',
+    'lxml-pth.lua',
+    'lxml-ent.lua',
+    'lxml-mis.lua',
+    'trac-tra.lua',
+    'luat-env.lua',
+    'trac-inf.lua',
+    'trac-log.lua',
+    'data-res.lua',
+    'data-tmp.lua',
+    'data-pre.lua',
+    'data-inp.lua',
+    'data-out.lua',
+    'data-con.lua',
+    'data-use.lua',
+--  'data-tex.lua',
+--  'data-bin.lua',
+    'data-zip.lua',
+    'data-crl.lua',
+--  'data-lua.lua',
+    'data-kps.lua', -- so that we can replace kpsewhich
+    'data-aux.lua', -- updater
+    'data-tmf.lua', -- tree files
+    -- needed ?
+    'luat-sta.lua', -- states
+}
+
+-- We need this hack till luatex is fixed.
+--
+-- for k,v in pairs(arg) do print(k,v) end
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
+-- End of hack.
+
+own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua'
+
+own.path = string.match(own.name,"^(.+)[\\/].-$") or "."
+own.list = { '.' }
+if own.path ~= '.' then
+    table.insert(own.list,own.path)
+end
+table.insert(own.list,own.path.."/../../../tex/context/base")
+table.insert(own.list,own.path.."/mtx")
+table.insert(own.list,own.path.."/../sources")
+
+local function locate_libs()
+    for _, lib in pairs(own.libs) do
+        for _, pth in pairs(own.list) do
+            local filename = string.gsub(pth .. "/" .. lib,"\\","/")
+            local codeblob = loadfile(filename)
+            if codeblob then
+                codeblob()
+                own.list = { pth } -- speed up te search
+                break
+            end
+        end
+    end
+end
+
+if not resolvers then
+    locate_libs()
+end
+
+if not resolvers then
+    print("")
+    print("Mtxrun is unable to start up due to lack of libraries. You may")
+    print("try to run 'lua mtxrun.lua --selfmerge' in the path where this")
+    print("script is located (normally under ..../scripts/context/lua) which")
+    print("will make this script library independent.")
+    os.exit()
+end
+
+logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false)
+
+local instance = resolvers.reset()
+
+runners  = runners  or { } -- global
+messages = messages or { }
+
+messages.help = [[
+--script              run an mtx script (--noquotes)
+--execute             run a script or program (--noquotes)
+--resolve             resolve prefixed arguments
+--ctxlua              run internally (using preloaded libs)
+--locate              locate given filename
+
+--autotree            use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree'
+--tree=pathtotree     use given texmf tree (default file: 'setuptex.tmf')
+--environment=name    use given (tmf) environment file
+--path=runpath        go to given path before execution
+--ifchanged=filename  only execute when given file has changed (md checksum)
+--iftouched=old,new   only execute when given file has changed (time stamp)
+
+--make                create stubs for (context related) scripts
+--remove              remove stubs (context related) scripts
+--stubpath=binpath    paths where stubs wil be written
+--windows             create windows (mswin) stubs
+--unix                create unix (linux) stubs
+
+--verbose             give a bit more info
+--engine=str          target engine
+--progname=str        format or backend
+
+--edit                launch editor with found file
+--launch (--all)      launch files like manuals, assumes os support
+
+--intern              run script using built in libraries
+
+--usekpse            use kpse as fallback (when no mkiv and cache installed, often slower)
+--forcekpse          force using kpse (handy when no mkiv and cache installed but less functionality)
+]]
+
+runners.applications = {
+    ["lua"] = "luatex --luaonly",
+    ["luc"] = "luatex --luaonly",
+    ["pl"] = "perl",
+    ["py"] = "python",
+    ["rb"] = "ruby",
+}
+
+runners.suffixes = {
+    'rb', 'lua', 'py', 'pl'
+}
+
+runners.registered = {
+    texexec      = { 'texexec.rb',      true  },  -- context mkii runner (only tool not to be luafied)
+    texutil      = { 'texutil.rb',      true  },  -- old perl based index sorter for mkii (old versions need it)
+    texfont      = { 'texfont.pl',      true  },  -- perl script that makes mkii font metric files
+    texfind      = { 'texfind.pl',      false },  -- perltk based tex searching tool, mostly used at pragma
+    texshow      = { 'texshow.pl',      false },  -- perltk based context help system, will be luafied
+ -- texwork      = { \texwork.pl',      false },  -- perltk based editing environment, only used at pragma
+
+    makempy      = { 'makempy.pl',      true  },
+    mptopdf      = { 'mptopdf.pl',      true  },
+    pstopdf      = { 'pstopdf.rb',      true  },  -- converts ps (and some more) images, does some cleaning (replaced)
+
+--  examplex     = { 'examplex.rb',     false },
+    concheck     = { 'concheck.rb',     false },
+
+    runtools     = { 'runtools.rb',     true  },
+    textools     = { 'textools.rb',     true  },
+    tmftools     = { 'tmftools.rb',     true  },
+    ctxtools     = { 'ctxtools.rb',     true  },
+    rlxtools     = { 'rlxtools.rb',     true  },
+    pdftools     = { 'pdftools.rb',     true  },
+    mpstools     = { 'mpstools.rb',     true  },
+--  exatools     = { 'exatools.rb',     true  },
+    xmltools     = { 'xmltools.rb',     true  },
+--  luatools     = { 'luatools.lua',    true  },
+    mtxtools     = { 'mtxtools.rb',     true  },
+
+    pdftrimwhite = { 'pdftrimwhite.pl', false }
+}
+
+runners.launchers = {
+    windows = { },
+    unix = { }
+}
+
+function runners.prepare()
+    local checkname = environment.argument("ifchanged")
+    if checkname and checkname ~= "" then
+        local oldchecksum = file.loadchecksum(checkname)
+        local newchecksum = file.checksum(checkname)
+        if oldchecksum == newchecksum then
+            logs.simple("file '%s' is unchanged",checkname)
+            return "skip"
+        else
+            logs.simple("file '%s' is changed, processing started",checkname)
+        end
+        file.savechecksum(checkname)
+    end
+    local oldname, newname = string.split(environment.argument("iftouched") or "", ",")
+    if oldname and newname and oldname ~= "" and newname ~= "" then
+        if not file.needs_updating(oldname,newname) then
+            logs.simple("file '%s' and '%s' have same age",oldname,newname)
+            return "skip"
+        else
+            logs.simple("file '%s' is older than '%s'",oldname,newname)
+        end
+    end
+    local tree = environment.argument('tree') or ""
+    if environment.argument('autotree') then
+        tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree
+    end
+    if tree and tree ~= "" then
+        resolvers.load_tree(tree)
+    end
+    local env = environment.argument('environment') or ""
+    if env and env ~= "" then
+        for _,e in pairs(string.split(env)) do
+            -- maybe force suffix when not given
+            resolvers.load_tree(e)
+        end
+    end
+    local runpath = environment.argument("path")
+    if runpath and not lfs.chdir(runpath) then
+        logs.simple("unable to change to path '%s'",runpath)
+        return "error"
+    end
+    return "run"
+end
+
+function runners.execute_script(fullname,internal)
+    local noquote = environment.argument("noquotes")
+    if fullname and fullname ~= "" then
+        local state = runners.prepare()
+        if state == 'error' then
+            return false
+        elseif state == 'skip' then
+            return true
+        elseif state == "run" then
+            instance.progname = environment.argument("progname") or instance.progname
+            instance.format   = environment.argument("format")   or instance.format
+            local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), ""
+            if path ~= "" then
+                result = fullname
+            elseif name then
+                name = name:gsub("^int[%a]*:",function()
+                    internal = true
+                    return ""
+                end )
+                name = name:gsub("^script:","")
+                if suffix == "" and runners.registered[name] and runners.registered[name][1] then
+                    name = runners.registered[name][1]
+                    suffix = file.extname(name)
+                end
+                if suffix == "" then
+                    -- loop over known suffixes
+                    for _,s in pairs(runners.suffixes) do
+                        result = resolvers.find_file(name .. "." .. s, 'texmfscripts')
+                        if result ~= "" then
+                            break
+                        end
+                    end
+                elseif runners.applications[suffix] then
+                    result = resolvers.find_file(name, 'texmfscripts')
+                else
+                    -- maybe look on path
+                    result = resolvers.find_file(name, 'other text files')
+                end
+            end
+            if result and result ~= "" then
+                local before, after = environment.split_arguments(fullname) -- already done
+                environment.arguments_before, environment.arguments_after = before, after
+                if internal then
+                    arg = { } for _,v in pairs(after) do arg[#arg+1] = v end
+                    dofile(result)
+                else
+                    local binary = runners.applications[file.extname(result)]
+                    if binary and binary ~= "" then
+                        result = binary .. " " .. result
+                    end
+                    local command = result .. " " .. environment.reconstruct_commandline(after,noquote)
+                    if logs.verbose then
+                        logs.simpleline()
+                        logs.simple("executing: %s",command)
+                        logs.simpleline()
+                        logs.simpleline()
+                        io.flush()
+                    end
+                    local code = os.exec(command) -- maybe spawn
+                    return code == 0
+                end
+            end
+        end
+    end
+    return false
+end
+
+function runners.execute_program(fullname)
+    local noquote = environment.argument("noquotes")
+    if fullname and fullname ~= "" then
+        local state = runners.prepare()
+        if state == 'error' then
+            return false
+        elseif state == 'skip' then
+            return true
+        elseif state == "run" then
+            local before, after = environment.split_arguments(fullname)
+            environment.initialize_arguments(after)
+            fullname = fullname:gsub("^bin:","")
+            local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "")
+            logs.simpleline()
+            logs.simple("executing: %s",command)
+            logs.simpleline()
+            logs.simpleline()
+            io.flush()
+            local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn
+            return code == 0
+        end
+    end
+    return false
+end
+
+-- the --usekpse flag will fallback on kpse
+
+local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010'
+local unix_stub    = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010'
+
+function runners.handle_stubs(create)
+    local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported
+    local windows  = environment.argument('windows') or environment.argument('mswin') or false
+    local unix     = environment.argument('unix') or environment.argument('linux') or false
+    if not windows and not unix then
+        if os.platform == "unix" then
+            unix = true
+        else
+            windows = true
+        end
+    end
+    for _,v in pairs(runners.registered) do
+        local name, doit = v[1], v[2]
+        if doit then
+            local base = string.gsub(file.basename(name), "%.(.-)$", "")
+            if create then
+                if windows then
+                    io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name))
+                    logs.simple("windows stub for '%s' created",base)
+                end
+                if unix then
+                    io.savedata(file.join(stubpath,base),string.format(unix_stub,name))
+                    logs.simple("unix stub for '%s' created",base)
+                end
+            else
+                if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then
+                    logs.simple("windows stub for '%s' removed", base)
+                end
+                if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then
+                    logs.simple("unix stub for '%s' removed",base)
+                end
+            end
+        end
+    end
+end
+
+function runners.resolve_string(filename)
+    if filename and filename ~= "" then
+        runners.report_location(resolvers.resolve(filename))
+    end
+end
+
+function runners.locate_file(filename)
+    -- differs from texmfstart where locate appends .com .exe .bat ... todo
+    if filename and filename ~= "" then
+        runners.report_location(resolvers.find_given_file(filename))
+    end
+end
+
+function runners.locate_platform()
+    runners.report_location(os.currentplatform())
+end
+
+function runners.report_location(result)
+    if logs.verbose then
+        logs.simpleline()
+        if result and result ~= "" then
+            logs.simple(result)
+        else
+            logs.simple("not found")
+        end
+    else
+        io.write(result)
+    end
+end
+
+function runners.edit_script(filename) -- we assume that vim is present on most systems
+    local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim'
+    local rest = resolvers.resolve(filename)
+    if rest ~= "" then
+        local command = editor .. " " .. rest
+        if logs.verbose then
+            logs.simpleline()
+            logs.simple("starting editor: %s",command)
+            logs.simple_line()
+            logs.simple_line()
+        end
+        os.launch(command)
+    end
+end
+
+function runners.save_script_session(filename, list)
+    local t = { }
+    for _, key in ipairs(list) do
+        t[key] = environment.arguments[key]
+    end
+    io.savedata(filename,table.serialize(t,true))
+end
+
+function runners.load_script_session(filename)
+    if lfs.isfile(filename) then
+        local t = io.loaddata(filename)
+        if t then
+            t = loadstring(t)
+            if t then t = t() end
+            for key, value in pairs(t) do
+                environment.arguments[key] = value
+            end
+        end
+    end
+end
+
+function resolvers.launch(str)
+    -- maybe we also need to test on mtxrun.launcher.suffix environment
+    -- variable or on windows consult the assoc and ftype vars and such
+    local launchers = runners.launchers[os.platform] if launchers then
+        local suffix = file.extname(str) if suffix then
+            local runner = launchers[suffix] if runner then
+                str = runner .. " " .. str
+            end
+        end
+    end
+    os.launch(str)
+end
+
+function runners.launch_file(filename)
+    instance.allresults = true
+    logs.setverbose(true)
+    local pattern = environment.arguments["pattern"]
+    if not pattern or pattern == "" then
+        pattern = filename
+    end
+    if not pattern or pattern == "" then
+        logs.simple("provide name or --pattern=")
+    else
+        local t = resolvers.find_files(pattern)
+        if not t or #t == 0 then
+            t = resolvers.find_files("*/" .. pattern)
+        end
+        if not t or #t == 0 then
+            t = resolvers.find_files("*/" .. pattern .. "*")
+        end
+        if t and #t > 0 then
+            if environment.arguments["all"] then
+                for _, v in pairs(t) do
+                    logs.simple("launching %s", v)
+                    resolvers.launch(v)
+                end
+            else
+                logs.simple("launching %s", t[1])
+                resolvers.launch(t[1])
+            end
+        else
+            logs.simple("no match for %s", pattern)
+        end
+    end
+end
+
+function runners.find_mtx_script(filename)
+    local function found(name)
+        local path = file.dirname(name)
+        if path and path ~= "" then
+            return false
+        else
+            local fullname = own and own.path and file.join(own.path,name)
+            return io.exists(fullname) and fullname
+        end
+    end
+    filename = file.addsuffix(filename,"lua")
+    local basename = file.removesuffix(file.basename(filename))
+    local suffix = file.extname(filename)
+    -- qualified path, raw name
+    local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- current path, raw name
+    fullname = "./" .. filename
+    fullname = io.exists(fullname) and fullname
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, mtx-
+    fullname = "mtx-" .. filename
+    fullname = found(fullname) or resolvers.find_file(fullname)
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, mtx-s
+    fullname = "mtx-" .. basename .. "s" .. "." .. suffix
+    fullname = found(fullname) or resolvers.find_file(fullname)
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, mtx-
+    fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix
+    fullname = found(fullname) or resolvers.find_file(fullname)
+    if fullname and fullname ~= "" then
+        return fullname
+    end
+    -- context namespace, just 
+    fullname = resolvers.find_file(filename)
+    return fullname
+end
+
+function runners.execute_ctx_script(filename,arguments)
+    local fullname = runners.find_mtx_script(filename) or ""
+    -- retyr after generate but only if --autogenerate
+    if fullname == "" and environment.argument("autogenerate") then -- might become the default
+        instance.renewcache = true
+        logs.setverbose(true)
+        resolvers.load()
+        --
+        fullname = runners.find_mtx_script(filename) or ""
+    end
+    -- that should do it
+    if fullname ~= "" then
+        local state = runners.prepare()
+        if state == 'error' then
+            return false
+        elseif state == 'skip' then
+            return true
+        elseif state == "run" then
+            -- load and save ... kind of undocumented
+            arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end
+            environment.initialize_arguments(arg)
+            local loadname = environment.arguments['load']
+            if loadname then
+                if type(loadname) ~= "string" then loadname = file.basename(fullname) end
+                loadname = file.replacesuffix(loadname,"cfg")
+                runners.load_script_session(loadname)
+            end
+            filename = environment.files[1]
+            if logs.verbose then
+                logs.simple("using script: %s\n",fullname)
+            end
+            dofile(fullname)
+            local savename = environment.arguments['save']
+            if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then
+                if type(savename) ~= "string" then savename = file.basename(fullname) end
+                savename = file.replacesuffix(savename,"cfg")
+                runners.save_script_session(savename, runners.save_list)
+            end
+            return true
+        end
+    else
+        logs.setverbose(true)
+        filename = file.addsuffix(filename,"lua")
+        if filename == "" then
+            logs.simple("unknown script, no name given")
+        elseif file.is_qualified_path(filename) then
+            logs.simple("unknown script '%s'",filename)
+        else
+            logs.simple("unknown script '%s' or 'mtx-%s'",filename,filename)
+        end
+        return false
+    end
+end
+
+function runners.timed(action)
+    statistics.timed(action)
+end
+
+-- this is a bit dirty ... first we store the first filename and next we
+-- split the arguments so that we only see the ones meant for this script
+-- ... later we will use the second half
+
+local filename = environment.files[1] or ""
+local ok      = true
+
+local before, after = environment.split_arguments(filename)
+environment.arguments_before, environment.arguments_after = before, after
+environment.initialize_arguments(before)
+
+instance.engine   = environment.argument("engine")   or 'luatex'
+instance.progname = environment.argument("progname") or 'context'
+instance.lsrmode  = environment.argument("lsr")      or false
+
+-- maybe the unset has to go to this level
+
+if environment.argument("usekpse") or environment.argument("forcekpse") then
+
+    os.setenv("engine","")
+    os.setenv("progname","")
+
+    local remapper = {
+        otf = "opentype fonts",
+        ttf = "truetype fonts",
+        ttc = "truetype fonts",
+        pfb = "type1 fonts",
+        other = "other text files",
+    }
+
+    local function kpse_initialized()
+        texconfig.kpse_init = true
+        local t = os.clock()
+        local k = kpse.original.new("luatex",instance.progname)
+        local dummy = k:find_file("mtxrun.lua") -- so that we're initialized
+        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t)
+        kpse_initialized = function() return k end
+        return k
+    end
+
+    local find_file = resolvers.find_file
+    local show_path = resolvers.show_path
+
+    if environment.argument("forcekpse") then
+
+        function resolvers.find_file(name,kind)
+            return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or ""
+        end
+        function resolvers.show_path(name)
+            return (kpse_initialized():show_path(name)) or ""
+        end
+
+    elseif environment.argument("usekpse") then
+
+        resolvers.load()
+
+        function resolvers.find_file(name,kind)
+            local found = find_file(name,kind) or ""
+            if found ~= "" then
+                return found
+            else
+                return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or ""
+            end
+        end
+        function resolvers.show_path(name)
+            local found = show_path(name) or ""
+            if found ~= "" then
+                return found
+            else
+                return (kpse_initialized():show_path(name)) or ""
+            end
+        end
+
+    end
+
+else
+
+    resolvers.load()
+
+end
+
+
+if environment.argument("selfmerge") then
+    -- embed used libraries
+    utils.merger.selfmerge(own.name,own.libs,own.list)
+elseif environment.argument("selfclean") then
+    -- remove embedded libraries
+    utils.merger.selfclean(own.name)
+elseif environment.argument("selfupdate") then
+    logs.setverbose(true)
+    resolvers.update_script(own.name,"mtxrun")
+elseif environment.argument("ctxlua") or environment.argument("internal") then
+    -- run a script by loading it (using libs)
+    ok = runners.execute_script(filename,true)
+elseif environment.argument("script") or environment.argument("s") then
+    -- run a script by loading it (using libs), pass args
+    ok = runners.execute_ctx_script(filename,after)
+elseif environment.argument("execute") then
+    -- execute script
+    ok = runners.execute_script(filename)
+elseif environment.argument("direct") then
+    -- equals bin:
+    ok = runners.execute_program(filename)
+elseif environment.argument("edit") then
+    -- edit file
+    runners.edit_script(filename)
+elseif environment.argument("launch") then
+    runners.launch_file(filename)
+elseif environment.argument("make") then
+    -- make stubs
+    runners.handle_stubs(true)
+elseif environment.argument("remove") then
+    -- remove stub
+    runners.handle_stubs(false)
+elseif environment.argument("resolve") then
+    -- resolve string
+    runners.resolve_string(filename)
+elseif environment.argument("locate") then
+    -- locate file
+    runners.locate_file(filename)
+elseif environment.argument("platform")then
+    -- locate platform
+    runners.locate_platform()
+elseif environment.argument("help") or filename=='help' or filename == "" then
+    logs.help(messages.help)
+    -- execute script
+elseif filename:find("^bin:") then
+    ok = runners.execute_program(filename)
+else
+    ok = runners.execute_script(filename)
+end
+
+if os.platform == "unix" then
+    io.write("\n")
+end
+
+if ok == false then ok = 1 elseif ok == true then ok = 0 end
+
+os.exit(ok)
diff --git a/scripts/context/stubs/unix/mtxtools b/scripts/context/stubs/unix/mtxtools
new file mode 100755
index 000000000..3803c1c6f
--- /dev/null
+++ b/scripts/context/stubs/unix/mtxtools
@@ -0,0 +1,2 @@
+#!/bin/sh
+mtxrun --usekpse --execute mtxtools.rb "$@"
diff --git a/scripts/context/stubs/unix/pdftools b/scripts/context/stubs/unix/pdftools
index 92ee803a8..da7bd64cf 100755
--- a/scripts/context/stubs/unix/pdftools
+++ b/scripts/context/stubs/unix/pdftools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart pdftools.rb "$@"
+mtxrun --usekpse --execute pdftools.rb "$@"
diff --git a/scripts/context/stubs/unix/pdftrimwhite b/scripts/context/stubs/unix/pdftrimwhite
deleted file mode 100755
index 00b5f525a..000000000
--- a/scripts/context/stubs/unix/pdftrimwhite
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-texmfstart pdftrimwhite.pl "$@"
diff --git a/scripts/context/stubs/unix/pstopdf b/scripts/context/stubs/unix/pstopdf
index 5b38ed426..059812cce 100755
--- a/scripts/context/stubs/unix/pstopdf
+++ b/scripts/context/stubs/unix/pstopdf
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart pstopdf.rb "$@"
+mtxrun --usekpse --execute pstopdf.rb "$@"
diff --git a/scripts/context/stubs/unix/rlxtools b/scripts/context/stubs/unix/rlxtools
index 41cea40fc..d01987b3c 100755
--- a/scripts/context/stubs/unix/rlxtools
+++ b/scripts/context/stubs/unix/rlxtools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart rlxtools.rb "$@"
+mtxrun --usekpse --execute rlxtools.rb "$@"
diff --git a/scripts/context/stubs/unix/runtools b/scripts/context/stubs/unix/runtools
index ff9a33379..e21c1a244 100755
--- a/scripts/context/stubs/unix/runtools
+++ b/scripts/context/stubs/unix/runtools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart runtools.rb "$@"
+mtxrun --usekpse --execute runtools.rb "$@"
diff --git a/scripts/context/stubs/unix/texexec b/scripts/context/stubs/unix/texexec
index 215817290..083e500c6 100755
--- a/scripts/context/stubs/unix/texexec
+++ b/scripts/context/stubs/unix/texexec
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart texexec.rb "$@"
+mtxrun --usekpse --execute texexec.rb "$@"
diff --git a/scripts/context/stubs/unix/texfind b/scripts/context/stubs/unix/texfind
deleted file mode 100755
index c054bdf52..000000000
--- a/scripts/context/stubs/unix/texfind
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-texmfstart texfind "$@"
diff --git a/scripts/context/stubs/unix/texfont b/scripts/context/stubs/unix/texfont
index a91f786e3..bc811a640 100755
--- a/scripts/context/stubs/unix/texfont
+++ b/scripts/context/stubs/unix/texfont
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart texfont.pl "$@"
+mtxrun --usekpse --execute texfont.pl "$@"
diff --git a/scripts/context/stubs/unix/texmfstart b/scripts/context/stubs/unix/texmfstart
new file mode 100755
index 000000000..1799b3579
--- /dev/null
+++ b/scripts/context/stubs/unix/texmfstart
@@ -0,0 +1,2 @@
+#!/bin/sh
+mtxrun --usekpse "$@"
diff --git a/scripts/context/stubs/unix/texshow b/scripts/context/stubs/unix/texshow
deleted file mode 100755
index afd62c339..000000000
--- a/scripts/context/stubs/unix/texshow
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-texmfstart texshow.pl "$@"
diff --git a/scripts/context/stubs/unix/textools b/scripts/context/stubs/unix/textools
index 7445eac37..76087ca57 100755
--- a/scripts/context/stubs/unix/textools
+++ b/scripts/context/stubs/unix/textools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart textools.rb "$@"
+mtxrun --usekpse --execute textools.rb "$@"
diff --git a/scripts/context/stubs/unix/texutil b/scripts/context/stubs/unix/texutil
index 607154af0..f5d9b6f1d 100755
--- a/scripts/context/stubs/unix/texutil
+++ b/scripts/context/stubs/unix/texutil
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart texutil.rb "$@"
+mtxrun --usekpse --execute texutil.rb "$@"
diff --git a/scripts/context/stubs/unix/tmftools b/scripts/context/stubs/unix/tmftools
index 7531a9663..48d32f0fd 100755
--- a/scripts/context/stubs/unix/tmftools
+++ b/scripts/context/stubs/unix/tmftools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart tmftools.rb "$@"
+mtxrun --usekpse --execute tmftools.rb "$@"
diff --git a/scripts/context/stubs/unix/xmltools b/scripts/context/stubs/unix/xmltools
index 03086d043..a673d1e7a 100755
--- a/scripts/context/stubs/unix/xmltools
+++ b/scripts/context/stubs/unix/xmltools
@@ -1,2 +1,2 @@
 #!/bin/sh
-texmfstart xmltools.rb "$@"
+mtxrun --usekpse --execute xmltools.rb "$@"
-- 
cgit v1.2.3