From 1d63a6eae86a6b78d4563ed60521449e4bf89f3c Mon Sep 17 00:00:00 2001
From: Hans Hagen <pragma@wxs.nl>
Date: Wed, 21 May 2008 15:21:00 +0200
Subject: stable 2008.05.21 15:21

---
 scripts/context/lua/luatools.lua    | 845 ++++++++++++++++++---------------
 scripts/context/lua/mtx-context.lua |   3 +-
 scripts/context/lua/mtx-server.lua  |  83 ++++
 scripts/context/lua/mtxrun.lua      | 920 ++++++++++++++++++++----------------
 4 files changed, 1046 insertions(+), 805 deletions(-)
 create mode 100644 scripts/context/lua/mtx-server.lua

(limited to 'scripts')

diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua
index b092c5050..89e5e0eb4 100644
--- a/scripts/context/lua/luatools.lua
+++ b/scripts/context/lua/luatools.lua
@@ -15,7 +15,7 @@
 -- the future. As long as Luatex is under development the
 -- interfaces and names of functions may change.
 
-banner = "version 1.1.1 - 2006+ - PRAGMA ADE / CONTEXT"
+banner = "version 1.2.0 - 2006+ - PRAGMA ADE / CONTEXT"
 texlua = true
 
 -- For the sake of independence we optionally can merge the library
@@ -374,7 +374,6 @@ local patterns_escapes = {
     ["]"] = "%]",
 }
 
-
 function string:pattesc()
     return (self:gsub(".",patterns_escapes))
 end
@@ -1687,14 +1686,22 @@ end
 --~ print("../test/"       .. " == " .. file.collapse_path("../test/"))
 --~ print("a/a"            .. " == " .. file.collapse_path("a/b/c/../../a"))
 
+--~ function file.collapse_path(str)
+--~     local ok, n = false, 0
+--~     while not ok do
+--~         ok = true
+--~         str, n = str:gsub("[^%./]+/%.%./", function(s)
+--~             ok = false
+--~             return ""
+--~         end)
+--~     end
+--~     return (str:gsub("/%./","/"))
+--~ end
+
 function file.collapse_path(str)
-    local ok, n = false, 0
-    while not ok do
-        ok = true
-        str, n = str:gsub("[^%./]+/%.%./", function(s)
-            ok = false
-            return ""
-        end)
+    local n = 1
+    while n > 0 do
+        str, n = str:gsub("([^/%.]+/%.%./)","")
     end
     return (str:gsub("/%./","/"))
 end
@@ -1762,7 +1769,7 @@ function url.hashed(str)
         path = s[3],
         query = s[4],
         fragment = s[5],
-        original=str
+        original = str
     }
 end
 
@@ -2560,14 +2567,32 @@ os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows")
 --
 -- for k,v in pairs(arg) do print(k,v) end
 
-if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
-    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
-end
-
 -- environment
 
 if not environment then environment = { } end
 
+environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex"
+
+local ownpath = nil -- we could use a metatable here
+
+function environment.ownpath()
+    if not ownpath then
+        for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+            local b = file.join(p,environment.ownbin)
+            if lfs.isfile(b..".exe") or lfs.isfile(b) then
+                ownpath = p
+                break
+            end
+        end
+        if not ownpath then ownpath = '.' end
+    end
+    return ownpath
+end
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
 environment.arguments            = { }
 environment.files                = { }
 environment.sorted_argument_keys = nil
@@ -2679,6 +2704,10 @@ end
 -- additional functionality becomes available. We will split this
 -- module in components when we're done with prototyping.
 
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
 -- This is the first code I wrote for LuaTeX, so it needs some cleanup.
 
 -- To be considered: hash key lowercase, first entry in table filename
@@ -2689,9 +2718,6 @@ end
 
 -- Beware, loading and saving is overloaded in luat-tmp!
 
--- todo: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
--- todo: check escaping in find etc, too much, too slow
-
 if not versions    then versions    = { } end versions['luat-inp'] = 1.001
 if not environment then environment = { } end
 if not file        then file        = { } end
@@ -2710,6 +2736,8 @@ if not input.hashers    then input.hashers    = { } end  -- load databases
 if not input.generators then input.generators = { } end  -- generate databases
 if not input.filters    then input.filters    = { } end  -- conversion filters
 
+local format = string.format
+
 input.locators.notfound   = { nil }
 input.hashers.notfound    = { nil }
 input.generators.notfound = { nil }
@@ -2719,6 +2747,7 @@ input.banner       = nil
 input.verbose      = false
 input.debug        = false
 input.cnfname      = 'texmf.cnf'
+input.luaname      = 'texmfcnf.lua'
 input.lsrname      = 'ls-R'
 input.luasuffix    = '.tma'
 input.lucsuffix    = '.tmc'
@@ -2808,12 +2837,14 @@ function input.reset()
     instance.files           = { }
     instance.remap           = { }
     instance.configuration   = { }
+    instance.setup           = { }
     instance.order           = { }
     instance.found           = { }
     instance.foundintrees    = { }
     instance.kpsevars        = { }
     instance.hashes          = { }
     instance.cnffiles        = { }
+    instance.luafiles        = { }
     instance.lists           = { }
     instance.remember        = true
     instance.diskcache       = true
@@ -2833,7 +2864,6 @@ function input.reset()
     instance.stoptime        = 0
     instance.validfile       = function(path,name) return true end
     instance.data            = { } -- only for loading
-    instance.sortdata        = false
     instance.force_suffixes  = true
     instance.dummy_path_expr = "^!*unset/*$"
     instance.fakepaths       = { }
@@ -2846,7 +2876,7 @@ function input.reset()
         end
     else
         -- we will access os.env frequently
-        for k,v in pairs({'HOME','TEXMF','TEXMFCNF','SELFAUTOPARENT'}) do
+        for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do
             local e = os.getenv(v)
             if e then
             --  input.report("setting",v,"to",input.bare_variable(e))
@@ -2874,7 +2904,7 @@ function input.reset_hashes(instance)
     instance.found = { }
 end
 
-function input.bare_variable(str)
+function input.bare_variable(str) -- assumes str is a string
  -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1")
     return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2"))
 end
@@ -2959,7 +2989,7 @@ do
                 instance.stoptime = stoptime
                 instance.loadtime = instance.loadtime + loadtime
                 if report then
-                    input.report('load time', string.format("%0.3f",loadtime))
+                    input.report('load time', format("%0.3f",loadtime))
                 end
                 return loadtime
             end
@@ -2970,7 +3000,7 @@ do
 end
 
 function input.elapsedtime(instance)
-    return string.format("%0.3f",(instance and instance.loadtime) or 0)
+    return format("%0.3f",(instance and instance.loadtime) or 0)
 end
 
 function input.report_loadtime(instance)
@@ -2986,15 +3016,19 @@ function input.env(instance,key)
 end
 
 function input.osenv(instance,key)
-    if instance.environment[key] == nil then
-        local e = os.getenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
         if e == nil then
-            instance.environment[key] = "" -- false
+         -- value = "" -- false
         else
-            instance.environment[key] = input.bare_variable(e)
+            value = input.bare_variable(e)
         end
+        ie[key] = value
     end
-    return instance.environment[key] or ""
+    return value or ""
 end
 
 -- we follow a rather traditional approach:
@@ -3005,66 +3039,103 @@ end
 -- for the moment we don't expect a configuration file in a zip
 
 function input.identify_cnf(instance)
+    -- we no longer support treepath and rootpath (was handy for testing);
+    -- also we now follow the stupid route: if not set then just assume *one*
+    -- cnf file under texmf (i.e. distribution)
     if #instance.cnffiles == 0 then
-        if instance.treepath ~= "" then
-            -- this is a special purpose branch, not really used
-            if instance.rootpath ~= "" then
-                local t = instance.treepath:splitchr(',')
-                for k,v in ipairs(t) do
-                    t[k] = file.join(instance.rootpath,v)
+        if input.env(instance,'TEXMFCNF') == "" then
+            local ownpath = environment.ownpath() or "."
+            if ownpath then
+                -- beware, this is tricky on my own system because at that location I do have
+                -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess
+                local function locate(filename,list)
+                    local ownroot = input.normalize_name(file.join(ownpath,"../.."))
+                    if not lfs.isdir(file.join(ownroot,"texmf")) then
+                        ownroot = input.normalize_name(file.join(ownpath,".."))
+                        if not lfs.isdir(file.join(ownroot,"texmf")) then
+                            input.verbose = true
+                            input.report("error", "unable to identify cnf file")
+                            return
+                        end
+                    end
+                    local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself
+                    if not lfs.isfile(texmfcnf) then
+                        texmfcnf = file.join(ownroot,"texmf/web2c",filename)
+                        if not lfs.isfile(texmfcnf) then
+                            input.verbose = true
+                            input.report("error", "unable to locate",filename)
+                            return
+                        end
+                    end
+                    table.insert(list,texmfcnf)
+                    local ie = instance.environment
+                    if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end
+                    if not ie['TEXMFCNF']       then ie['TEXMFCNF']       = file.dirname(texmfcnf) end
+                end
+                locate(input.luaname,instance.luafiles)
+                locate(input.cnfname,instance.cnffiles)
+                if #instance.luafiles == 0 and instance.cnffiles == 0 then
+                    input.verbose = true
+                    input.report("error", "unable to locate",filename)
+                    os.exit()
                 end
-                instance.treepath = table.concat(t,',')
+                -- here we also assume then TEXMF is set in the distribution, if this trickery is
+                -- used in the minimals, then users who don't use setuptex are on their own with
+                -- regards to extra trees
+            else
+                input.verbose = true
+                input.report("error", "unable to identify own path")
+                os.exit()
             end
-            local t = instance.treepath:splitchr(',')
-            instance.environment['TEXMF'] = input.bare_variable(instance.treepath)
-            instance.environment['TEXMFCNF'] = file.join(t[1] or '.','texmf/web2c')
-        end
-        if instance.rootpath ~= "" then
-            -- this assumes a single path, maybe do an expanded split here too
-            instance.environment['TEXMFCNF'] = file.join(instance.rootpath,'texmf/web2c')
-            instance.environment['SELFAUTOPARENT'] = instance.rootpath
-        end
-        if input.env(instance,'TEXMFCNF') ~= "" then
+        else
             local t = input.split_path(input.env(instance,'TEXMFCNF'))
             t = input.aux.expanded_path(instance,t)
             input.aux.expand_vars(instance,t)
-            for _,v in ipairs(t) do
-                table.insert(instance.cnffiles,file.join(v,input.cnfname))
-            end
-        elseif input.env(instance,'SELFAUTOPARENT') == '.' then
-            table.insert(instance.cnffiles,file.join('.',input.cnfname))
-        else
-            for _,v in ipairs({'texmf-local','texmf'}) do
-                table.insert(instance.cnffiles,file.join(input.env(instance,'SELFAUTOPARENT'),v,'web2c',input.cnfname))
+            local function locate(filename,list)
+                for _,v in ipairs(t) do
+                    local texmfcnf = input.normalize_name(file.join(v,filename))
+                    if lfs.isfile(texmfcnf) then
+                        table.insert(list,texmfcnf)
+                    end
+                end
             end
+            locate(input.luaname,instance.luafiles)
+            locate(input.cnfname,instance.cnffiles)
         end
     end
 end
 
 function input.load_cnf(instance)
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            input.aux.load_cnf(instance,fname)
+        end
+    end
     -- instance.cnffiles contain complete names now !
     if #instance.cnffiles == 0 then
         input.report("no cnf files found (TEXMFCNF may not be set/known)")
     else
         instance.rootpath = instance.cnffiles[1]
         for k,fname in ipairs(instance.cnffiles) do
-            instance.cnffiles[k] = fname:gsub("\\",'/') -- needed?
+            instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/'))
         end
         for i=1,3 do
             instance.rootpath = file.dirname(instance.rootpath)
         end
+        instance.rootpath = input.normalize_name(instance.rootpath)
+        instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure
         if instance.lsrmode then
-            input.loadconfigdata(instance,instance.cnffiles)
+            loadoldconfigdata()
         elseif instance.diskcache and not instance.renewcache then
-            input.loadconfig(instance,instance.cnffiles)
+            input.loadoldconfig(instance,instance.cnffiles)
             if instance.loaderror then
-                input.loadconfigdata(instance,instance.cnffiles)
-                input.saveconfig(instance)
+                loadoldconfigdata()
+                input.saveoldconfig(instance)
             end
         else
-            input.loadconfigdata(instance,instance.cnffiles)
+            loadoldconfigdata()
             if instance.renewcache then
-                input.saveconfig(instance)
+                input.saveoldconfig(instance)
             end
         end
         input.aux.collapse_cnf_data(instance)
@@ -3072,40 +3143,34 @@ function input.load_cnf(instance)
     input.checkconfigdata(instance)
 end
 
-function input.loadconfigdata(instance)
-    for _, fname in ipairs(instance.cnffiles) do
-        input.aux.load_cnf(instance,fname)
+function input.load_lua(instance)
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = input.normalize_name(instance.rootpath)
+        instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure
+        input.loadnewconfig(instance)
+        input.aux.collapse_cnf_data(instance)
     end
+    input.checkconfigdata(instance)
 end
 
-if os.env then
-    function input.aux.collapse_cnf_data(instance)
-        for _,c in ipairs(instance.order) do
-            for k,v in pairs(c) do
-                if not instance.variables[k] then
-                    if instance.environment[k] then
-                        instance.variables[k] = instance.environment[k]
-                    else
-                        instance.kpsevars[k] = true
-                        instance.variables[k] = input.bare_variable(v)
-                    end
-                end
-            end
-        end
-    end
-else
-    function input.aux.collapse_cnf_data(instance)
-        for _,c in ipairs(instance.order) do
-            for k,v in pairs(c) do
-                if not instance.variables[k] then
-                    local e = os.getenv(k)
-                    if e then
-                        instance.environment[k] = input.bare_variable(e)
-                        instance.variables[k]   = instance.environment[k]
-                    else
-                        instance.variables[k] = input.bare_variable(v)
-                        instance.kpsevars[k]  = true
-                    end
+function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in pairs(c) do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = input.bare_variable(v)
                 end
             end
         end
@@ -3116,11 +3181,11 @@ function input.aux.load_cnf(instance,fname)
     fname = input.clean_path(fname)
     local lname = fname:gsub("%.%a+$",input.luasuffix)
     local f = io.open(lname)
-    if f then
+    if f then -- this will go
         f:close()
         local dname = file.dirname(fname)
         if not instance.configuration[dname] then
-            input.aux.load_data(instance,dname,'configuration',file.basename(lname))
+            input.aux.load_configuration(instance,dname,lname)
             instance.order[#instance.order+1] = instance.configuration[dname]
         end
     else
@@ -3135,7 +3200,7 @@ function input.aux.load_cnf(instance,fname)
             end
             local data = instance.configuration[dname]
             while true do
-                line = f:read()
+                local line, n = f:read(), 0
                 if line then
                     while true do -- join lines
                         line, n = line:gsub("\\%s*$", "")
@@ -3146,7 +3211,7 @@ function input.aux.load_cnf(instance,fname)
                         end
                     end
                     if not line:find("^[%%#]") then
-                        k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$")
+                        local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$")
                         if k and v and not data[k] then
                             data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME")
                             instance.kpsevars[k] = true
@@ -3220,6 +3285,7 @@ end
 
 function input.locatelists(instance)
     for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do
+        path = file.collapse_path(path)
         input.report("locating list of",path)
         input.locatedatabase(instance,input.normalize_name(path))
     end
@@ -3256,7 +3322,7 @@ function input.loadfiles(instance)
 end
 
 function input.hashers.tex(instance,tag,name)
-    input.aux.load_data(instance,tag,'files')
+    input.aux.load_files(instance,tag)
 end
 
 -- generators:
@@ -3333,7 +3399,7 @@ do
                 end
             end
             action()
-            input.report(string.format("%s files found on %s directories with %s uppercase remappings",n,m,r))
+            input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r))
         else
             local fullname = file.join(specification,input.lsrname)
             local path     = '.'
@@ -3386,7 +3452,7 @@ end
 -- is more convenient.
 
 function input.splitconfig(instance)
-    for i,c in ipairs(instance.order) do
+    for i,c in ipairs(instance) do
         for k,v in pairs(c) do
             if type(v) == 'string' then
                 local t = file.split_path(v)
@@ -3420,14 +3486,7 @@ function input.join_path(str)
         return str
     end
 end
---~ function input.splitexpansions(instance)
---~     for k,v in pairs(instance.expansions) do
---~         local t = file.split_path(v)
---~         if #t >  1 then
---~             instance.expansions[k] = t
---~         end
---~     end
---~ end
+
 function input.splitexpansions(instance)
     for k,v in pairs(instance.expansions) do
         local t, h = { }, { }
@@ -3447,7 +3506,7 @@ end
 
 -- end of split/join code
 
-function input.saveconfig(instance)
+function input.saveoldconfig(instance)
     input.splitconfig(instance)
     input.aux.save_data(instance, 'configuration', nil)
     input.joinconfig(instance)
@@ -3460,44 +3519,83 @@ input.configbanner = [[
 -- not copyrighted. [HH & TH]
 ]]
 
-function input.aux.save_data(instance, dataname, check)
-    for cachename, files in pairs(instance[dataname]) do
-        local name = file.join(cachename,dataname)
-        local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
-        local f = io.open(luaname,'w')
-        if f then
-            input.report("saving " .. dataname .. " in", luaname)
-            f:write(input.configbanner)
-            f:write("\n")
-            f:write("if not texmf      then texmf      = { } end\n")
-            f:write("if not texmf.data then texmf.data = { } end\n")
-            f:write("\n")
-            f:write("texmf.data.type    = '" .. dataname .. "'\n")
-            f:write("texmf.data.version = '" .. input.cacheversion .. "'\n")
-            f:write("texmf.data.date    = '" .. os.date("%Y-%m-%d") .. "'\n")
-            f:write("texmf.data.time    = '" .. os.date("%H:%M:%S") .. "'\n")
-            f:write('texmf.data.content = {\n')
-            local function dump(k,v)
-                if not check or check(v,k) then -- path, name
-                    if type(v) == 'string' then
-                        f:write("\t['" .. k .. "'] = '" .. v .. "',\n")
-                    elseif #v == 1 then
-                        f:write("\t['" .. k .. "'] = '" .. v[1] .. "',\n")
-                    else
-                        f:write("\t['" .. k .. "'] = {'" .. table.concat(v,"','").. "'},\n")
-                    end
+function input.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local concat = table.concat
+    local sorted = table.sortedkeys
+    local function dump(k,v,m)
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+        end
+    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sorted(files)) do
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sorted(fk)) do
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
                 end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
             end
-            if instance.sortdata then
-                for _, k in pairs(table.sortedkeys(files)) do
-                    dump(k,files[k])
+        end
+    else
+        for k, v in pairs(files) do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in pairs(v) do
+                    t[#t+1] = dump(kk,vv,"\t\t")
                 end
+                t[#t+1] = "\t},"
             else
-                for k, v in pairs(files) do
-                    dump(k,v)
+                t[#t+1] = dump(k,v,"\t")
+            end
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+if not texmf then texmf = {} end -- no longer needed, at least not here
+
+function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload
+    for cachename, files in pairs(instance[dataname]) do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
+        input.report("preparing " .. dataname .. " for", luaname)
+        for k, v in pairs(files) do
+            if not check or check(v,k) then -- path, name
+                if type(v) == "table" and #v == 1 then
+                    files[k] = v[1]
                 end
+            else
+                files[k] = nil -- false
             end
-            f:write('}\n')
+        end
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = input.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local f = io.open(luaname,'w')
+        if f then
+            input.report("saving " .. dataname .. " in", luaname)
+            f:write(input.serialize(data))
             f:close()
             input.report("compiling " .. dataname .. " to", lucname)
             if not utils.lua.compile(luaname,lucname) then
@@ -3510,49 +3608,106 @@ function input.aux.save_data(instance, dataname, check)
     end
 end
 
-function input.loadconfig(instance)
-    instance.configuration, instance.order, instance.loaderror = { }, { }, false
-    if not instance.renewcache then
-        for _, cnf in ipairs(instance.cnffiles) do
-            local dname = file.dirname(cnf)
-            input.aux.load_data(instance,dname,'configuration')
-            instance.order[#instance.order+1] = instance.configuration[dname]
-            if instance.loaderror then break end
+function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix)
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == input.cacheversion then
+            input.report("loading",dataname,"for",pathname,"from",filename)
+            instance[dataname][pathname] = data.content
+        else
+            input.report("skipping",dataname,"for",pathname,"from",filename)
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
         end
+    else
+        input.report("skipping",dataname,"for",pathname,"from",filename)
     end
-    input.joinconfig(instance)
 end
 
-if not texmf      then texmf      = {} end
-if not texmf.data then texmf.data = {} end
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
 
-function input.aux.load_data(instance,pathname,dataname,filename)
-    if not filename or (filename == "") then
-        filename = dataname .. input.lucsuffix
-    end
-    local blob = loadfile(file.join(pathname,filename))
-    if not blob then
-        filename = dataname .. input.luasuffix
-        blob = loadfile(file.join(pathname,filename))
-    end
+function input.aux.load_texmfcnf(instance,dataname,pathname)
+    local filename = file.join(pathname,input.luaname)
+    local blob = loadfile(filename)
     if blob then
-        blob()
-        if (texmf.data.type == dataname) and (texmf.data.version == input.cacheversion) and texmf.data.content then
-            input.report("loading",dataname,"for",pathname,"from",filename)
-            instance[dataname][pathname] = texmf.data.content
+        local data = blob()
+        if data then
+            input.report("loading","configuration file",filename)
+            if true then
+                -- flatten to variable.progname
+                local t = { }
+                for k, v in pairs(data) do -- v = progname
+                    if type(v) == "string" then
+                        t[k] = v
+                    else
+                        for kk, vv in pairs(v) do -- vv = variable
+                            if type(vv) == "string" then
+                                t[vv.."."..v] = kk
+                            end
+                        end
+                    end
+                end
+                instance[dataname][pathname] = t
+            else
+                instance[dataname][pathname] = data
+            end
         else
-            input.report("skipping",dataname,"for",pathname,"from",filename)
+            input.report("skipping","configuration file",filename)
             instance[dataname][pathname] = { }
             instance.loaderror = true
         end
+    else
+        input.report("skipping","configuration file",filename)
+    end
+end
+
+function input.aux.load_configuration(instance,dname,lname)
+    input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname))
+end
+function input.aux.load_files(instance,tag)
+    input.aux.load_data(instance,tag,'files')
+end
+
+function input.resetconfig(instance)
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function input.loadnewconfig(instance)
+    for _, cnf in ipairs(instance.luafiles) do
+        local dname = file.dirname(cnf)
+        input.aux.load_texmfcnf(instance,'setup',dname)
+        instance.order[#instance.order+1] = instance.setup[dname]
+        if instance.loaderror then break end
+    end
+end
+
+function input.loadoldconfig(instance)
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            input.aux.load_configuration(instance,dname)
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
+        end
     end
-    texmf.data.content = { }
+    input.joinconfig(instance)
 end
 
 function input.expand_variables(instance)
     instance.expansions = { }
-    if instance.engine   ~= "" then instance.environment['engine']   = instance.engine end
-    if instance.progname ~= "" then instance.environment['progname'] = instance.engine end
+--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath
+    if instance.engine   ~= "" then instance.environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then instance.environment['progname'] = instance.progname end
     for k,v in pairs(instance.environment) do
         local a, b = k:match("^(%a+)%_(.*)%s*$")
         if a and b then
@@ -3643,53 +3798,6 @@ function input.is_expansion(instance,name)
     return input.aux.is_entry(instance,instance.expansions,name)
 end
 
-function input.aux.list(instance,list)
-    local pat = string.upper(instance.pattern or "","")
-    for _,key in pairs(table.sortedkeys(list)) do
-        if (instance.pattern=="") or string.find(key:upper(),pat) then
-            if instance.kpseonly then
-                if instance.kpsevars[key] then
-                    print(key .. "=" .. input.aux.tabstr(list[key]))
-                end
-            elseif instance.kpsevars[key] then
-                print('K ' .. key .. "=" .. input.aux.tabstr(list[key]))
-            else
-                print('E ' .. key .. "=" .. input.aux.tabstr(list[key]))
-            end
-        end
-    end
-end
-
-function input.list_variables(instance)
-    input.aux.list(instance,instance.variables)
-end
-function input.list_expansions(instance)
-    input.aux.list(instance,instance.expansions)
-end
-
-function input.list_configurations(instance)
-    for _,key in pairs(table.sortedkeys(instance.kpsevars)) do
-        if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then
-            print(key.."\n")
-            for i,c in ipairs(instance.order) do
-                local str = c[key]
-                if str then
-                    print("\t" .. i .. "\t\t" .. input.aux.tabstr(str))
-                end
-            end
-            print()
-        end
-    end
-end
-
-function input.aux.tabstr(str)
-    if type(str) == 'table' then
-        return table.concat(str," | ")
-    else
-        return str
-    end
-end
-
 function input.simplified_list(str)
     if type(str) == 'table' then
         return str -- troubles ; ipv , in texmf
@@ -3713,23 +3821,6 @@ function input.unexpanded_path(instance,str)
     return file.join_path(input.unexpanded_path_list(instance,str))
 end
 
---~ function input.expanded_path_list(instance,str)
---~     if not str then
---~         return { }
---~     elseif instance.savelists then
---~         -- engine+progname hash
---~         str = str:gsub("%$","")
---~         if not instance.lists[str] then -- cached
---~             local lst = input.split_path(input.expansion(instance,str))
---~             instance.lists[str] = input.aux.expanded_path(instance,lst)
---~         end
---~         return instance.lists[str]
---~     else
---~         local lst = input.split_path(input.expansion(instance,str))
---~         return input.aux.expanded_path(instance,lst)
---~     end
---~ end
-
 do
     local done = { }
 
@@ -4142,6 +4233,8 @@ do
         return original
     end
 
+    input.normalize_name = file.collapse_path
+
 end
 
 function input.aux.register_in_trees(instance,name)
@@ -4507,7 +4600,10 @@ end
 
 function input.load(instance)
     input.starttiming(instance)
+    input.resetconfig(instance)
     input.identify_cnf(instance)
+    input.load_lua(instance)
+    input.expand_variables(instance)
     input.load_cnf(instance)
     input.expand_variables(instance)
     input.load_hash(instance)
@@ -4894,28 +4990,45 @@ caches.more   = caches.more or "context"
 caches.direct = false -- true is faster but may need huge amounts of memory
 caches.trace  = false
 caches.tree   = false
-caches.temp   = caches.temp or os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("VARTEXMF") or os.getenv("TEXMFVAR") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil
-caches.paths  = caches.paths or { caches.temp }
+caches.paths  = caches.paths or nil
 caches.force  = false
 
 input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true
 
-if caches.temp and caches.temp ~= "" and lfs.attributes(caches.temp,"mode") ~= "directory" then
-    if caches.force or io.ask(string.format("Should I create the cache path %s?",caches.temp), "no", { "yes", "no" }) == "yes" then
-        dir.mkdirs(caches.temp)
+function caches.temp(instance)
+    local function checkpath(cachepath)
+        if not cachepath or cachepath == "" then
+            return nil
+        elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then
+            return cachepath
+        elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
+            dir.mkdirs(cachepath)
+            return (lfs.attributes(cachepath,"mode") == "directory") and cachepath
+        else
+            return nil
+        end
     end
-end
-if not caches.temp or caches.temp == "" then
-    print("\nfatal error: there is no valid cache path defined\n")
-    os.exit()
-elseif lfs.attributes(caches.temp,"mode") ~= "directory" then
-    print(string.format("\nfatal error: cache path %s is not a directory\n",caches.temp))
-    os.exit()
+    local cachepath = input.expanded_path_list(instance,"TEXMFCACHE")
+    cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1])
+    if not cachepath then
+        cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil
+        cachepath = checkpath(cachepath)
+    end
+    if not cachepath then
+        print("\nfatal error: there is no valid cache path defined\n")
+        os.exit()
+    elseif lfs.attributes(cachepath,"mode") ~= "directory" then
+        print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath))
+        os.exit()
+    end
+    function caches.temp(instance)
+        return cachepath
+    end
+    return cachepath
 end
 
 function caches.configpath(instance)
     return table.concat(instance.cnffiles,";")
---~     return input.expand_var(instance,"TEXMFCNF")
 end
 
 function caches.hashed(tree)
@@ -4933,19 +5046,8 @@ end
 
 function caches.setpath(instance,...)
     if not caches.path then
-        if lfs and instance then
-            for _,v in pairs(caches.paths) do
-                for _,vv in pairs(input.expanded_path_list(instance,v)) do
-                    if lfs.isdir(vv) then
-                        caches.path = vv
-                        break
-                    end
-                end
-                if caches.path then break end
-            end
-        end
         if not caches.path then
-            caches.path = caches.temp
+            caches.path = caches.temp(instance)
         end
         caches.path = input.clean_path(caches.path) -- to be sure
         if lfs then
@@ -4969,6 +5071,12 @@ function caches.setpath(instance,...)
     return caches.path
 end
 
+function caches.definepath(instance,category,subcategory)
+    return function()
+        return caches.setpath(instance,category,subcategory)
+    end
+end
+
 function caches.setluanames(path,name)
     return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
 end
@@ -5034,19 +5142,35 @@ do -- local report
         end
     end
 
+    local allocated = { }
+
+    -- tracing
+
     function containers.define(category, subcategory, version, enabled)
-        if category and subcategory then
-            return {
-                category = category,
-                subcategory = subcategory,
-                storage = { },
-                enabled = enabled,
-                version = version or 1.000,
-                trace = false,
-                path = caches.setpath(texmf.instance,category,subcategory),
-            }
-        else
-            return nil
+        return function()
+            if category and subcategory then
+                local c = allocated[category]
+                if not c then
+                    c  = { }
+                    allocated[category] = c
+                end
+                local s = c[subcategory]
+                if not s then
+                    s = {
+                        category = category,
+                        subcategory = subcategory,
+                        storage = { },
+                        enabled = enabled,
+                        version = version or 1.000,
+                        trace = false,
+                        path = caches.setpath(texmf.instance,category,subcategory),
+                    }
+                    c[subcategory] = s
+                end
+                return s
+            else
+                return nil
+            end
         end
     end
 
@@ -5103,129 +5227,35 @@ end
 -- since we want to use the cache instead of the tree, we will now
 -- reimplement the saver.
 
+local save_data = input.aux.save_data
+
+input.cachepath = nil
+
 function input.aux.save_data(instance, dataname, check)
-    for cachename, files in pairs(instance[dataname]) do
-        local name
+    input.cachepath = input.cachepath or caches.definepath(instance,"trees")
+    save_data(instance, dataname, check, function(cachename,dataname)
         if input.usecache then
-            name = file.join(caches.setpath(instance,"trees"),caches.hashed(cachename))
-        else
-            name = file.join(cachename,dataname)
-        end
-        local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
-        input.report("preparing " .. dataname .. " in", luaname)
-        for k, v in pairs(files) do
-            if not check or check(v,k) then -- path, name
-                if type(v) == "table" and #v == 1 then
-                    files[k] = v[1]
-                end
-            else
-                files[k] = nil -- false
-            end
-        end
-        local data = {
-            type    = dataname,
-            root    = cachename,
-            version = input.cacheversion,
-            date    = os.date("%Y-%m-%d"),
-            time    = os.date("%H:%M:%S"),
-            content = files,
-        }
-        local f = io.open(luaname,'w')
-        if f then
-            input.report("saving " .. dataname .. " in", luaname)
-        --  f:write(table.serialize(data,'return'))
-            f:write(input.serialize(data))
-            f:close()
-            input.report("compiling " .. dataname .. " to", lucname)
-            if not utils.lua.compile(luaname,lucname) then
-                input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname)
-                os.remove(lucname)
-            end
+            return file.join(input.cachepath(),caches.hashed(cachename))
         else
-            input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix)
+            return file.join(cachename,dataname)
         end
-    end
+    end)
 end
 
-function input.serialize(files)
-    -- This version is somewhat optimized for the kind of
-    -- tables that we deal with, so it's much faster than
-    -- the generic serializer. This makes sense because
-    -- luatools and mtxtools are called frequently. Okay,
-    -- we pay a small price for properly tabbed tables.
-    local t = { }
-    local concat = table.concat
-    local sorted = table.sortedkeys
-    local function dump(k,v,m)
-        if type(v) == 'string' then
-            return m .. "['" .. k .. "']='" .. v .. "',"
-        elseif #v == 1 then
-            return m .. "['" .. k .. "']='" .. v[1] .. "',"
-        else
-            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
-        end
-    end
-    t[#t+1] = "return {"
-    if instance.sortdata then
-        for _, k in pairs(sorted(files)) do
-            local fk  = files[k]
-            if type(fk) == 'table' then
-                t[#t+1] = "\t['" .. k .. "']={"
-                for _, kk in pairs(sorted(fk)) do
-                    t[#t+1] = dump(kk,fk[kk],"\t\t")
-                end
-                t[#t+1] = "\t},"
-            else
-                t[#t+1] = dump(k,fk,"\t")
-            end
-        end
-    else
-        for k, v in pairs(files) do
-            if type(v) == 'table' then
-                t[#t+1] = "\t['" .. k .. "']={"
-                for kk,vv in pairs(v) do
-                    t[#t+1] = dump(kk,vv,"\t\t")
-                end
-                t[#t+1] = "\t},"
-            else
-                t[#t+1] = dump(k,v,"\t")
-            end
-        end
-    end
-    t[#t+1] = "}"
-    return concat(t,"\n")
-end
+local load_data = input.aux.load_data
 
 function input.aux.load_data(instance,pathname,dataname,filename)
-    local luaname, lucname, pname, fname
-    if input.usecache then
-        pname, fname = caches.setpath(instance,"trees"), caches.hashed(pathname)
-        filename = file.join(pname,fname)
-    else
-        if not filename or (filename == "") then
-            filename = dataname
-        end
-        pname, fname = pathname, filename
-    end
-    luaname = file.join(pname,fname) .. input.luasuffix
-    lucname = file.join(pname,fname) .. input.lucsuffix
-    local blob = loadfile(lucname)
-    if not blob then
-        blob = loadfile(luaname)
-    end
-    if blob then
-        local data = blob()
-        if data and data.content and data.type == dataname and data.version == input.cacheversion then
-            input.report("loading",dataname,"for",pathname,"from",filename)
-            instance[dataname][pathname] = data.content
+    input.cachepath = input.cachepath or caches.definepath(instance,"trees")
+    load_data(instance,pathname,dataname,filename,function(dataname,filename)
+        if input.usecache then
+            return file.join(input.cachepath(),caches.hashed(pathname))
         else
-            input.report("skipping",dataname,"for",pathname,"from",filename)
-            instance[dataname][pathname] = { }
-            instance.loaderror = true
+            if not filename or (filename == "") then
+                filename = dataname
+            end
+            return file.join(pathname,filename)
         end
-    else
-        input.report("skipping",dataname,"for",pathname,"from",filename)
-    end
+    end)
 end
 
 -- we will make a better format, maybe something xml or just text or lua
@@ -5853,7 +5883,6 @@ if texconfig and not texlua then
 
         if not texmf.instance then -- prevent a second loading
 
-
             texmf.instance            = input.reset()
             texmf.instance.progname   = environment.progname or 'context'
             texmf.instance.engine     = environment.engine   or 'luatex'
@@ -6297,7 +6326,10 @@ if environment.arguments["minimize"] then
 end
 
 function input.my_prepare_a(instance)
+    input.resetconfig(instance)
     input.identify_cnf(instance)
+    input.load_lua(instance)
+    input.expand_variables(instance)
     input.load_cnf(instance)
     input.expand_variables(instance)
 end
@@ -6472,6 +6504,51 @@ function input.my_run_format(instance,name,data,more)
     end
 end
 
+-- helpers for verbose lists
+
+input.listers = input.listers or { }
+
+local function tabstr(str)
+    if type(str) == 'table' then
+        return table.concat(str," | ")
+    else
+        return str
+    end
+end
+
+local function list(instance,list)
+    local pat = string.upper(instance.pattern or "","")
+    for _,key in pairs(table.sortedkeys(list)) do
+        if instance.pattern == "" or string.find(key:upper(),pat) then
+            if instance.kpseonly then
+                if instance.kpsevars[key] then
+                    print(format("%s=%s",key,tabstr(list[key])))
+                end
+            else
+                print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key])))
+            end
+        end
+    end
+end
+
+function input.listers.variables (instance) list(instance,instance.variables ) end
+function input.listers.expansions(instance) list(instance,instance.expansions) end
+
+function input.listers.configurations(instance)
+    for _,key in pairs(table.sortedkeys(instance.kpsevars)) do
+        if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then
+            print(key.."\n")
+            for i,c in ipairs(instance.order) do
+                local str = c[key]
+                if str then
+                    print(format("\t%s\t\t%s",i,input.aux.tabstr(str)))
+                end
+            end
+            print()
+        end
+    end
+end
+
 input.report(banner,"\n")
 
 local ok = true
@@ -6545,13 +6622,13 @@ elseif environment.arguments["selfupdate"] then
     input.update_script(instance,own.name,"luatools")
 elseif environment.arguments["variables"] or environment.arguments["show-variables"] then
     input.my_prepare_a(instance)
-    input.list_variables(instance)
+    input.listers.variables(instance)
 elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then
     input.my_prepare_a(instance)
-    input.list_expansions(instance)
+    input.listers.expansions(instance)
 elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then
     input.my_prepare_a(instance)
-    input.list_configurations(instance)
+    input.listers.configurations(instance)
 elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then
     if not input.verbose then
         input.verbose = true
diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua
index ae0ee1b8e..6c444d531 100644
--- a/scripts/context/lua/mtx-context.lua
+++ b/scripts/context/lua/mtx-context.lua
@@ -583,7 +583,8 @@ function scripts.context.run(ctxdata)
                 --
             end
         else
-            input.error("no format found with name " .. formatname)
+            input.verbose = true
+            input.report("error", "no format found with name " .. formatname)
         end
     end
 end
diff --git a/scripts/context/lua/mtx-server.lua b/scripts/context/lua/mtx-server.lua
new file mode 100644
index 000000000..293bc0c1c
--- /dev/null
+++ b/scripts/context/lua/mtx-server.lua
@@ -0,0 +1,83 @@
+if not modules then modules = { } end modules ['mtx-server'] = {
+    version   = 1.001,
+    comment   = "companion to mtxrun.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+texmf.instance = instance -- we need to get rid of this / maybe current instance in global table
+
+-- The starting point was stripped down webserver.lua by Samuel
+-- Saint-Pettersen (as downloaded on 21-5-2008) which only served
+-- html and was not configureable. In due time I will extend the
+-- next code. Eventually we may move code to l-server.
+
+scripts           = scripts           or { }
+scripts.webserver = scripts.webserver or {}
+
+local socket = require("socket")
+
+local function message(str)
+    return string.format("<h1>%s</h1>",str)
+end
+
+function scripts.webserver.run(configuration)
+    local server = assert(socket.bind("*", tonumber(configuration.port or 8080)))
+    while true do
+        local client = server:accept()
+        client:settimeout(configuration.timeout or 60)
+        local request, e = client:receive()
+        if e then
+            client:send(message("404 Not Found"))
+        else
+            -- GET /showcase.pdf HTTP/1.1
+            local filename = request:match("GET (.+) HTTP/.*$") -- todo: more clever
+         -- filename = filename:gsub("%%(%d%d)",function(c) return string.char(tonumber(c,16)) end)
+            filename = socket.url.unescape(filename)
+            if filename == nil or filename == "" then
+                filename = configuration.index or "index.html"
+            end
+            -- todo chunked
+            local fullname = file.join(configuration.root,filename)
+            local data = io.loaddata(fullname)
+            if data and data ~= "" then
+                local result
+                client:send("HTTP/1.1 200 OK\r\n")
+                client:send("Connection: close\r\n")
+                if filename:find(".pdf$") then -- todo: special handler
+                    client:send(string.format("Content-Length: %s\r\n",#data))
+                    client:send("Content-Type: application/pdf\r\n")
+                else
+                    client:send("Content-Type: text/html\r\n")
+                end
+                client:send("\r\n")
+                client:send(data)
+                client:send("\r\n")
+            else
+                client:send(message("404 Not Found"))
+            end
+        end
+        client:close()
+    end
+end
+
+
+banner = banner .. " | webserver "
+
+messages.help = [[
+--start               start server
+--port                port to listen to
+--root                server root
+--index               index file
+]]
+
+if environment.argument("start") then
+    scripts.webserver.run {
+        port  = environment.argument("port")  or "8080",
+        root  = environment.argument("root")  or ".",           -- "e:/websites/www.pragma-ade.com",
+        index = environment.argument("index") or "index.html",
+    }
+else
+    input.help(banner,messages.help)
+end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 63605b42f..040214178 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -388,7 +388,6 @@ local patterns_escapes = {
     ["]"] = "%]",
 }
 
-
 function string:pattesc()
     return (self:gsub(".",patterns_escapes))
 end
@@ -1701,14 +1700,22 @@ end
 --~ print("../test/"       .. " == " .. file.collapse_path("../test/"))
 --~ print("a/a"            .. " == " .. file.collapse_path("a/b/c/../../a"))
 
+--~ function file.collapse_path(str)
+--~     local ok, n = false, 0
+--~     while not ok do
+--~         ok = true
+--~         str, n = str:gsub("[^%./]+/%.%./", function(s)
+--~             ok = false
+--~             return ""
+--~         end)
+--~     end
+--~     return (str:gsub("/%./","/"))
+--~ end
+
 function file.collapse_path(str)
-    local ok, n = false, 0
-    while not ok do
-        ok = true
-        str, n = str:gsub("[^%./]+/%.%./", function(s)
-            ok = false
-            return ""
-        end)
+    local n = 1
+    while n > 0 do
+        str, n = str:gsub("([^/%.]+/%.%./)","")
     end
     return (str:gsub("/%./","/"))
 end
@@ -2307,7 +2314,7 @@ do
 
     local remove, nsremap, resolvens = table.remove, xml.xmlns, xml.resolvens
 
-    local stack, top, dt, at, xmlns, errorstr = {}, {}, {}, {}, {}, nil
+    local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {}
 
     local mt = { __tostring = xml.text }
 
@@ -2383,14 +2390,6 @@ do
             dt[#dt+1] = text
         end
     end
-    --~ local function add_special(what, spacing, text)
-    --~     if #spacing > 0 then
-    --~         dt[#dt+1] = spacing
-    --~     end
-    --~     top = stack[#stack]    -- hm, left over 1
-    --~     setmetatable(top, mt)  -- hm, left over 2
-    --~     dt[#dt+1] = { special=true, ns="", tg=what, dt={text} }
-    --~ end
     local function add_special(what, spacing, text)
         if #spacing > 0 then
             dt[#dt+1] = spacing
@@ -2450,12 +2449,20 @@ do
     local somecomment      = C((1 - endcomment    )^0)
     local somecdata        = C((1 - endcdata      )^0)
 
+    function entity(k,v) entities[k] = v end
+
     local begindoctype     = open * P("!DOCTYPE")
     local enddoctype       = close
-    local publicdoctype    = P("PUBLIC") * somespace * value * somespace * value * somespace * balanced^0
-    local systemdoctype    = P("SYSTEM") * somespace * value * somespace                     * balanced^0
-    local simpledoctype    = (1-close)^1                                                          * balanced^0
-    local somedoctype      = C((somespace * P(publicdoctype + systemdoctype + simpledoctype) * optionalspace)^0)
+    local beginset         = P("[")
+    local endset           = P("]")
+    local doctypename      = C((1-somespace)^0)
+    local elementdoctype   = optionalspace * P("<!ELEMENT") * (1-close)^0 * close
+    local entitydoctype    = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close
+    local publicdoctype    = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace
+    local systemdoctype    = doctypename * somespace * P("SYSTEM") * somespace * value * somespace
+    local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset
+    local simpledoctype    = (1-close)^1                     -- * balanced^0
+    local somedoctype      = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0)
 
     local instruction      = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end
     local comment          = (spacing * begincomment     * somecomment     * endcomment    ) / function(...) add_special("@cm@",...) end
@@ -2481,9 +2488,11 @@ do
         children = text + V("parent") + emptyelement + comment + cdata + instruction,
     }
 
-    function xml.convert(data, no_root, strip_cm_and_dt)
+    -- todo: xml.new + properties like entities and strip and such (store in root)
+
+    function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear)
         strip = strip_cm_and_dt or xml.strip_cm_and_dt
-        stack, top, at, xmlns, errorstr, result = {}, {}, {}, {}, nil, nil
+        stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {}
         stack[#stack+1] = top
         top.dt = { }
         dt = top.dt
@@ -2502,7 +2511,7 @@ do
             result = stack[1]
         end
         if not no_root then
-            result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={} }
+            result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities }
             setmetatable(result, mt)
             local rdt = result.dt
             for k=1,#rdt do
@@ -2629,6 +2638,22 @@ do
         elseif not nocommands then
             local ec = e.command
             if ec ~= nil then -- we can have all kind of types
+
+if e.special then -- todo test for true/false
+    local etg, edt = e.tg, e.dt
+    local spc = specialconverter and specialconverter[etg]
+    if spc then
+--~ print("SPECIAL",etg,table.serialize(specialconverter), spc)
+        local result = spc(edt[1])
+        if result then
+            handle(result)
+            return
+        else
+            -- no need to handle any further
+        end
+    end
+end
+
                 local xc = xml.command
                 if xc then
                     xc(e,ec)
@@ -2678,12 +2703,18 @@ do
                         end
                     end
                 end
-                if ern and xml.trace_remap then
-                    if ats then
-                        ats[#ats+1] = format("xmlns:remapped='%s'",ern)
-                    else
-                        ats = { format("xmlns:remapped='%s'",ern) }
-                    end
+                if ern and xml.trace_remap and ern ~= ens then
+--~                     if ats then
+--~                         ats[#ats+1] = format("xmlns:remapped='%s'",ern)
+--~                     else
+--~                         ats = { format("xmlns:remapped='%s'",ern) }
+--~                     end
+--~ if ats then
+--~     ats[#ats+1] = format("remappedns='%s'",ens or '-')
+--~ else
+--~     ats = { format("remappedns='%s'",ens or '-') }
+--~ end
+ens = ern
                 end
                 if ens ~= "" then
                     if edt and #edt > 0 then
@@ -3261,10 +3292,10 @@ do
         end
     end
     functions.name = function(root,k,n)
--- way too fuzzy
+        -- way too fuzzy
         local found
         if not k or not n then
-            local ns, tg = root.ns, root.tg
+            local ns, tg = root.rn or root.ns or "", root.tg
             if not tg then
                 for i=1,#root do
                     local e = root[i]
@@ -3273,10 +3304,10 @@ do
                         break
                     end
                 end
-            elseif ns == "" then
-                return tg
-            else
+            elseif ns ~= "" then
                 return ns .. ":" .. tg
+            else
+                return tg
             end
         elseif n == 0 then
             local e = root[k]
@@ -3296,6 +3327,7 @@ do
                 end
             end
         else
+--~ print(k,n,#root)
             for i=k+1,#root,1 do
                 local e = root[i]
                 if type(e) == "table" then
@@ -3309,7 +3341,7 @@ do
             end
         end
         if found then
-            local ns, tg = found.ns, found.tg
+            local ns, tg = found.rn or found.ns or "", found.tg
             if ns ~= "" then
                 return ns .. ":" .. tg
             else
@@ -3709,7 +3741,7 @@ do
     end
     function xml.filters.tag(root,pattern,n)
         local tag = ""
-        xml.traverse(root, xml.lpath(pattern), function(r,d,k)
+        traverse(root, lpath(pattern), function(r,d,k)
             tag = xml.functions.tag(d,k,n and tonumber(n))
             return true
         end)
@@ -3717,7 +3749,7 @@ do
     end
     function xml.filters.name(root,pattern,n)
         local tag = ""
-        xml.traverse(root, xml.lpath(pattern), function(r,d,k)
+        traverse(root, lpath(pattern), function(r,d,k)
             tag = xml.functions.name(d,k,n and tonumber(n))
             return true
         end)
@@ -3748,8 +3780,13 @@ do
     local attribute_filter = xml.filters.attributes
     local default_filter   = xml.filters.default
 
+    -- todo: also hash, could be gc'd
+
     function xml.filter(root,pattern)
         local kind, a, b, c = parser:match(pattern)
+--~ if xml.trace_lpath then
+--~     print(pattern,kind,a,b,c)
+--~ end
         if kind == 1 or kind == 3 then
             return (filters[b] or default_filter)(root,a,c)
         elseif kind == 2 then
@@ -4292,7 +4329,7 @@ do if unicode and unicode.utf8 then
     xml.entities = xml.entities or { } -- xml.entities.handler == function
 
     function xml.entities.handler(e)
-        return format("[s]",e)
+        return format("[%s]",e)
     end
 
     local char = unicode.utf8.char
@@ -4301,6 +4338,8 @@ do if unicode and unicode.utf8 then
         return char(tonumber(s,16))
     end
 
+    local entities = xml.entities -- global entities
+
     function utfize(root)
         local d = root.dt
         for k=1,#d do
@@ -4318,8 +4357,6 @@ do if unicode and unicode.utf8 then
 
     xml.utfize = utfize
 
-    local entities = xml.entities
-
     local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks
         if e:find("#x") then
             return char(tonumber(e:sub(3),16))
@@ -4334,20 +4371,24 @@ do if unicode and unicode.utf8 then
         end
     end
 
-    function xml.resolve_entities(root)
-        local d = root.dt
-        for k=1,#d do
-            local dk = d[k]
-            if type(dk) == "string" then
-                if dk:find("&.-;") then
-                    d[k] = dk:gsub("&(.-);",resolve)
+    local function resolve_entities(root)
+        if not root.special or root.tg == "@rt@" then
+            local d = root.dt
+            for k=1,#d do
+                local dk = d[k]
+                if type(dk) == "string" then
+                    if dk:find("&.-;") then
+                        d[k] = dk:gsub("&(.-);",resolve)
+                    end
+                else
+                    resolve_entities(dk)
                 end
-            else
-                utfize(dk)
             end
         end
     end
 
+    xml.resolve_entities = resolve_entities
+
     function xml.utfize_text(str)
         if str:find("&#") then
             return (str:gsub("&#x(.-);",toutf))
@@ -4372,6 +4413,18 @@ do if unicode and unicode.utf8 then
         end
     end
 
+    -- experimental, this will be done differently
+
+    function xml.merge_entities(root)
+        local documententities = root.entities
+        local allentities = xml.entities
+        if documententities then
+            for k, v in pairs(documententities) do
+                allentities[k] = v
+            end
+        end
+    end
+
 end end
 
 --  xml.set_text_cleanup(xml.show_text_entities)
@@ -4590,14 +4643,32 @@ os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows")
 --
 -- for k,v in pairs(arg) do print(k,v) end
 
-if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
-    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
-end
-
 -- environment
 
 if not environment then environment = { } end
 
+environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex"
+
+local ownpath = nil -- we could use a metatable here
+
+function environment.ownpath()
+    if not ownpath then
+        for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do
+            local b = file.join(p,environment.ownbin)
+            if lfs.isfile(b..".exe") or lfs.isfile(b) then
+                ownpath = p
+                break
+            end
+        end
+        if not ownpath then ownpath = '.' end
+    end
+    return ownpath
+end
+
+if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then
+    arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil
+end
+
 environment.arguments            = { }
 environment.files                = { }
 environment.sorted_argument_keys = nil
@@ -4709,6 +4780,10 @@ end
 -- additional functionality becomes available. We will split this
 -- module in components when we're done with prototyping.
 
+-- TODO: os.getenv -> os.env[]
+-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
+-- TODO: check escaping in find etc, too much, too slow
+
 -- This is the first code I wrote for LuaTeX, so it needs some cleanup.
 
 -- To be considered: hash key lowercase, first entry in table filename
@@ -4719,9 +4794,6 @@ end
 
 -- Beware, loading and saving is overloaded in luat-tmp!
 
--- todo: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller)
--- todo: check escaping in find etc, too much, too slow
-
 if not versions    then versions    = { } end versions['luat-inp'] = 1.001
 if not environment then environment = { } end
 if not file        then file        = { } end
@@ -4740,6 +4812,8 @@ if not input.hashers    then input.hashers    = { } end  -- load databases
 if not input.generators then input.generators = { } end  -- generate databases
 if not input.filters    then input.filters    = { } end  -- conversion filters
 
+local format = string.format
+
 input.locators.notfound   = { nil }
 input.hashers.notfound    = { nil }
 input.generators.notfound = { nil }
@@ -4749,6 +4823,7 @@ input.banner       = nil
 input.verbose      = false
 input.debug        = false
 input.cnfname      = 'texmf.cnf'
+input.luaname      = 'texmfcnf.lua'
 input.lsrname      = 'ls-R'
 input.luasuffix    = '.tma'
 input.lucsuffix    = '.tmc'
@@ -4838,12 +4913,14 @@ function input.reset()
     instance.files           = { }
     instance.remap           = { }
     instance.configuration   = { }
+    instance.setup           = { }
     instance.order           = { }
     instance.found           = { }
     instance.foundintrees    = { }
     instance.kpsevars        = { }
     instance.hashes          = { }
     instance.cnffiles        = { }
+    instance.luafiles        = { }
     instance.lists           = { }
     instance.remember        = true
     instance.diskcache       = true
@@ -4863,7 +4940,6 @@ function input.reset()
     instance.stoptime        = 0
     instance.validfile       = function(path,name) return true end
     instance.data            = { } -- only for loading
-    instance.sortdata        = false
     instance.force_suffixes  = true
     instance.dummy_path_expr = "^!*unset/*$"
     instance.fakepaths       = { }
@@ -4876,7 +4952,7 @@ function input.reset()
         end
     else
         -- we will access os.env frequently
-        for k,v in pairs({'HOME','TEXMF','TEXMFCNF','SELFAUTOPARENT'}) do
+        for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do
             local e = os.getenv(v)
             if e then
             --  input.report("setting",v,"to",input.bare_variable(e))
@@ -4904,7 +4980,7 @@ function input.reset_hashes(instance)
     instance.found = { }
 end
 
-function input.bare_variable(str)
+function input.bare_variable(str) -- assumes str is a string
  -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1")
     return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2"))
 end
@@ -4989,7 +5065,7 @@ do
                 instance.stoptime = stoptime
                 instance.loadtime = instance.loadtime + loadtime
                 if report then
-                    input.report('load time', string.format("%0.3f",loadtime))
+                    input.report('load time', format("%0.3f",loadtime))
                 end
                 return loadtime
             end
@@ -5000,7 +5076,7 @@ do
 end
 
 function input.elapsedtime(instance)
-    return string.format("%0.3f",(instance and instance.loadtime) or 0)
+    return format("%0.3f",(instance and instance.loadtime) or 0)
 end
 
 function input.report_loadtime(instance)
@@ -5016,15 +5092,19 @@ function input.env(instance,key)
 end
 
 function input.osenv(instance,key)
-    if instance.environment[key] == nil then
-        local e = os.getenv(key)
+    local ie = instance.environment
+    local value = ie[key]
+    if value == nil then
+     -- local e = os.getenv(key)
+        local e = os.env[key]
         if e == nil then
-            instance.environment[key] = "" -- false
+         -- value = "" -- false
         else
-            instance.environment[key] = input.bare_variable(e)
+            value = input.bare_variable(e)
         end
+        ie[key] = value
     end
-    return instance.environment[key] or ""
+    return value or ""
 end
 
 -- we follow a rather traditional approach:
@@ -5035,66 +5115,103 @@ end
 -- for the moment we don't expect a configuration file in a zip
 
 function input.identify_cnf(instance)
+    -- we no longer support treepath and rootpath (was handy for testing);
+    -- also we now follow the stupid route: if not set then just assume *one*
+    -- cnf file under texmf (i.e. distribution)
     if #instance.cnffiles == 0 then
-        if instance.treepath ~= "" then
-            -- this is a special purpose branch, not really used
-            if instance.rootpath ~= "" then
-                local t = instance.treepath:splitchr(',')
-                for k,v in ipairs(t) do
-                    t[k] = file.join(instance.rootpath,v)
+        if input.env(instance,'TEXMFCNF') == "" then
+            local ownpath = environment.ownpath() or "."
+            if ownpath then
+                -- beware, this is tricky on my own system because at that location I do have
+                -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess
+                local function locate(filename,list)
+                    local ownroot = input.normalize_name(file.join(ownpath,"../.."))
+                    if not lfs.isdir(file.join(ownroot,"texmf")) then
+                        ownroot = input.normalize_name(file.join(ownpath,".."))
+                        if not lfs.isdir(file.join(ownroot,"texmf")) then
+                            input.verbose = true
+                            input.report("error", "unable to identify cnf file")
+                            return
+                        end
+                    end
+                    local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself
+                    if not lfs.isfile(texmfcnf) then
+                        texmfcnf = file.join(ownroot,"texmf/web2c",filename)
+                        if not lfs.isfile(texmfcnf) then
+                            input.verbose = true
+                            input.report("error", "unable to locate",filename)
+                            return
+                        end
+                    end
+                    table.insert(list,texmfcnf)
+                    local ie = instance.environment
+                    if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end
+                    if not ie['TEXMFCNF']       then ie['TEXMFCNF']       = file.dirname(texmfcnf) end
                 end
-                instance.treepath = table.concat(t,',')
+                locate(input.luaname,instance.luafiles)
+                locate(input.cnfname,instance.cnffiles)
+                if #instance.luafiles == 0 and instance.cnffiles == 0 then
+                    input.verbose = true
+                    input.report("error", "unable to locate",filename)
+                    os.exit()
+                end
+                -- here we also assume then TEXMF is set in the distribution, if this trickery is
+                -- used in the minimals, then users who don't use setuptex are on their own with
+                -- regards to extra trees
+            else
+                input.verbose = true
+                input.report("error", "unable to identify own path")
+                os.exit()
             end
-            local t = instance.treepath:splitchr(',')
-            instance.environment['TEXMF'] = input.bare_variable(instance.treepath)
-            instance.environment['TEXMFCNF'] = file.join(t[1] or '.','texmf/web2c')
-        end
-        if instance.rootpath ~= "" then
-            -- this assumes a single path, maybe do an expanded split here too
-            instance.environment['TEXMFCNF'] = file.join(instance.rootpath,'texmf/web2c')
-            instance.environment['SELFAUTOPARENT'] = instance.rootpath
-        end
-        if input.env(instance,'TEXMFCNF') ~= "" then
+        else
             local t = input.split_path(input.env(instance,'TEXMFCNF'))
             t = input.aux.expanded_path(instance,t)
             input.aux.expand_vars(instance,t)
-            for _,v in ipairs(t) do
-                table.insert(instance.cnffiles,file.join(v,input.cnfname))
-            end
-        elseif input.env(instance,'SELFAUTOPARENT') == '.' then
-            table.insert(instance.cnffiles,file.join('.',input.cnfname))
-        else
-            for _,v in ipairs({'texmf-local','texmf'}) do
-                table.insert(instance.cnffiles,file.join(input.env(instance,'SELFAUTOPARENT'),v,'web2c',input.cnfname))
+            local function locate(filename,list)
+                for _,v in ipairs(t) do
+                    local texmfcnf = input.normalize_name(file.join(v,filename))
+                    if lfs.isfile(texmfcnf) then
+                        table.insert(list,texmfcnf)
+                    end
+                end
             end
+            locate(input.luaname,instance.luafiles)
+            locate(input.cnfname,instance.cnffiles)
         end
     end
 end
 
 function input.load_cnf(instance)
+    local function loadoldconfigdata()
+        for _, fname in ipairs(instance.cnffiles) do
+            input.aux.load_cnf(instance,fname)
+        end
+    end
     -- instance.cnffiles contain complete names now !
     if #instance.cnffiles == 0 then
         input.report("no cnf files found (TEXMFCNF may not be set/known)")
     else
         instance.rootpath = instance.cnffiles[1]
         for k,fname in ipairs(instance.cnffiles) do
-            instance.cnffiles[k] = fname:gsub("\\",'/') -- needed?
+            instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/'))
         end
         for i=1,3 do
             instance.rootpath = file.dirname(instance.rootpath)
         end
+        instance.rootpath = input.normalize_name(instance.rootpath)
+        instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure
         if instance.lsrmode then
-            input.loadconfigdata(instance,instance.cnffiles)
+            loadoldconfigdata()
         elseif instance.diskcache and not instance.renewcache then
-            input.loadconfig(instance,instance.cnffiles)
+            input.loadoldconfig(instance,instance.cnffiles)
             if instance.loaderror then
-                input.loadconfigdata(instance,instance.cnffiles)
-                input.saveconfig(instance)
+                loadoldconfigdata()
+                input.saveoldconfig(instance)
             end
         else
-            input.loadconfigdata(instance,instance.cnffiles)
+            loadoldconfigdata()
             if instance.renewcache then
-                input.saveconfig(instance)
+                input.saveoldconfig(instance)
             end
         end
         input.aux.collapse_cnf_data(instance)
@@ -5102,40 +5219,34 @@ function input.load_cnf(instance)
     input.checkconfigdata(instance)
 end
 
-function input.loadconfigdata(instance)
-    for _, fname in ipairs(instance.cnffiles) do
-        input.aux.load_cnf(instance,fname)
+function input.load_lua(instance)
+    if #instance.luafiles == 0 then
+        -- yet harmless
+    else
+        instance.rootpath = instance.luafiles[1]
+        for k,fname in ipairs(instance.luafiles) do
+            instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/'))
+        end
+        for i=1,3 do
+            instance.rootpath = file.dirname(instance.rootpath)
+        end
+        instance.rootpath = input.normalize_name(instance.rootpath)
+        instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure
+        input.loadnewconfig(instance)
+        input.aux.collapse_cnf_data(instance)
     end
+    input.checkconfigdata(instance)
 end
 
-if os.env then
-    function input.aux.collapse_cnf_data(instance)
-        for _,c in ipairs(instance.order) do
-            for k,v in pairs(c) do
-                if not instance.variables[k] then
-                    if instance.environment[k] then
-                        instance.variables[k] = instance.environment[k]
-                    else
-                        instance.kpsevars[k] = true
-                        instance.variables[k] = input.bare_variable(v)
-                    end
-                end
-            end
-        end
-    end
-else
-    function input.aux.collapse_cnf_data(instance)
-        for _,c in ipairs(instance.order) do
-            for k,v in pairs(c) do
-                if not instance.variables[k] then
-                    local e = os.getenv(k)
-                    if e then
-                        instance.environment[k] = input.bare_variable(e)
-                        instance.variables[k]   = instance.environment[k]
-                    else
-                        instance.variables[k] = input.bare_variable(v)
-                        instance.kpsevars[k]  = true
-                    end
+function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared)
+    for _,c in ipairs(instance.order) do
+        for k,v in pairs(c) do
+            if not instance.variables[k] then
+                if instance.environment[k] then
+                    instance.variables[k] = instance.environment[k]
+                else
+                    instance.kpsevars[k] = true
+                    instance.variables[k] = input.bare_variable(v)
                 end
             end
         end
@@ -5146,11 +5257,11 @@ function input.aux.load_cnf(instance,fname)
     fname = input.clean_path(fname)
     local lname = fname:gsub("%.%a+$",input.luasuffix)
     local f = io.open(lname)
-    if f then
+    if f then -- this will go
         f:close()
         local dname = file.dirname(fname)
         if not instance.configuration[dname] then
-            input.aux.load_data(instance,dname,'configuration',file.basename(lname))
+            input.aux.load_configuration(instance,dname,lname)
             instance.order[#instance.order+1] = instance.configuration[dname]
         end
     else
@@ -5165,7 +5276,7 @@ function input.aux.load_cnf(instance,fname)
             end
             local data = instance.configuration[dname]
             while true do
-                line = f:read()
+                local line, n = f:read(), 0
                 if line then
                     while true do -- join lines
                         line, n = line:gsub("\\%s*$", "")
@@ -5176,7 +5287,7 @@ function input.aux.load_cnf(instance,fname)
                         end
                     end
                     if not line:find("^[%%#]") then
-                        k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$")
+                        local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$")
                         if k and v and not data[k] then
                             data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME")
                             instance.kpsevars[k] = true
@@ -5250,6 +5361,7 @@ end
 
 function input.locatelists(instance)
     for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do
+        path = file.collapse_path(path)
         input.report("locating list of",path)
         input.locatedatabase(instance,input.normalize_name(path))
     end
@@ -5286,7 +5398,7 @@ function input.loadfiles(instance)
 end
 
 function input.hashers.tex(instance,tag,name)
-    input.aux.load_data(instance,tag,'files')
+    input.aux.load_files(instance,tag)
 end
 
 -- generators:
@@ -5363,7 +5475,7 @@ do
                 end
             end
             action()
-            input.report(string.format("%s files found on %s directories with %s uppercase remappings",n,m,r))
+            input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r))
         else
             local fullname = file.join(specification,input.lsrname)
             local path     = '.'
@@ -5416,7 +5528,7 @@ end
 -- is more convenient.
 
 function input.splitconfig(instance)
-    for i,c in ipairs(instance.order) do
+    for i,c in ipairs(instance) do
         for k,v in pairs(c) do
             if type(v) == 'string' then
                 local t = file.split_path(v)
@@ -5450,14 +5562,7 @@ function input.join_path(str)
         return str
     end
 end
---~ function input.splitexpansions(instance)
---~     for k,v in pairs(instance.expansions) do
---~         local t = file.split_path(v)
---~         if #t >  1 then
---~             instance.expansions[k] = t
---~         end
---~     end
---~ end
+
 function input.splitexpansions(instance)
     for k,v in pairs(instance.expansions) do
         local t, h = { }, { }
@@ -5477,7 +5582,7 @@ end
 
 -- end of split/join code
 
-function input.saveconfig(instance)
+function input.saveoldconfig(instance)
     input.splitconfig(instance)
     input.aux.save_data(instance, 'configuration', nil)
     input.joinconfig(instance)
@@ -5490,44 +5595,83 @@ input.configbanner = [[
 -- not copyrighted. [HH & TH]
 ]]
 
-function input.aux.save_data(instance, dataname, check)
-    for cachename, files in pairs(instance[dataname]) do
-        local name = file.join(cachename,dataname)
-        local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
-        local f = io.open(luaname,'w')
-        if f then
-            input.report("saving " .. dataname .. " in", luaname)
-            f:write(input.configbanner)
-            f:write("\n")
-            f:write("if not texmf      then texmf      = { } end\n")
-            f:write("if not texmf.data then texmf.data = { } end\n")
-            f:write("\n")
-            f:write("texmf.data.type    = '" .. dataname .. "'\n")
-            f:write("texmf.data.version = '" .. input.cacheversion .. "'\n")
-            f:write("texmf.data.date    = '" .. os.date("%Y-%m-%d") .. "'\n")
-            f:write("texmf.data.time    = '" .. os.date("%H:%M:%S") .. "'\n")
-            f:write('texmf.data.content = {\n')
-            local function dump(k,v)
-                if not check or check(v,k) then -- path, name
-                    if type(v) == 'string' then
-                        f:write("\t['" .. k .. "'] = '" .. v .. "',\n")
-                    elseif #v == 1 then
-                        f:write("\t['" .. k .. "'] = '" .. v[1] .. "',\n")
-                    else
-                        f:write("\t['" .. k .. "'] = {'" .. table.concat(v,"','").. "'},\n")
-                    end
+function input.serialize(files)
+    -- This version is somewhat optimized for the kind of
+    -- tables that we deal with, so it's much faster than
+    -- the generic serializer. This makes sense because
+    -- luatools and mtxtools are called frequently. Okay,
+    -- we pay a small price for properly tabbed tables.
+    local t = { }
+    local concat = table.concat
+    local sorted = table.sortedkeys
+    local function dump(k,v,m)
+        if type(v) == 'string' then
+            return m .. "['" .. k .. "']='" .. v .. "',"
+        elseif #v == 1 then
+            return m .. "['" .. k .. "']='" .. v[1] .. "',"
+        else
+            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
+        end
+    end
+    t[#t+1] = "return {"
+    if instance.sortdata then
+        for _, k in pairs(sorted(files)) do
+            local fk  = files[k]
+            if type(fk) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for _, kk in pairs(sorted(fk)) do
+                    t[#t+1] = dump(kk,fk[kk],"\t\t")
                 end
+                t[#t+1] = "\t},"
+            else
+                t[#t+1] = dump(k,fk,"\t")
             end
-            if instance.sortdata then
-                for _, k in pairs(table.sortedkeys(files)) do
-                    dump(k,files[k])
+        end
+    else
+        for k, v in pairs(files) do
+            if type(v) == 'table' then
+                t[#t+1] = "\t['" .. k .. "']={"
+                for kk,vv in pairs(v) do
+                    t[#t+1] = dump(kk,vv,"\t\t")
                 end
+                t[#t+1] = "\t},"
             else
-                for k, v in pairs(files) do
-                    dump(k,v)
+                t[#t+1] = dump(k,v,"\t")
+            end
+        end
+    end
+    t[#t+1] = "}"
+    return concat(t,"\n")
+end
+
+if not texmf then texmf = {} end -- no longer needed, at least not here
+
+function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload
+    for cachename, files in pairs(instance[dataname]) do
+        local name = (makename or file.join)(cachename,dataname)
+        local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
+        input.report("preparing " .. dataname .. " for", luaname)
+        for k, v in pairs(files) do
+            if not check or check(v,k) then -- path, name
+                if type(v) == "table" and #v == 1 then
+                    files[k] = v[1]
                 end
+            else
+                files[k] = nil -- false
             end
-            f:write('}\n')
+        end
+        local data = {
+            type    = dataname,
+            root    = cachename,
+            version = input.cacheversion,
+            date    = os.date("%Y-%m-%d"),
+            time    = os.date("%H:%M:%S"),
+            content = files,
+        }
+        local f = io.open(luaname,'w')
+        if f then
+            input.report("saving " .. dataname .. " in", luaname)
+            f:write(input.serialize(data))
             f:close()
             input.report("compiling " .. dataname .. " to", lucname)
             if not utils.lua.compile(luaname,lucname) then
@@ -5540,49 +5684,106 @@ function input.aux.save_data(instance, dataname, check)
     end
 end
 
-function input.loadconfig(instance)
-    instance.configuration, instance.order, instance.loaderror = { }, { }, false
-    if not instance.renewcache then
-        for _, cnf in ipairs(instance.cnffiles) do
-            local dname = file.dirname(cnf)
-            input.aux.load_data(instance,dname,'configuration')
-            instance.order[#instance.order+1] = instance.configuration[dname]
-            if instance.loaderror then break end
+function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload
+    filename = ((not filename or (filename == "")) and dataname) or filename
+    filename = (makename and makename(dataname,filename)) or file.join(pathname,filename)
+    local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix)
+    if blob then
+        local data = blob()
+        if data and data.content and data.type == dataname and data.version == input.cacheversion then
+            input.report("loading",dataname,"for",pathname,"from",filename)
+            instance[dataname][pathname] = data.content
+        else
+            input.report("skipping",dataname,"for",pathname,"from",filename)
+            instance[dataname][pathname] = { }
+            instance.loaderror = true
         end
+    else
+        input.report("skipping",dataname,"for",pathname,"from",filename)
     end
-    input.joinconfig(instance)
 end
 
-if not texmf      then texmf      = {} end
-if not texmf.data then texmf.data = {} end
+-- some day i'll use the nested approach, but not yet (actually we even drop
+-- engine/progname support since we have only luatex now)
+--
+-- first texmfcnf.lua files are located, next the cached texmf.cnf files
+--
+-- return {
+--     TEXMFBOGUS = 'effe checken of dit werkt',
+-- }
 
-function input.aux.load_data(instance,pathname,dataname,filename)
-    if not filename or (filename == "") then
-        filename = dataname .. input.lucsuffix
-    end
-    local blob = loadfile(file.join(pathname,filename))
-    if not blob then
-        filename = dataname .. input.luasuffix
-        blob = loadfile(file.join(pathname,filename))
-    end
+function input.aux.load_texmfcnf(instance,dataname,pathname)
+    local filename = file.join(pathname,input.luaname)
+    local blob = loadfile(filename)
     if blob then
-        blob()
-        if (texmf.data.type == dataname) and (texmf.data.version == input.cacheversion) and texmf.data.content then
-            input.report("loading",dataname,"for",pathname,"from",filename)
-            instance[dataname][pathname] = texmf.data.content
+        local data = blob()
+        if data then
+            input.report("loading","configuration file",filename)
+            if true then
+                -- flatten to variable.progname
+                local t = { }
+                for k, v in pairs(data) do -- v = progname
+                    if type(v) == "string" then
+                        t[k] = v
+                    else
+                        for kk, vv in pairs(v) do -- vv = variable
+                            if type(vv) == "string" then
+                                t[vv.."."..v] = kk
+                            end
+                        end
+                    end
+                end
+                instance[dataname][pathname] = t
+            else
+                instance[dataname][pathname] = data
+            end
         else
-            input.report("skipping",dataname,"for",pathname,"from",filename)
+            input.report("skipping","configuration file",filename)
             instance[dataname][pathname] = { }
             instance.loaderror = true
         end
+    else
+        input.report("skipping","configuration file",filename)
+    end
+end
+
+function input.aux.load_configuration(instance,dname,lname)
+    input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname))
+end
+function input.aux.load_files(instance,tag)
+    input.aux.load_data(instance,tag,'files')
+end
+
+function input.resetconfig(instance)
+    instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false
+end
+
+function input.loadnewconfig(instance)
+    for _, cnf in ipairs(instance.luafiles) do
+        local dname = file.dirname(cnf)
+        input.aux.load_texmfcnf(instance,'setup',dname)
+        instance.order[#instance.order+1] = instance.setup[dname]
+        if instance.loaderror then break end
+    end
+end
+
+function input.loadoldconfig(instance)
+    if not instance.renewcache then
+        for _, cnf in ipairs(instance.cnffiles) do
+            local dname = file.dirname(cnf)
+            input.aux.load_configuration(instance,dname)
+            instance.order[#instance.order+1] = instance.configuration[dname]
+            if instance.loaderror then break end
+        end
     end
-    texmf.data.content = { }
+    input.joinconfig(instance)
 end
 
 function input.expand_variables(instance)
     instance.expansions = { }
-    if instance.engine   ~= "" then instance.environment['engine']   = instance.engine end
-    if instance.progname ~= "" then instance.environment['progname'] = instance.engine end
+--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath
+    if instance.engine   ~= "" then instance.environment['engine']   = instance.engine   end
+    if instance.progname ~= "" then instance.environment['progname'] = instance.progname end
     for k,v in pairs(instance.environment) do
         local a, b = k:match("^(%a+)%_(.*)%s*$")
         if a and b then
@@ -5673,53 +5874,6 @@ function input.is_expansion(instance,name)
     return input.aux.is_entry(instance,instance.expansions,name)
 end
 
-function input.aux.list(instance,list)
-    local pat = string.upper(instance.pattern or "","")
-    for _,key in pairs(table.sortedkeys(list)) do
-        if (instance.pattern=="") or string.find(key:upper(),pat) then
-            if instance.kpseonly then
-                if instance.kpsevars[key] then
-                    print(key .. "=" .. input.aux.tabstr(list[key]))
-                end
-            elseif instance.kpsevars[key] then
-                print('K ' .. key .. "=" .. input.aux.tabstr(list[key]))
-            else
-                print('E ' .. key .. "=" .. input.aux.tabstr(list[key]))
-            end
-        end
-    end
-end
-
-function input.list_variables(instance)
-    input.aux.list(instance,instance.variables)
-end
-function input.list_expansions(instance)
-    input.aux.list(instance,instance.expansions)
-end
-
-function input.list_configurations(instance)
-    for _,key in pairs(table.sortedkeys(instance.kpsevars)) do
-        if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then
-            print(key.."\n")
-            for i,c in ipairs(instance.order) do
-                local str = c[key]
-                if str then
-                    print("\t" .. i .. "\t\t" .. input.aux.tabstr(str))
-                end
-            end
-            print()
-        end
-    end
-end
-
-function input.aux.tabstr(str)
-    if type(str) == 'table' then
-        return table.concat(str," | ")
-    else
-        return str
-    end
-end
-
 function input.simplified_list(str)
     if type(str) == 'table' then
         return str -- troubles ; ipv , in texmf
@@ -5743,23 +5897,6 @@ function input.unexpanded_path(instance,str)
     return file.join_path(input.unexpanded_path_list(instance,str))
 end
 
---~ function input.expanded_path_list(instance,str)
---~     if not str then
---~         return { }
---~     elseif instance.savelists then
---~         -- engine+progname hash
---~         str = str:gsub("%$","")
---~         if not instance.lists[str] then -- cached
---~             local lst = input.split_path(input.expansion(instance,str))
---~             instance.lists[str] = input.aux.expanded_path(instance,lst)
---~         end
---~         return instance.lists[str]
---~     else
---~         local lst = input.split_path(input.expansion(instance,str))
---~         return input.aux.expanded_path(instance,lst)
---~     end
---~ end
-
 do
     local done = { }
 
@@ -6172,6 +6309,8 @@ do
         return original
     end
 
+    input.normalize_name = file.collapse_path
+
 end
 
 function input.aux.register_in_trees(instance,name)
@@ -6537,7 +6676,10 @@ end
 
 function input.load(instance)
     input.starttiming(instance)
+    input.resetconfig(instance)
     input.identify_cnf(instance)
+    input.load_lua(instance)
+    input.expand_variables(instance)
     input.load_cnf(instance)
     input.expand_variables(instance)
     input.load_hash(instance)
@@ -6924,28 +7066,45 @@ caches.more   = caches.more or "context"
 caches.direct = false -- true is faster but may need huge amounts of memory
 caches.trace  = false
 caches.tree   = false
-caches.temp   = caches.temp or os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("VARTEXMF") or os.getenv("TEXMFVAR") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil
-caches.paths  = caches.paths or { caches.temp }
+caches.paths  = caches.paths or nil
 caches.force  = false
 
 input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true
 
-if caches.temp and caches.temp ~= "" and lfs.attributes(caches.temp,"mode") ~= "directory" then
-    if caches.force or io.ask(string.format("Should I create the cache path %s?",caches.temp), "no", { "yes", "no" }) == "yes" then
-        dir.mkdirs(caches.temp)
+function caches.temp(instance)
+    local function checkpath(cachepath)
+        if not cachepath or cachepath == "" then
+            return nil
+        elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then
+            return cachepath
+        elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
+            dir.mkdirs(cachepath)
+            return (lfs.attributes(cachepath,"mode") == "directory") and cachepath
+        else
+            return nil
+        end
     end
-end
-if not caches.temp or caches.temp == "" then
-    print("\nfatal error: there is no valid cache path defined\n")
-    os.exit()
-elseif lfs.attributes(caches.temp,"mode") ~= "directory" then
-    print(string.format("\nfatal error: cache path %s is not a directory\n",caches.temp))
-    os.exit()
+    local cachepath = input.expanded_path_list(instance,"TEXMFCACHE")
+    cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1])
+    if not cachepath then
+        cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil
+        cachepath = checkpath(cachepath)
+    end
+    if not cachepath then
+        print("\nfatal error: there is no valid cache path defined\n")
+        os.exit()
+    elseif lfs.attributes(cachepath,"mode") ~= "directory" then
+        print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath))
+        os.exit()
+    end
+    function caches.temp(instance)
+        return cachepath
+    end
+    return cachepath
 end
 
 function caches.configpath(instance)
     return table.concat(instance.cnffiles,";")
---~     return input.expand_var(instance,"TEXMFCNF")
 end
 
 function caches.hashed(tree)
@@ -6963,19 +7122,8 @@ end
 
 function caches.setpath(instance,...)
     if not caches.path then
-        if lfs and instance then
-            for _,v in pairs(caches.paths) do
-                for _,vv in pairs(input.expanded_path_list(instance,v)) do
-                    if lfs.isdir(vv) then
-                        caches.path = vv
-                        break
-                    end
-                end
-                if caches.path then break end
-            end
-        end
         if not caches.path then
-            caches.path = caches.temp
+            caches.path = caches.temp(instance)
         end
         caches.path = input.clean_path(caches.path) -- to be sure
         if lfs then
@@ -6999,6 +7147,12 @@ function caches.setpath(instance,...)
     return caches.path
 end
 
+function caches.definepath(instance,category,subcategory)
+    return function()
+        return caches.setpath(instance,category,subcategory)
+    end
+end
+
 function caches.setluanames(path,name)
     return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
 end
@@ -7064,19 +7218,35 @@ do -- local report
         end
     end
 
+    local allocated = { }
+
+    -- tracing
+
     function containers.define(category, subcategory, version, enabled)
-        if category and subcategory then
-            return {
-                category = category,
-                subcategory = subcategory,
-                storage = { },
-                enabled = enabled,
-                version = version or 1.000,
-                trace = false,
-                path = caches.setpath(texmf.instance,category,subcategory),
-            }
-        else
-            return nil
+        return function()
+            if category and subcategory then
+                local c = allocated[category]
+                if not c then
+                    c  = { }
+                    allocated[category] = c
+                end
+                local s = c[subcategory]
+                if not s then
+                    s = {
+                        category = category,
+                        subcategory = subcategory,
+                        storage = { },
+                        enabled = enabled,
+                        version = version or 1.000,
+                        trace = false,
+                        path = caches.setpath(texmf.instance,category,subcategory),
+                    }
+                    c[subcategory] = s
+                end
+                return s
+            else
+                return nil
+            end
         end
     end
 
@@ -7133,129 +7303,35 @@ end
 -- since we want to use the cache instead of the tree, we will now
 -- reimplement the saver.
 
+local save_data = input.aux.save_data
+
+input.cachepath = nil
+
 function input.aux.save_data(instance, dataname, check)
-    for cachename, files in pairs(instance[dataname]) do
-        local name
+    input.cachepath = input.cachepath or caches.definepath(instance,"trees")
+    save_data(instance, dataname, check, function(cachename,dataname)
         if input.usecache then
-            name = file.join(caches.setpath(instance,"trees"),caches.hashed(cachename))
-        else
-            name = file.join(cachename,dataname)
-        end
-        local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
-        input.report("preparing " .. dataname .. " in", luaname)
-        for k, v in pairs(files) do
-            if not check or check(v,k) then -- path, name
-                if type(v) == "table" and #v == 1 then
-                    files[k] = v[1]
-                end
-            else
-                files[k] = nil -- false
-            end
-        end
-        local data = {
-            type    = dataname,
-            root    = cachename,
-            version = input.cacheversion,
-            date    = os.date("%Y-%m-%d"),
-            time    = os.date("%H:%M:%S"),
-            content = files,
-        }
-        local f = io.open(luaname,'w')
-        if f then
-            input.report("saving " .. dataname .. " in", luaname)
-        --  f:write(table.serialize(data,'return'))
-            f:write(input.serialize(data))
-            f:close()
-            input.report("compiling " .. dataname .. " to", lucname)
-            if not utils.lua.compile(luaname,lucname) then
-                input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname)
-                os.remove(lucname)
-            end
+            return file.join(input.cachepath(),caches.hashed(cachename))
         else
-            input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix)
+            return file.join(cachename,dataname)
         end
-    end
+    end)
 end
 
-function input.serialize(files)
-    -- This version is somewhat optimized for the kind of
-    -- tables that we deal with, so it's much faster than
-    -- the generic serializer. This makes sense because
-    -- luatools and mtxtools are called frequently. Okay,
-    -- we pay a small price for properly tabbed tables.
-    local t = { }
-    local concat = table.concat
-    local sorted = table.sortedkeys
-    local function dump(k,v,m)
-        if type(v) == 'string' then
-            return m .. "['" .. k .. "']='" .. v .. "',"
-        elseif #v == 1 then
-            return m .. "['" .. k .. "']='" .. v[1] .. "',"
-        else
-            return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
-        end
-    end
-    t[#t+1] = "return {"
-    if instance.sortdata then
-        for _, k in pairs(sorted(files)) do
-            local fk  = files[k]
-            if type(fk) == 'table' then
-                t[#t+1] = "\t['" .. k .. "']={"
-                for _, kk in pairs(sorted(fk)) do
-                    t[#t+1] = dump(kk,fk[kk],"\t\t")
-                end
-                t[#t+1] = "\t},"
-            else
-                t[#t+1] = dump(k,fk,"\t")
-            end
-        end
-    else
-        for k, v in pairs(files) do
-            if type(v) == 'table' then
-                t[#t+1] = "\t['" .. k .. "']={"
-                for kk,vv in pairs(v) do
-                    t[#t+1] = dump(kk,vv,"\t\t")
-                end
-                t[#t+1] = "\t},"
-            else
-                t[#t+1] = dump(k,v,"\t")
-            end
-        end
-    end
-    t[#t+1] = "}"
-    return concat(t,"\n")
-end
+local load_data = input.aux.load_data
 
 function input.aux.load_data(instance,pathname,dataname,filename)
-    local luaname, lucname, pname, fname
-    if input.usecache then
-        pname, fname = caches.setpath(instance,"trees"), caches.hashed(pathname)
-        filename = file.join(pname,fname)
-    else
-        if not filename or (filename == "") then
-            filename = dataname
-        end
-        pname, fname = pathname, filename
-    end
-    luaname = file.join(pname,fname) .. input.luasuffix
-    lucname = file.join(pname,fname) .. input.lucsuffix
-    local blob = loadfile(lucname)
-    if not blob then
-        blob = loadfile(luaname)
-    end
-    if blob then
-        local data = blob()
-        if data and data.content and data.type == dataname and data.version == input.cacheversion then
-            input.report("loading",dataname,"for",pathname,"from",filename)
-            instance[dataname][pathname] = data.content
+    input.cachepath = input.cachepath or caches.definepath(instance,"trees")
+    load_data(instance,pathname,dataname,filename,function(dataname,filename)
+        if input.usecache then
+            return file.join(input.cachepath(),caches.hashed(pathname))
         else
-            input.report("skipping",dataname,"for",pathname,"from",filename)
-            instance[dataname][pathname] = { }
-            instance.loaderror = true
+            if not filename or (filename == "") then
+                filename = dataname
+            end
+            return file.join(pathname,filename)
         end
-    else
-        input.report("skipping",dataname,"for",pathname,"from",filename)
-    end
+    end)
 end
 
 -- we will make a better format, maybe something xml or just text or lua
@@ -8027,7 +8103,10 @@ messages.help = [[
 ]]
 
 function input.runners.my_prepare_a(instance)
+    input.resetconfig(instance)
     input.identify_cnf(instance)
+    input.load_lua(instance)
+    input.expand_variables(instance)
     input.load_cnf(instance)
     input.expand_variables(instance)
 end
@@ -8035,6 +8114,7 @@ end
 function input.runners.my_prepare_b(instance)
     input.runners.my_prepare_a(instance)
     input.load_hash(instance)
+    input.automount(instance)
 end
 
 function input.runners.prepare(instance)
-- 
cgit v1.2.3