beta 2011.02.01 22:02

2 files changed, 428 insertions, 524 deletions

diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 6b34f0f92..fc5dd3354 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -10164,7 +10164,7 @@ local function identify()
     if texmfcaches then
         for k=1,#texmfcaches do
             local cachepath = texmfcaches[k]
-            cachepath = resolvers.getenv(cachepath)
+            cachepath = resolvers.expansion(cachepath) -- was getenv
             if cachepath ~= "" then
                 cachepath = resolvers.resolve(cachepath)
                 cachepath = resolvers.cleanpath(cachepath)
@@ -10595,7 +10595,7 @@ if not modules then modules = { } end modules ['data-res'] = {
 
 local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
 local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
-local next, type = next, type
+local next, type, rawget, setmetatable = next, type, rawget, setmetatable
 local os = os
 
 local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg
@@ -10646,20 +10646,89 @@ resolvers.defaultsuffixes = { "tex" } --  "mkiv", "cld" -- too tricky
 resolvers.instance = resolvers.instance or nil -- the current one (slow access)
 local     instance = resolvers.instance or nil -- the current one (fast access)
 
-function resolvers.newinstance()
+-- An instance has an environment (coming from the outside, kept raw), variables
+-- (coming from the configuration file), and expansions (variables with nested
+-- variables replaced). One can push something into the outer environment and
+-- its internal copy, but only the later one will be the raw unprefixed variant.
+
+function resolvers.setenv(key,value)
+    if instance then
+        -- this one will be consulted first when we stay inside
+        -- the current environment
+        instance.environment[key] = value
+        -- we feed back into the environment, and as this is used
+        -- by other applications (via os.execute) we need to make
+        -- sure that prefixes are resolved
+        ossetenv(key,resolvers.resolve(value))
+    end
+end
+
+-- Beware we don't want empty here as this one can be called early on
+-- and therefore we use rawget.
+
+local function getenv(key)
+    local value = rawget(instance.environment,key)
+    if value and value ~= "" then
+        return value
+    else
+        local e = osgetenv(key)
+        return e ~= nil and e ~= "" and checkedvariable(e) or ""
+    end
+end
+
+resolvers.getenv = getenv
+resolvers.env    = getenv
+
+-- We are going to use some metatable trickery where we backtrack from
+-- expansion to variable to environment.
+
+local function resolve(k)
+    return instance.expansions[k]
+end
+
+local dollarstripper   = lpeg.stripper("$")
+local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
+local backslashswapper = lpeg.replacer("\\","/")
+
+local somevariable     = P("$") / ""
+local somekey          = C(R("az","AZ","09","__","--")^1)
+local somethingelse    = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
+                       + P(";") * (P(";") / "")
+                       + P(1)
+local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 )
+
+local cleaner          = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
+local variablecleaner  = Cs((cleaner  + P(1))^0)
+
+local somevariable     = R("az","AZ","09","__","--")^1 / resolve
+local variable         = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/""))
+local variableresolver = Cs((variable + P(1))^0)
+
+local function expandedvariable(var)
+    return lpegmatch(variableexpander,var) or var
+end
+
+function resolvers.expandvariables()
+    -- no longer needed
+end
+
+local function collapse_configuration_data()
+    -- no longer needed
+end
+
+function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only
+
+    local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate()
 
     local newinstance = {
-        progname        = 'context',
-        engine          = 'luatex',
-        environment     = allocate(),
-        variables       = allocate(),
-        expansions      = allocate(),
+        environment     = environment,
+        variables       = variables,
+        expansions      = expansions,
+        order           = order,
         files           = allocate(),
         setups          = allocate(),
-        order           = allocate(),
         found           = allocate(),
         foundintrees    = allocate(),
-        origins         = allocate(),
         hashes          = allocate(),
         specification   = allocate(),
         lists           = allocate(),
@@ -10674,11 +10743,42 @@ function resolvers.newinstance()
         force_suffixes  = true,
     }
 
-    local ne = newinstance.environment
+    setmetatable(variables, { __index = function(t,k)
+        for i=1,#order do
+            v = order[i][k]
+            if v ~= nil then
+                t[k] = v
+                return v
+            end
+        end
+        if v == nil then
+            v = ""
+        end
+        t[k] = v
+        return v
+    end } )
 
-    for k, v in next, osenv do
-        ne[upper(k)] = checkedvariable(v)
-    end
+    setmetatable(environment, { __index = function(t,k)
+        v = osgetenv(k)
+        if v == nil then
+            v = variables[k]
+        end
+        if v ~= nil then
+            v = checkedvariable(v) or ""
+        end
+        t[k] = v
+        return v
+    end } )
+
+    setmetatable(expansions, { __index = function(t,k)
+        local v = environment[k]
+        if type(v) == "string" then
+            v = lpegmatch(variableresolver,v)
+            v = lpegmatch(variablecleaner,v)
+        end
+        t[k] = v
+        return v
+    end } )
 
     return newinstance
 
@@ -10699,40 +10799,9 @@ local function reset_hashes()
     instance.found = { }
 end
 
-function resolvers.setenv(key,value)
-    if instance then
-        instance.environment[key] = value
-        ossetenv(key,resolvers.resolve(value)) -- brr, we need info about the var being a path
-    end
-end
-
-local function getenv(key)
-    local value = instance.environment[key]
-    if value and value ~= "" then
-        return value
-    else
-        local e = osgetenv(key)
-        return e ~= nil and e ~= "" and checkedvariable(e) or ""
-    end
-end
-
-resolvers.getenv = getenv
-resolvers.env    = getenv
-
-local dollarstripper   = lpeg.stripper("$")
-local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
-local backslashswapper = lpeg.replacer("\\","/")
-
-local somevariable     = P("$") / ""
-local somekey          = C(R("az","AZ","09","__","--")^1)
-local somethingelse    = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
-                       + P(";") * (P(";") / "")
-                       + P(1)
-
-
 local slash = P("/")
 
-local pattern = Cs (
+local pathexpressionpattern = Cs (
     Cc("^") * (
         Cc("%") * S(".-")
       + slash^2 * P(-1) / "/.*"
@@ -10750,64 +10819,19 @@ local function makepathexpression(str)
     else
         local c = cache[str]
         if not c then
-            c = lpegmatch(pattern,str)
+            c = lpegmatch(pathexpressionpattern,str)
             cache[str] = c
         end
         return c
     end
 end
 
-local function resolve(key)
-    local value = instance.variables[key]
-    if value and value ~= "" then
-        return value
-    end
-    local value = instance.environment[key]
-    if value and value ~= "" then
-        return value
-    end
-    local e = osgetenv(key)
-    return e ~= nil and e ~= "" and checkedvariable(e) or ""
-end
-
-local pattern = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 )
-
-local function expandedvariable(var) -- simple vars
-    return lpegmatch(pattern,var) or var
-end
-
-local function entry(entries,name)
-    if name and name ~= "" then
-        name = lpegmatch(dollarstripper,name)
-        local result = entries[instance.progname .. '.' .. name] or entries[name]
-        if result then
-            return result
-        else
-            result = getenv(name)
-            if result then
-                instance.variables[name] = result
-                resolvers.expandvariables()
-                return instance.expansions[name] or ""
-            end
-        end
-    end
-    return ""
-end
-
-local function is_entry(entries,name)
-    if name and name ~= "" then
-        name = lpegmatch(dollarstripper,name)
-        return (entries[instance.progname .. '.' .. name] or entries[name]) ~= nil
-    else
-        return false
-    end
-end
-
 local function reportcriticalvariables()
     if trace_locating then
         for i=1,#resolvers.criticalvars do
-            local v = resolvers.criticalvars[i]
-            report_resolvers("variable '%s' set to '%s'",v,getenv(v) or "unknown")
+            local k = resolvers.criticalvars[i]
+            local v = resolvers.getenv(k) or "unknown" -- this one will not resolve !
+            report_resolvers("variable '%s' set to '%s'",k,v)
         end
         report_resolvers()
     end
@@ -10825,18 +10849,13 @@ local function identify_configuration_files()
             resolvers.luacnfstate = "environment"
         end
         reportcriticalvariables()
-        resolvers.expandvariables()
         local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec))
-     -- expandvars(cnfpaths) --- hm
         local luacnfname = resolvers.luacnfname
         for i=1,#cnfpaths do
             local filename = collapsepath(filejoin(cnfpaths[i],luacnfname))
-            local realname = resolvers.resolve(filename) -- no shortcut
-         -- if trace_locating then
-         --     report_resolvers("checking configuration file '%s'",filename)
-         -- end
+            local realname = resolvers.resolve(filename)
             if lfs.isfile(realname) then
-                specification[#specification+1] = filename -- or realname?
+                specification[#specification+1] = filename
                 if trace_locating then
                     report_resolvers("found configuration file '%s'",realname)
                 end
@@ -10869,42 +10888,25 @@ local function load_configuration_files()
                         report_resolvers("loading configuration file '%s'",filename)
                         report_resolvers()
                     end
-                    -- flattening is easier to deal with as we need to collapse
-                    local t = { }
-                    for k, v in next, data do -- k = progname or setter or variables
-                        if v ~= unset_variable then
-                            local kind = type(v)
-                            if kind == "string" then
-                                -- still supported, but preferably use the variables subtable
-                                t[k] = v
-                            elseif kind == "table" then
-                                if initializesetter(filename,k,v) then
-                                    -- directives, experiments, trackers, ...
-                                else
-                                    for kk, vv in next, v do -- vv = variable
-                                        if vv ~= unset_variable then
-                                            if type(vv) == "string" then
-                                             -- t[kk.."."..k] = vv
-                                                if k == "variables" then
-                                                 -- special table, shared variables can be grouped
-                                                    t[kk] = vv
-                                                else
-                                                 -- category.variable (progname)
-                                                    t[k .. "." .. kk] = vv
-                                                end
-                                            end
-                                        end
-                                    end
-                                end
-                            else
-                             -- report_resolvers("strange key '%s' in configuration file '%s'",k,filename)
+                    local variables = data.variables or { }
+                    local warning = false
+                    for k, v in next, data do
+                        local kind = type(v)
+                        if kind == "table" then
+                            initializesetter(filename,k,v)
+                        elseif variables[k] == nil then
+                            if trace_locating and not warning then
+                                report_resolvers("variables like '%s' in configuration file '%s' should move to the 'variables' subtable",
+                                    k,resolvers.resolve(filename))
+                                warning = true
                             end
+                            variables[k] = v
                         end
                     end
-                    setups[pathname] = t
+                    setups[pathname] = variables
                     if resolvers.luacnfstate == "default" then
                         -- the following code is not tested
-                        local cnfspec = t["TEXMFCNF"]
+                        local cnfspec = variables["TEXMFCNF"]
                         if cnfspec then
                             -- we push the value into the main environment (osenv) so
                             -- that it takes precedence over the default one and therefore
@@ -10941,29 +10943,7 @@ local function load_configuration_files()
     end
 end
 
-local function collapse_configuration_data() -- potential optimization: pass start index (setup and configuration are shared)
-    local order, variables, environment, origins = instance.order, instance.variables, instance.environment, instance.origins
-    for i=1,#order do
-        local c = order[i]
-        for k,v in next, c do
-            if variables[k] then
-                -- okay
-            else
-                local ek = environment[k]
-                if ek and ek ~= "" then
-                    variables[k], origins[k] = ek, "env"
-                else
-                    local bv = checkedvariable(v)
-                    variables[k], origins[k] = bv, "cnf"
-                end
-            end
-        end
-    end
-end
-
--- scheme magic
-
--- database loading
+-- scheme magic ... database loading
 
 local function load_file_databases()
     instance.loaderror, instance.files = false, allocate()
@@ -10980,30 +10960,34 @@ end
 local function locate_file_databases()
     -- todo: cache:// and tree:// (runtime)
     local texmfpaths = resolvers.expandedpathlist('TEXMF')
-    for i=1,#texmfpaths do
-        local path = collapsepath(texmfpaths[i])
-        local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
-        if stripped ~= "" then
-            local runtime = stripped == path
-            path = resolvers.cleanpath(path)
-            local spec = resolvers.splitmethod(stripped)
-            if spec.scheme == "cache" or spec.scheme == "file" then
-                stripped = spec.path
-            elseif runtime and (spec.noscheme or spec.scheme == "file") then
-                stripped = "tree:///" .. stripped
-            end
-            if trace_locating then
-                if runtime then
-                    report_resolvers("locating list of '%s' (runtime)",path)
-                else
-                    report_resolvers("locating list of '%s' (cached)",path)
+    if #texmfpaths > 0 then
+        for i=1,#texmfpaths do
+            local path = collapsepath(texmfpaths[i])
+            local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
+            if stripped ~= "" then
+                local runtime = stripped == path
+                path = resolvers.cleanpath(path)
+                local spec = resolvers.splitmethod(stripped)
+                if spec.scheme == "cache" or spec.scheme == "file" then
+                    stripped = spec.path
+                elseif runtime and (spec.noscheme or spec.scheme == "file") then
+                    stripped = "tree:///" .. stripped
+                end
+                if trace_locating then
+                    if runtime then
+                        report_resolvers("locating list of '%s' (runtime)",path)
+                    else
+                        report_resolvers("locating list of '%s' (cached)",path)
+                    end
                 end
+                methodhandler('locators',stripped)
             end
-            methodhandler('locators',stripped)
         end
-    end
-    if trace_locating then
-        report_resolvers()
+        if trace_locating then
+            report_resolvers()
+        end
+    elseif trace_locating then
+        report_resolvers("no texmf paths are defined (using TEXMF)")
     end
 end
 
@@ -11076,7 +11060,6 @@ function resolvers.extendtexmfvariable(specification) -- crap, we could better p
     else
         -- weird
     end
-    resolvers.expandvariables()
     reset_hashes()
 end
 
@@ -11109,67 +11092,16 @@ function resolvers.datastate()
     return caches.contentstate()
 end
 
-local function resolve(a)
-    return instance.expansions[a] or getenv(a)
-end
-
-local cleaner  = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
-
-local variable = R("az","AZ","09","__","--")^1 / resolve
-      variable = (P("$")/"") * (variable + (P("{")/"") * variable * (P("}")/""))
-
-      cleaner  = Cs((cleaner  + P(1))^0)
-      variable = Cs((variable + P(1))^0)
-
-function resolvers.expandvariables()
-    local expansions, environment, variables = allocate(), instance.environment, instance.variables
-    instance.expansions = expansions
-    local engine, progname = instance.engine, instance.progname
-    if type(engine)   ~= "string" then instance.engine,   engine   = "", "" end
-    if type(progname) ~= "string" then instance.progname, progname = "", "" end
-    if engine   ~= "" then environment.engine   = engine   end
-    if progname ~= "" then environment.progname = progname end
-    for k,v in next, environment do
-        expansions[k] = v
-    end
- -- for k,v in next, environment do -- move environment to expansions (variables are already in there)
- --     if expansions[k] == nil then expansions[k] = v end
- -- end
-    for k,v in next, variables do -- move variables to expansions
-        if expansions[k] == nil then expansions[k] = v end
-    end
-    repeat
-        local busy = false
-        for k,v in next, expansions do
-            local s = lpegmatch(variable,v)
-            if s ~= v then
-                busy = true
-                expansions[k] = s
-            end
-        end
-    until not busy
-
-    for k,v in next, expansions do
-        expansions[k] = lpegmatch(cleaner,v)
-    end
-end
-
-
-
 function resolvers.variable(name)
-    return entry(instance.variables,name)
+    local name = name and lpegmatch(dollarstripper,name)
+    local result = name and instance.variables[name]
+    return result ~= nil and result or ""
 end
 
 function resolvers.expansion(name)
-    return entry(instance.expansions,name)
-end
-
-function resolvers.is_variable(name)
-    return is_entry(instance.variables,name)
-end
-
-function resolvers.is_expansion(name)
-    return is_entry(instance.expansions,name)
+    local name = name and lpegmatch(dollarstripper,name)
+    local result = name and instance.expansions[name]
+    return result ~= nil and result or ""
 end
 
 function resolvers.unexpandedpathlist(str)
@@ -11298,9 +11230,8 @@ end
 
 function resolvers.expandedpathlist(str)
     if not str then
-        return ep or { } -- ep ?
+        return { }
     elseif instance.savelists then
-        -- engine+progname hash
         str = lpegmatch(dollarstripper,str)
         if not instance.lists[str] then -- cached
             local lst = made_list(instance,resolvers.splitpath(resolvers.expansion(str)))
@@ -11457,7 +11388,7 @@ local function collect_instance_files(filename,askedformat,allresults) -- todo :
     filename = collapsepath(filename)
     -- speed up / beware: format problem
     if instance.remember and not allresults then
-        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. askedformat
+        stamp = filename .. "--" .. askedformat
         if instance.found[stamp] then
             if trace_locating then
                 report_resolvers("remembered file '%s'",filename)
@@ -11848,8 +11779,6 @@ function resolvers.load(option)
     statistics.starttiming(instance)
     identify_configuration_files()
     load_configuration_files()
-    collapse_configuration_data()
-    resolvers.expandvariables()
     if option ~= "nofiles" then
         load_databases()
         resolvers.automount()
@@ -12020,16 +11949,16 @@ if not modules then modules = { } end modules ['data-pre'] = {
 -- of loading.
 
 
-local upper, lower, gsub = string.upper, string.lower, string.gsub
+local resolvers    = resolvers
+local prefixes     = utilities.storage.allocate()
+resolvers.prefixes = prefixes
 
-local resolvers = resolvers
-
-local prefixes = { }
-
-local getenv, cleanpath, findgivenfile = resolvers.getenv, resolvers.cleanpath, resolvers.findgivenfile
+local gsub = string.gsub
+local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion
+local getenv = resolvers.getenv -- we can probably also use resolvers.expansion
 
-prefixes.environment = function(str) -- getenv is case insensitive anyway
-    return cleanpath(getenv(str) or getenv(upper(str)) or getenv(lower(str)) or "")
+prefixes.environment = function(str)
+    return cleanpath(expansion(str))
 end
 
 prefixes.relative = function(str,n)
@@ -12118,9 +12047,11 @@ local function _resolve_(method,target)
     end
 end
 
+local resolved, abstract = { }, { }
 
-local resolved = { }
-local abstract = { }
+function resolvers.resetresolve(str)
+    resolved, abstract = { }, { }
+end
 
 local function resolve(str) -- use schemes, this one is then for the commandline only
     local res = resolved[str]
@@ -13300,12 +13231,36 @@ local function list(list,report,pattern)
     local instance = resolvers.instance
     local report = report or texio.write_nl
     local sorted = table.sortedkeys(list)
+    local result = { }
     for i=1,#sorted do
         local key = sorted[i]
-        if pattern == "" or find(upper(key),pattern) then
-            report(format('%s  %s=%s',instance.origins[key] or "---",key,tabstr(list[key])))
+        if key ~= "" and (pattern == "" or find(upper(key),pattern)) then
+            local raw = tabstr(rawget(list,key))
+            local val = tabstr(list[key])
+            local res = resolvers.resolve(val)
+            if raw and raw ~= "" then
+                if raw == val then
+                    if val == res then
+                        result[#result+1] = { key, raw }
+                    else
+                        result[#result+1] = { key, format('%s => %s',raw,res) }
+                    end
+                else
+                    if val == res then
+                        result[#result+1] = { key, format('%s => %s',raw,val) }
+                    else
+                        result[#result+1] = { key, format('%s => %s => %s',raw,val,res) }
+                    end
+                end
+            else
+                result[#result+1] = { key, "unset" }
+            end
         end
     end
+    utilities.formatters.formatcolumns(result)
+    for i=1,#result do
+        report(result[i])
+    end
 end
 
 function resolvers.listers.variables (report,pattern) list(resolvers.instance.variables, report,pattern) end
@@ -13730,7 +13685,7 @@ if not resolvers then
     os.exit()
 end
 
-logs.setprogram('MTXrun',"TDS Runner Tool 1.26")
+logs.setprogram('MTXrun',"TDS Runner Tool 1.30")
 
 if environment.arguments["verbose"] then
     trackers.enable("resolvers.locating")
@@ -13770,7 +13725,6 @@ messages.help = [[
 
 --verbose             give a bit more info
 --trackers=list       enable given trackers
---engine=str          target engine
 --progname=str        format or backend
 
 --edit                launch editor with found file
@@ -13899,8 +13853,6 @@ function runners.execute_script(fullname,internal,nosplit)
         elseif state == 'skip' then
             return true
         elseif state == "run" then
-            instance.progname = environment.argument("progname") or instance.progname
-            instance.format   = environment.argument("format")   or instance.format
             local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), ""
             if path ~= "" then
                 result = fullname
@@ -14340,9 +14292,7 @@ local before, after = environment.splitarguments(filename)
 environment.arguments_before, environment.arguments_after = before, after
 environment.initializearguments(before)
 
-instance.engine   = environment.argument("engine")   or 'luatex'
-instance.progname = environment.argument("progname") or 'context'
-instance.lsrmode  = environment.argument("lsr")      or false
+instance.lsrmode  = environment.argument("lsr") or false
 
 -- maybe the unset has to go to this level
 
@@ -14361,12 +14311,14 @@ if environment.argument("usekpse") or environment.argument("forcekpse") or is_mk
         other = "other text files",
     }
 
+    local progname = environment.argument("progname") or 'context'
+
     local function kpse_initialized()
         texconfig.kpse_init = true
         local t = os.clock()
-        local k = kpse.original.new("luatex",instance.progname)
+        local k = kpse.original.new("luatex",progname)
         local dummy = k:find_file("mtxrun.lua") -- so that we're initialized
-        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t)
+        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",progname,os.clock()-t)
         kpse_initialized = function() return k end
         return k
     end
@@ -14566,7 +14518,7 @@ elseif environment.argument("find-file") then
 
     resolvers.load()
     local pattern = environment.argument("pattern")
-    local format = environment.arguments["format"] or instance.format
+    local format = environment.arguments("format")
     if not pattern then
         runners.register_arguments(filename)
         environment.initializearguments(environment.arguments_after)
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 6b34f0f92..fc5dd3354 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -10164,7 +10164,7 @@ local function identify()
     if texmfcaches then
         for k=1,#texmfcaches do
             local cachepath = texmfcaches[k]
-            cachepath = resolvers.getenv(cachepath)
+            cachepath = resolvers.expansion(cachepath) -- was getenv
             if cachepath ~= "" then
                 cachepath = resolvers.resolve(cachepath)
                 cachepath = resolvers.cleanpath(cachepath)
@@ -10595,7 +10595,7 @@ if not modules then modules = { } end modules ['data-res'] = {
 
 local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
 local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
-local next, type = next, type
+local next, type, rawget, setmetatable = next, type, rawget, setmetatable
 local os = os
 
 local P, S, R, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg
@@ -10646,20 +10646,89 @@ resolvers.defaultsuffixes = { "tex" } --  "mkiv", "cld" -- too tricky
 resolvers.instance = resolvers.instance or nil -- the current one (slow access)
 local     instance = resolvers.instance or nil -- the current one (fast access)
 
-function resolvers.newinstance()
+-- An instance has an environment (coming from the outside, kept raw), variables
+-- (coming from the configuration file), and expansions (variables with nested
+-- variables replaced). One can push something into the outer environment and
+-- its internal copy, but only the later one will be the raw unprefixed variant.
+
+function resolvers.setenv(key,value)
+    if instance then
+        -- this one will be consulted first when we stay inside
+        -- the current environment
+        instance.environment[key] = value
+        -- we feed back into the environment, and as this is used
+        -- by other applications (via os.execute) we need to make
+        -- sure that prefixes are resolved
+        ossetenv(key,resolvers.resolve(value))
+    end
+end
+
+-- Beware we don't want empty here as this one can be called early on
+-- and therefore we use rawget.
+
+local function getenv(key)
+    local value = rawget(instance.environment,key)
+    if value and value ~= "" then
+        return value
+    else
+        local e = osgetenv(key)
+        return e ~= nil and e ~= "" and checkedvariable(e) or ""
+    end
+end
+
+resolvers.getenv = getenv
+resolvers.env    = getenv
+
+-- We are going to use some metatable trickery where we backtrack from
+-- expansion to variable to environment.
+
+local function resolve(k)
+    return instance.expansions[k]
+end
+
+local dollarstripper   = lpeg.stripper("$")
+local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
+local backslashswapper = lpeg.replacer("\\","/")
+
+local somevariable     = P("$") / ""
+local somekey          = C(R("az","AZ","09","__","--")^1)
+local somethingelse    = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
+                       + P(";") * (P(";") / "")
+                       + P(1)
+local variableexpander = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 )
+
+local cleaner          = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
+local variablecleaner  = Cs((cleaner  + P(1))^0)
+
+local somevariable     = R("az","AZ","09","__","--")^1 / resolve
+local variable         = (P("$")/"") * (somevariable + (P("{")/"") * somevariable * (P("}")/""))
+local variableresolver = Cs((variable + P(1))^0)
+
+local function expandedvariable(var)
+    return lpegmatch(variableexpander,var) or var
+end
+
+function resolvers.expandvariables()
+    -- no longer needed
+end
+
+local function collapse_configuration_data()
+    -- no longer needed
+end
+
+function resolvers.newinstance() -- todo: all vars will become lowercase and alphanum only
+
+    local environment, variables, expansions, order = allocate(), allocate(), allocate(), allocate()
 
     local newinstance = {
-        progname        = 'context',
-        engine          = 'luatex',
-        environment     = allocate(),
-        variables       = allocate(),
-        expansions      = allocate(),
+        environment     = environment,
+        variables       = variables,
+        expansions      = expansions,
+        order           = order,
         files           = allocate(),
         setups          = allocate(),
-        order           = allocate(),
         found           = allocate(),
         foundintrees    = allocate(),
-        origins         = allocate(),
         hashes          = allocate(),
         specification   = allocate(),
         lists           = allocate(),
@@ -10674,11 +10743,42 @@ function resolvers.newinstance()
         force_suffixes  = true,
     }
 
-    local ne = newinstance.environment
+    setmetatable(variables, { __index = function(t,k)
+        for i=1,#order do
+            v = order[i][k]
+            if v ~= nil then
+                t[k] = v
+                return v
+            end
+        end
+        if v == nil then
+            v = ""
+        end
+        t[k] = v
+        return v
+    end } )
 
-    for k, v in next, osenv do
-        ne[upper(k)] = checkedvariable(v)
-    end
+    setmetatable(environment, { __index = function(t,k)
+        v = osgetenv(k)
+        if v == nil then
+            v = variables[k]
+        end
+        if v ~= nil then
+            v = checkedvariable(v) or ""
+        end
+        t[k] = v
+        return v
+    end } )
+
+    setmetatable(expansions, { __index = function(t,k)
+        local v = environment[k]
+        if type(v) == "string" then
+            v = lpegmatch(variableresolver,v)
+            v = lpegmatch(variablecleaner,v)
+        end
+        t[k] = v
+        return v
+    end } )
 
     return newinstance
 
@@ -10699,40 +10799,9 @@ local function reset_hashes()
     instance.found = { }
 end
 
-function resolvers.setenv(key,value)
-    if instance then
-        instance.environment[key] = value
-        ossetenv(key,resolvers.resolve(value)) -- brr, we need info about the var being a path
-    end
-end
-
-local function getenv(key)
-    local value = instance.environment[key]
-    if value and value ~= "" then
-        return value
-    else
-        local e = osgetenv(key)
-        return e ~= nil and e ~= "" and checkedvariable(e) or ""
-    end
-end
-
-resolvers.getenv = getenv
-resolvers.env    = getenv
-
-local dollarstripper   = lpeg.stripper("$")
-local inhibitstripper  = P("!")^0 * Cs(P(1)^0)
-local backslashswapper = lpeg.replacer("\\","/")
-
-local somevariable     = P("$") / ""
-local somekey          = C(R("az","AZ","09","__","--")^1)
-local somethingelse    = P(";") * ((1-S("!{}/\\"))^1 * P(";") / "")
-                       + P(";") * (P(";") / "")
-                       + P(1)
-
-
 local slash = P("/")
 
-local pattern = Cs (
+local pathexpressionpattern = Cs (
     Cc("^") * (
         Cc("%") * S(".-")
       + slash^2 * P(-1) / "/.*"
@@ -10750,64 +10819,19 @@ local function makepathexpression(str)
     else
         local c = cache[str]
         if not c then
-            c = lpegmatch(pattern,str)
+            c = lpegmatch(pathexpressionpattern,str)
             cache[str] = c
         end
         return c
     end
 end
 
-local function resolve(key)
-    local value = instance.variables[key]
-    if value and value ~= "" then
-        return value
-    end
-    local value = instance.environment[key]
-    if value and value ~= "" then
-        return value
-    end
-    local e = osgetenv(key)
-    return e ~= nil and e ~= "" and checkedvariable(e) or ""
-end
-
-local pattern = Cs( (somevariable * (somekey/resolve) + somethingelse)^1 )
-
-local function expandedvariable(var) -- simple vars
-    return lpegmatch(pattern,var) or var
-end
-
-local function entry(entries,name)
-    if name and name ~= "" then
-        name = lpegmatch(dollarstripper,name)
-        local result = entries[instance.progname .. '.' .. name] or entries[name]
-        if result then
-            return result
-        else
-            result = getenv(name)
-            if result then
-                instance.variables[name] = result
-                resolvers.expandvariables()
-                return instance.expansions[name] or ""
-            end
-        end
-    end
-    return ""
-end
-
-local function is_entry(entries,name)
-    if name and name ~= "" then
-        name = lpegmatch(dollarstripper,name)
-        return (entries[instance.progname .. '.' .. name] or entries[name]) ~= nil
-    else
-        return false
-    end
-end
-
 local function reportcriticalvariables()
     if trace_locating then
         for i=1,#resolvers.criticalvars do
-            local v = resolvers.criticalvars[i]
-            report_resolvers("variable '%s' set to '%s'",v,getenv(v) or "unknown")
+            local k = resolvers.criticalvars[i]
+            local v = resolvers.getenv(k) or "unknown" -- this one will not resolve !
+            report_resolvers("variable '%s' set to '%s'",k,v)
         end
         report_resolvers()
     end
@@ -10825,18 +10849,13 @@ local function identify_configuration_files()
             resolvers.luacnfstate = "environment"
         end
         reportcriticalvariables()
-        resolvers.expandvariables()
         local cnfpaths = expandedpathfromlist(resolvers.splitpath(cnfspec))
-     -- expandvars(cnfpaths) --- hm
         local luacnfname = resolvers.luacnfname
         for i=1,#cnfpaths do
             local filename = collapsepath(filejoin(cnfpaths[i],luacnfname))
-            local realname = resolvers.resolve(filename) -- no shortcut
-         -- if trace_locating then
-         --     report_resolvers("checking configuration file '%s'",filename)
-         -- end
+            local realname = resolvers.resolve(filename)
             if lfs.isfile(realname) then
-                specification[#specification+1] = filename -- or realname?
+                specification[#specification+1] = filename
                 if trace_locating then
                     report_resolvers("found configuration file '%s'",realname)
                 end
@@ -10869,42 +10888,25 @@ local function load_configuration_files()
                         report_resolvers("loading configuration file '%s'",filename)
                         report_resolvers()
                     end
-                    -- flattening is easier to deal with as we need to collapse
-                    local t = { }
-                    for k, v in next, data do -- k = progname or setter or variables
-                        if v ~= unset_variable then
-                            local kind = type(v)
-                            if kind == "string" then
-                                -- still supported, but preferably use the variables subtable
-                                t[k] = v
-                            elseif kind == "table" then
-                                if initializesetter(filename,k,v) then
-                                    -- directives, experiments, trackers, ...
-                                else
-                                    for kk, vv in next, v do -- vv = variable
-                                        if vv ~= unset_variable then
-                                            if type(vv) == "string" then
-                                             -- t[kk.."."..k] = vv
-                                                if k == "variables" then
-                                                 -- special table, shared variables can be grouped
-                                                    t[kk] = vv
-                                                else
-                                                 -- category.variable (progname)
-                                                    t[k .. "." .. kk] = vv
-                                                end
-                                            end
-                                        end
-                                    end
-                                end
-                            else
-                             -- report_resolvers("strange key '%s' in configuration file '%s'",k,filename)
+                    local variables = data.variables or { }
+                    local warning = false
+                    for k, v in next, data do
+                        local kind = type(v)
+                        if kind == "table" then
+                            initializesetter(filename,k,v)
+                        elseif variables[k] == nil then
+                            if trace_locating and not warning then
+                                report_resolvers("variables like '%s' in configuration file '%s' should move to the 'variables' subtable",
+                                    k,resolvers.resolve(filename))
+                                warning = true
                             end
+                            variables[k] = v
                         end
                     end
-                    setups[pathname] = t
+                    setups[pathname] = variables
                     if resolvers.luacnfstate == "default" then
                         -- the following code is not tested
-                        local cnfspec = t["TEXMFCNF"]
+                        local cnfspec = variables["TEXMFCNF"]
                         if cnfspec then
                             -- we push the value into the main environment (osenv) so
                             -- that it takes precedence over the default one and therefore
@@ -10941,29 +10943,7 @@ local function load_configuration_files()
     end
 end
 
-local function collapse_configuration_data() -- potential optimization: pass start index (setup and configuration are shared)
-    local order, variables, environment, origins = instance.order, instance.variables, instance.environment, instance.origins
-    for i=1,#order do
-        local c = order[i]
-        for k,v in next, c do
-            if variables[k] then
-                -- okay
-            else
-                local ek = environment[k]
-                if ek and ek ~= "" then
-                    variables[k], origins[k] = ek, "env"
-                else
-                    local bv = checkedvariable(v)
-                    variables[k], origins[k] = bv, "cnf"
-                end
-            end
-        end
-    end
-end
-
--- scheme magic
-
--- database loading
+-- scheme magic ... database loading
 
 local function load_file_databases()
     instance.loaderror, instance.files = false, allocate()
@@ -10980,30 +10960,34 @@ end
 local function locate_file_databases()
     -- todo: cache:// and tree:// (runtime)
     local texmfpaths = resolvers.expandedpathlist('TEXMF')
-    for i=1,#texmfpaths do
-        local path = collapsepath(texmfpaths[i])
-        local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
-        if stripped ~= "" then
-            local runtime = stripped == path
-            path = resolvers.cleanpath(path)
-            local spec = resolvers.splitmethod(stripped)
-            if spec.scheme == "cache" or spec.scheme == "file" then
-                stripped = spec.path
-            elseif runtime and (spec.noscheme or spec.scheme == "file") then
-                stripped = "tree:///" .. stripped
-            end
-            if trace_locating then
-                if runtime then
-                    report_resolvers("locating list of '%s' (runtime)",path)
-                else
-                    report_resolvers("locating list of '%s' (cached)",path)
+    if #texmfpaths > 0 then
+        for i=1,#texmfpaths do
+            local path = collapsepath(texmfpaths[i])
+            local stripped = lpegmatch(inhibitstripper,path) -- the !! thing
+            if stripped ~= "" then
+                local runtime = stripped == path
+                path = resolvers.cleanpath(path)
+                local spec = resolvers.splitmethod(stripped)
+                if spec.scheme == "cache" or spec.scheme == "file" then
+                    stripped = spec.path
+                elseif runtime and (spec.noscheme or spec.scheme == "file") then
+                    stripped = "tree:///" .. stripped
+                end
+                if trace_locating then
+                    if runtime then
+                        report_resolvers("locating list of '%s' (runtime)",path)
+                    else
+                        report_resolvers("locating list of '%s' (cached)",path)
+                    end
                 end
+                methodhandler('locators',stripped)
             end
-            methodhandler('locators',stripped)
         end
-    end
-    if trace_locating then
-        report_resolvers()
+        if trace_locating then
+            report_resolvers()
+        end
+    elseif trace_locating then
+        report_resolvers("no texmf paths are defined (using TEXMF)")
     end
 end
 
@@ -11076,7 +11060,6 @@ function resolvers.extendtexmfvariable(specification) -- crap, we could better p
     else
         -- weird
     end
-    resolvers.expandvariables()
     reset_hashes()
 end
 
@@ -11109,67 +11092,16 @@ function resolvers.datastate()
     return caches.contentstate()
 end
 
-local function resolve(a)
-    return instance.expansions[a] or getenv(a)
-end
-
-local cleaner  = P("\\") / "/" + P(";") * S("!{}/\\")^0 * P(";")^1 / ";"
-
-local variable = R("az","AZ","09","__","--")^1 / resolve
-      variable = (P("$")/"") * (variable + (P("{")/"") * variable * (P("}")/""))
-
-      cleaner  = Cs((cleaner  + P(1))^0)
-      variable = Cs((variable + P(1))^0)
-
-function resolvers.expandvariables()
-    local expansions, environment, variables = allocate(), instance.environment, instance.variables
-    instance.expansions = expansions
-    local engine, progname = instance.engine, instance.progname
-    if type(engine)   ~= "string" then instance.engine,   engine   = "", "" end
-    if type(progname) ~= "string" then instance.progname, progname = "", "" end
-    if engine   ~= "" then environment.engine   = engine   end
-    if progname ~= "" then environment.progname = progname end
-    for k,v in next, environment do
-        expansions[k] = v
-    end
- -- for k,v in next, environment do -- move environment to expansions (variables are already in there)
- --     if expansions[k] == nil then expansions[k] = v end
- -- end
-    for k,v in next, variables do -- move variables to expansions
-        if expansions[k] == nil then expansions[k] = v end
-    end
-    repeat
-        local busy = false
-        for k,v in next, expansions do
-            local s = lpegmatch(variable,v)
-            if s ~= v then
-                busy = true
-                expansions[k] = s
-            end
-        end
-    until not busy
-
-    for k,v in next, expansions do
-        expansions[k] = lpegmatch(cleaner,v)
-    end
-end
-
-
-
 function resolvers.variable(name)
-    return entry(instance.variables,name)
+    local name = name and lpegmatch(dollarstripper,name)
+    local result = name and instance.variables[name]
+    return result ~= nil and result or ""
 end
 
 function resolvers.expansion(name)
-    return entry(instance.expansions,name)
-end
-
-function resolvers.is_variable(name)
-    return is_entry(instance.variables,name)
-end
-
-function resolvers.is_expansion(name)
-    return is_entry(instance.expansions,name)
+    local name = name and lpegmatch(dollarstripper,name)
+    local result = name and instance.expansions[name]
+    return result ~= nil and result or ""
 end
 
 function resolvers.unexpandedpathlist(str)
@@ -11298,9 +11230,8 @@ end
 
 function resolvers.expandedpathlist(str)
     if not str then
-        return ep or { } -- ep ?
+        return { }
     elseif instance.savelists then
-        -- engine+progname hash
         str = lpegmatch(dollarstripper,str)
         if not instance.lists[str] then -- cached
             local lst = made_list(instance,resolvers.splitpath(resolvers.expansion(str)))
@@ -11457,7 +11388,7 @@ local function collect_instance_files(filename,askedformat,allresults) -- todo :
     filename = collapsepath(filename)
     -- speed up / beware: format problem
     if instance.remember and not allresults then
-        stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. askedformat
+        stamp = filename .. "--" .. askedformat
         if instance.found[stamp] then
             if trace_locating then
                 report_resolvers("remembered file '%s'",filename)
@@ -11848,8 +11779,6 @@ function resolvers.load(option)
     statistics.starttiming(instance)
     identify_configuration_files()
     load_configuration_files()
-    collapse_configuration_data()
-    resolvers.expandvariables()
     if option ~= "nofiles" then
         load_databases()
         resolvers.automount()
@@ -12020,16 +11949,16 @@ if not modules then modules = { } end modules ['data-pre'] = {
 -- of loading.
 
 
-local upper, lower, gsub = string.upper, string.lower, string.gsub
+local resolvers    = resolvers
+local prefixes     = utilities.storage.allocate()
+resolvers.prefixes = prefixes
 
-local resolvers = resolvers
-
-local prefixes = { }
-
-local getenv, cleanpath, findgivenfile = resolvers.getenv, resolvers.cleanpath, resolvers.findgivenfile
+local gsub = string.gsub
+local cleanpath, findgivenfile, expansion = resolvers.cleanpath, resolvers.findgivenfile, resolvers.expansion
+local getenv = resolvers.getenv -- we can probably also use resolvers.expansion
 
-prefixes.environment = function(str) -- getenv is case insensitive anyway
-    return cleanpath(getenv(str) or getenv(upper(str)) or getenv(lower(str)) or "")
+prefixes.environment = function(str)
+    return cleanpath(expansion(str))
 end
 
 prefixes.relative = function(str,n)
@@ -12118,9 +12047,11 @@ local function _resolve_(method,target)
     end
 end
 
+local resolved, abstract = { }, { }
 
-local resolved = { }
-local abstract = { }
+function resolvers.resetresolve(str)
+    resolved, abstract = { }, { }
+end
 
 local function resolve(str) -- use schemes, this one is then for the commandline only
     local res = resolved[str]
@@ -13300,12 +13231,36 @@ local function list(list,report,pattern)
     local instance = resolvers.instance
     local report = report or texio.write_nl
     local sorted = table.sortedkeys(list)
+    local result = { }
     for i=1,#sorted do
         local key = sorted[i]
-        if pattern == "" or find(upper(key),pattern) then
-            report(format('%s  %s=%s',instance.origins[key] or "---",key,tabstr(list[key])))
+        if key ~= "" and (pattern == "" or find(upper(key),pattern)) then
+            local raw = tabstr(rawget(list,key))
+            local val = tabstr(list[key])
+            local res = resolvers.resolve(val)
+            if raw and raw ~= "" then
+                if raw == val then
+                    if val == res then
+                        result[#result+1] = { key, raw }
+                    else
+                        result[#result+1] = { key, format('%s => %s',raw,res) }
+                    end
+                else
+                    if val == res then
+                        result[#result+1] = { key, format('%s => %s',raw,val) }
+                    else
+                        result[#result+1] = { key, format('%s => %s => %s',raw,val,res) }
+                    end
+                end
+            else
+                result[#result+1] = { key, "unset" }
+            end
         end
     end
+    utilities.formatters.formatcolumns(result)
+    for i=1,#result do
+        report(result[i])
+    end
 end
 
 function resolvers.listers.variables (report,pattern) list(resolvers.instance.variables, report,pattern) end
@@ -13730,7 +13685,7 @@ if not resolvers then
     os.exit()
 end
 
-logs.setprogram('MTXrun',"TDS Runner Tool 1.26")
+logs.setprogram('MTXrun',"TDS Runner Tool 1.30")
 
 if environment.arguments["verbose"] then
     trackers.enable("resolvers.locating")
@@ -13770,7 +13725,6 @@ messages.help = [[
 
 --verbose             give a bit more info
 --trackers=list       enable given trackers
---engine=str          target engine
 --progname=str        format or backend
 
 --edit                launch editor with found file
@@ -13899,8 +13853,6 @@ function runners.execute_script(fullname,internal,nosplit)
         elseif state == 'skip' then
             return true
         elseif state == "run" then
-            instance.progname = environment.argument("progname") or instance.progname
-            instance.format   = environment.argument("format")   or instance.format
             local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), ""
             if path ~= "" then
                 result = fullname
@@ -14340,9 +14292,7 @@ local before, after = environment.splitarguments(filename)
 environment.arguments_before, environment.arguments_after = before, after
 environment.initializearguments(before)
 
-instance.engine   = environment.argument("engine")   or 'luatex'
-instance.progname = environment.argument("progname") or 'context'
-instance.lsrmode  = environment.argument("lsr")      or false
+instance.lsrmode  = environment.argument("lsr") or false
 
 -- maybe the unset has to go to this level
 
@@ -14361,12 +14311,14 @@ if environment.argument("usekpse") or environment.argument("forcekpse") or is_mk
         other = "other text files",
     }
 
+    local progname = environment.argument("progname") or 'context'
+
     local function kpse_initialized()
         texconfig.kpse_init = true
         local t = os.clock()
-        local k = kpse.original.new("luatex",instance.progname)
+        local k = kpse.original.new("luatex",progname)
         local dummy = k:find_file("mtxrun.lua") -- so that we're initialized
-        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t)
+        logs.simple("kpse fallback with progname '%s' initialized in %s seconds",progname,os.clock()-t)
         kpse_initialized = function() return k end
         return k
     end
@@ -14566,7 +14518,7 @@ elseif environment.argument("find-file") then
 
     resolvers.load()
     local pattern = environment.argument("pattern")
-    local format = environment.arguments["format"] or instance.format
+    local format = environment.arguments("format")
     if not pattern then
         runners.register_arguments(filename)
         environment.initializearguments(environment.arguments_after)