beta 2011.06.23 19:25

1 files changed, 507 insertions, 304 deletions

diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 449c341ab..488594892 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -1114,6 +1114,13 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
 
 local lpeg = require("lpeg")
 
+-- tracing (only used when we encounter a problem in integration of lpeg in luatex)
+
+local report = texio and texio.write_nl or print
+
+
+
+
 local type = type
 local byte, char = string.byte, string.char
 
@@ -1222,17 +1229,17 @@ patterns.unspacer      = ((patterns.spacer^1)/"")^0
 patterns.somecontent   = (anything - newline - space)^1 -- (utf8char - newline - space)^1
 patterns.beginline     = #(1-newline)
 
-local unquoted = Cs(patterns.unquoted * endofstring) -- not C
-
-function string.unquoted(str)
-    return match(unquoted,str) or str
-end
-
--- more efficient:
+-- local unquoted = Cs(patterns.unquoted * endofstring) -- not C
+--
+-- function string.unquoted(str)
+--     return match(unquoted,str) or str
+-- end
+--
+-- more efficient on long strings:
 
 local unquoted = (
-    squote * Cs(1 - P(-2)) * squote
-  + dquote * Cs(1 - P(-2)) * dquote
+    squote * Cs((1 - P(-2))^0) * squote
+  + dquote * Cs((1 - P(-2))^0) * dquote
 )
 
 function string.unquoted(str)
@@ -1241,6 +1248,12 @@ end
 
 patterns.unquoted = unquoted
 
+-- print(string.unquoted("test"))
+-- print(string.unquoted([["t\"est"]]))
+-- print(string.unquoted([["t\"est"x]]))
+-- print(string.unquoted("\'test\'"))
+-- print(string.unquoted('"test"'))
+-- print(string.unquoted('"test"'))
 
 function lpeg.anywhere(pattern) --slightly adapted from website
     return P { P(pattern) + 1 * V(1) } -- why so complex?
@@ -1667,6 +1680,7 @@ end
 
 
 
+
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
@@ -2937,12 +2951,19 @@ local nothing     = Cc("")
 local escaped     = (plus / " ") + (percent * C(hexdigit * hexdigit) / tochar)
 
 -- we assume schemes with more than 1 character (in order to avoid problems with windows disks)
+-- we also assume that when we have a scheme, we also have an authority
+
+local schemestr    = Cs((escaped+(1-colon-slash-qmark-hash))^2)
+local authoritystr = Cs((escaped+(1-      slash-qmark-hash))^0)
+local pathstr      = Cs((escaped+(1-            qmark-hash))^0)
+local querystr     = Cs((escaped+(1-                  hash))^0)
+local fragmentstr  = Cs((escaped+(1-           endofstring))^0)
 
-local scheme    =                 Cs((escaped+(1-colon-slash-qmark-hash))^2) * colon + nothing
-local authority = slash * slash * Cs((escaped+(1-      slash-qmark-hash))^0)         + nothing
-local path      = slash *         Cs((escaped+(1-            qmark-hash))^0)         + nothing
-local query     = qmark         * Cs((escaped+(1-                  hash))^0)         + nothing
-local fragment  = hash          * Cs((escaped+(1-           endofstring))^0)         + nothing
+local scheme    =                 schemestr    * colon + nothing
+local authority = slash * slash * authoritystr         + nothing
+local path      = slash         * pathstr              + nothing
+local query     = qmark         * querystr             + nothing
+local fragment  = hash          * fragmentstr          + nothing
 
 local validurl  = scheme * authority * path * query * fragment
 local parser    = Ct(validurl)
@@ -2963,11 +2984,14 @@ local function split(str)
     return (type(str) == "string" and lpegmatch(parser,str)) or str
 end
 
+local isscheme = schemestr * colon * slash * slash -- this test also assumes authority
+
 local function hasscheme(str)
-    local scheme = lpegmatch(scheme,str) -- at least one character
-    return scheme and scheme ~= ""
+    local scheme = lpegmatch(isscheme,str) -- at least one character
+    return scheme ~= "" and scheme or false
 end
 
+
 -- todo: cache them
 
 local rootletter       = R("az","AZ")
@@ -4815,6 +4839,23 @@ function inspect(i) -- global function
     end
 end
 
+-- from the lua book:
+
+function traceback()
+    local level = 1
+    while true do
+        local info = debug.getinfo(level, "Sl")
+        if not info then
+            break
+        elseif info.what == "C" then
+            print(format("%3i : C function",level))
+        else
+            print(format("%3i : [%s]:%d",level,info.short_src,info.currentline))
+        end
+        level = level + 1
+    end
+end
+
 
 end -- of closure
 
@@ -6821,9 +6862,7 @@ local function handle_any_entity(str)
     if resolve then
         local a = acache[str] -- per instance ! todo
         if not a then
-print(">1",str,a)
             a = resolve_predefined and predefined_simplified[str]
-print(">2",str,a)
             if a then
                 -- one of the predefined
             elseif type(resolve) == "function" then
@@ -6831,7 +6870,6 @@ print(">2",str,a)
             else
                 a = entities[str]
             end
-print(">3",str,a)
             if a then
 if type(a) == "function" then
     if trace_entities then
@@ -6839,9 +6877,7 @@ if type(a) == "function" then
     end
     a = a(str) or ""
 end
-print(">4",str,a)
                 a = lpegmatch(parsedentity,a) or a
-print(">5",str,a)
                 if trace_entities then
                     report_xml("resolved entity &%s; -> %s (internal)",str,a)
                 end
@@ -10156,19 +10192,6 @@ local resolvers = resolvers
 -- all, when working on the main resolver code, I don't want to scroll
 -- past this every time. See data-obs.lua for the gsub variant.
 
--- {a,b,c,d}
--- a,b,c/{p,q,r},d
--- a,b,c/{p,q,r}/d/{x,y,z}//
--- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
--- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
--- a{b,c}{d,e}f
--- {a,b,c,d}
--- {a,b,c/{p,q,r},d}
--- {a,b,c/{p,q,r}/d/{x,y,z}//}
--- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
--- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
--- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
-
 local function f_first(a,b)
     local t, n = { }, 0
     for s in gmatch(b,"[^,]+") do
@@ -10260,6 +10283,19 @@ function resolvers.expandedpathfromlist(pathlist)
     return newlist
 end
 
+-- {a,b,c,d}
+-- a,b,c/{p,q,r},d
+-- a,b,c/{p,q,r}/d/{x,y,z}//
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r}
+-- a{b,c}{d,e}f
+-- {a,b,c,d}
+-- {a,b,c/{p,q,r},d}
+-- {a,b,c/{p,q,r}/d/{x,y,z}//}
+-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}}
+-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}}
+-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}
+
 local cleanup = lpeg.replacer {
     { "!"  , ""  },
     { "\\" , "/" },
@@ -10427,9 +10463,21 @@ local function scan(files,spec,path,n,m,r)
     return files, n, m, r
 end
 
-function resolvers.scanfiles(path,branch)
+local cache = { }
+
+function resolvers.scanfiles(path,branch,usecache)
+    statistics.starttiming(cache)
+    if usecache then
+        local files = cache[path]
+        if files then
+            if trace_locating then
+                report_expansions("using caches scan of path '%s', branch '%s'",path,branch or path)
+            end
+            return files
+        end
+    end
     if trace_locating then
-        report_expansions("scanning path '%s', branch '%s'",path, branch or path)
+        report_expansions("scanning path '%s', branch '%s'",path,branch or path)
     end
     local realpath = resolvers.resolve(path) -- no shortcut
     local files, n, m, r = scan({ },realpath .. '/',"",0,0,0)
@@ -10440,9 +10488,18 @@ function resolvers.scanfiles(path,branch)
     if trace_locating then
         report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r)
     end
+    if usecache then
+        cache[path] = files
+    end
+    statistics.stoptiming(cache)
     return files
 end
 
+function resolvers.scantime()
+    return statistics.elapsedtime(cache)
+end
+
+
 
 
 end -- of closure
@@ -10831,7 +10888,7 @@ local function identify()
                     end
                 elseif not writable and caches.force then
                     local cacheparent = file.dirname(cachepath)
-                    if file.is_writable(cacheparent) then
+                    if file.is_writable(cacheparent) and true then -- we go on anyway (needed for mojca's kind of paths)
                         if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then
                             mkdirs(cachepath)
                             if isdir(cachepath) and file.is_writable(cachepath) then
@@ -11288,7 +11345,7 @@ if not modules then modules = { } end modules ['data-res'] = {
 -- instance but for practical purposes we now avoid this and use a
 -- instance variable. We always have one instance active (sort of global).
 
--- todo: cache:/// home:///
+-- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012)
 
 local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch
 local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys
@@ -11641,19 +11698,19 @@ local function load_configuration_files()
             if blob then
                 local setups = instance.setups
                 local data = blob()
-local parent = data and data.parent
-if parent then
-    local filename = filejoin(pathname,parent)
-    local realname = resolvers.resolve(filename) -- no shortcut
-    local blob = loadfile(realname)
-    if blob then
-        local parentdata = blob()
-        if parentdata then
-            report_resolving("loading configuration file '%s'",filename)
-            data = table.merged(parentdata,data)
-        end
-    end
-end
+                local parent = data and data.parent
+                if parent then
+                    local filename = filejoin(pathname,parent)
+                    local realname = resolvers.resolve(filename) -- no shortcut
+                    local blob = loadfile(realname)
+                    if blob then
+                        local parentdata = blob()
+                        if parentdata then
+                            report_resolving("loading configuration file '%s'",filename)
+                            data = table.merged(parentdata,data)
+                        end
+                    end
+                end
                 data = data and data.content
                 if data then
                     if trace_locating then
@@ -11743,14 +11800,14 @@ local function locate_file_databases()
                 local runtime = stripped == path
                 path = resolvers.cleanpath(path)
                 local spec = resolvers.splitmethod(stripped)
-                if spec.scheme == "cache" or spec.scheme == "file" then
-                    stripped = spec.path
-                elseif runtime and (spec.noscheme or spec.scheme == "file") then
+                if runtime and (spec.noscheme or spec.scheme == "file") then
                     stripped = "tree:///" .. stripped
+                elseif spec.scheme == "cache" or spec.scheme == "file" then
+                    stripped = spec.path
                 end
                 if trace_locating then
                     if runtime then
-                        report_resolving("locating list of '%s' (runtime)",path)
+                        report_resolving("locating list of '%s' (runtime) (%s)",path,stripped)
                     else
                         report_resolving("locating list of '%s' (cached)",path)
                     end
@@ -12162,287 +12219,402 @@ end
 
 local preparetreepattern = Cs((P(".")/"%%." + P("-")/"%%-" + P(1))^0 * Cc("$"))
 
--- this one is split in smaller functions but it needs testing
+-- -- -- begin of main file search routing -- -- -- needs checking as previous has been patched
 
-local function collect_instance_files(filename,askedformat,allresults) -- todo : plugin (scanners, checkers etc)
-    local result = { }
-    local stamp  = nil
-    askedformat = askedformat or ""
-    filename = collapsepath(filename)
-    -- speed up / beware: format problem
-    if instance.remember and not allresults then
-        stamp = filename .. "--" .. askedformat
-        if instance.found[stamp] then
-            if trace_locating then
-                report_resolving("remembered file '%s'",filename)
-            end
-            resolvers.registerintrees(filename) -- for tracing used files
-            return instance.found[stamp]
-        end
-    end
-    if not dangerous[askedformat] then
-        if isreadable(filename) then
-            if trace_detail then
-                report_resolving("file '%s' found directly",filename)
-            end
-            if stamp then
-                instance.found[stamp] = { filename }
-            end
-            return { filename }
+local collect_instance_files
+
+local function find_direct(filename,allresults)
+    if not dangerous[askedformat] and isreadable(filename) then
+        if trace_detail then
+            report_resolving("file '%s' found directly",filename)
         end
+        return { filename }
     end
+end
+
+local function find_wildcard(filename,allresults)
     if find(filename,'%*') then
         if trace_locating then
             report_resolving("checking wildcard '%s'", filename)
         end
-        result = resolvers.findwildcardfiles(filename) -- we can use th elocal
-    elseif file.is_qualified_path(filename) then
-        if isreadable(filename) then
-            if trace_locating then
-                report_resolving("qualified name '%s'", filename)
-            end
-            result = { filename }
-        else
-            local forcedname, ok, suffix = "", false, fileextname(filename)
-            if suffix == "" then -- why
-                local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat]
-                if format_suffixes then
-                    for i=1,#format_suffixes do
-                        local s = format_suffixes[i]
-                        forcedname = filename .. "." .. s
-                        if isreadable(forcedname) then
-                            if trace_locating then
-                                report_resolving("no suffix, forcing format filetype '%s'", s)
-                            end
-                            result, ok = { forcedname }, true
-                            break
-                        end
+        return resolvers.findwildcardfiles(filename) -- we can use the local
+    end
+end
+
+local function find_qualified(filename,allresults) -- this one will be split too
+    if not file.is_qualified_path(filename) then
+        return
+    end
+    if trace_locating then
+        report_resolving("checking qualified name '%s'", filename)
+    end
+    if isreadable(filename) then
+        if trace_detail then
+            report_resolving("qualified file '%s' found", filename)
+        end
+        return { filename }
+    end
+    if trace_detail then
+        report_resolving("locating qualified file '%s'", filename)
+    end
+    local forcedname, suffix = "", fileextname(filename)
+    if suffix == "" then -- why
+        local format_suffixes = askedformat == "" and resolvers.defaultsuffixes or suffixes[askedformat]
+        if format_suffixes then
+            for i=1,#format_suffixes do
+                local s = format_suffixes[i]
+                forcedname = filename .. "." .. s
+                if isreadable(forcedname) then
+                    if trace_locating then
+                        report_resolving("no suffix, forcing format filetype '%s'", s)
                     end
+                    return { forcedname }
                 end
             end
-            if not ok and suffix ~= "" then
-                -- try to find in tree (no suffix manipulation), here we search for the
-                -- matching last part of the name
-                local basename = filebasename(filename)
-                local pattern = lpegmatch(preparetreepattern,filename)
-                -- messy .. to be sorted out
-                local savedformat = askedformat
-                local format = savedformat or ""
-                if format == "" then
-                    askedformat = resolvers.formatofsuffix(suffix)
-                end
-                if not format then
-                    askedformat = "othertextfiles" -- kind of everything, maybe texinput is better
-                end
-                --
-                if basename ~= filename then
-                    local resolved = collect_instance_files(basename,askedformat,allresults)
-                    if #result == 0 then -- shouldn't this be resolved ?
-                        local lowered = lower(basename)
-                        if filename ~= lowered then
-                            resolved = collect_instance_files(lowered,askedformat,allresults)
-                        end
-                    end
-                    resolvers.format = savedformat
-                    --
-                    for r=1,#resolved do
-                        local rr = resolved[r]
-                        if find(rr,pattern) then
-                            result[#result+1], ok = rr, true
-                        end
+        end
+    end
+    if suffix ~= "" then
+        -- try to find in tree (no suffix manipulation), here we search for the
+        -- matching last part of the name
+        local basename = filebasename(filename)
+        local pattern = lpegmatch(preparetreepattern,filename)
+        -- messy .. to be sorted out
+        local savedformat = askedformat
+        local format = savedformat or ""
+        if format == "" then
+            askedformat = resolvers.formatofsuffix(suffix)
+        end
+        if not format then
+            askedformat = "othertextfiles" -- kind of everything, maybe all
+        end
+        --
+        if basename ~= filename then
+            local resolved = collect_instance_files(basename,askedformat,allresults)
+            if #resolved == 0 then
+                local lowered = lower(basename)
+                if filename ~= lowered then
+                    resolved = collect_instance_files(lowered,askedformat,allresults)
+                end
+            end
+            resolvers.format = savedformat
+            --
+            if #resolved > 0 then
+                local result = { }
+                for r=1,#resolved do
+                    local rr = resolved[r]
+                    if find(rr,pattern) then
+                        result[#result+1] = rr
                     end
                 end
-                -- a real wildcard:
-                --
-                -- if not ok then
-                --     local filelist = collect_files({basename})
-                --     for f=1,#filelist do
-                --         local ff = filelist[f][3] or ""
-                --         if find(ff,pattern) then
-                --             result[#result+1], ok = ff, true
-                --         end
-                --     end
-                -- end
-            end
-            if not ok and trace_locating then
-                report_resolving("qualified name '%s'", filename)
+                if #result > 0 then
+                    return result
+                end
             end
         end
-    else
-        -- search spec
-        local filetype, done, wantedfiles, ext = '', false, { }, fileextname(filename)
-        -- -- tricky as filename can be bla.1.2.3
-        -- if not suffixmap[ext] then --- probably needs to be done elsewhere too
-        --     wantedfiles[#wantedfiles+1] = filename
+        -- a real wildcard:
+        --
+        -- local filelist = collect_files({basename})
+        -- result = { }
+        -- for f=1,#filelist do
+        --     local ff = filelist[f][3] or ""
+        --     if find(ff,pattern) then
+        --         result[#result+1], ok = ff, true
+        --     end
         -- end
-        wantedfiles[#wantedfiles+1] = filename
-        if askedformat == "" then
-            if ext == "" or not suffixmap[ext] then
-                local defaultsuffixes = resolvers.defaultsuffixes
-                for i=1,#defaultsuffixes do
-                    local forcedname = filename .. '.' .. defaultsuffixes[i]
-                    wantedfiles[#wantedfiles+1] = forcedname
-                    filetype = resolvers.formatofsuffix(forcedname)
-                    if trace_locating then
-                        report_resolving("forcing filetype '%s'",filetype)
-                    end
-                end
-            else
-                filetype = resolvers.formatofsuffix(filename)
+        -- if #result > 0 then
+        --     return result
+        -- end
+    end
+end
+
+local function find_analyze(filename,askedformat,allresults)
+    local filetype, wantedfiles, ext = '', { }, fileextname(filename)
+    -- too tricky as filename can be bla.1.2.3:
+    --
+    -- if not suffixmap[ext] then
+    --     wantedfiles[#wantedfiles+1] = filename
+    -- end
+    wantedfiles[#wantedfiles+1] = filename
+    if askedformat == "" then
+        if ext == "" or not suffixmap[ext] then
+            local defaultsuffixes = resolvers.defaultsuffixes
+            for i=1,#defaultsuffixes do
+                local forcedname = filename .. '.' .. defaultsuffixes[i]
+                wantedfiles[#wantedfiles+1] = forcedname
+                filetype = resolvers.formatofsuffix(forcedname)
                 if trace_locating then
-                    report_resolving("using suffix based filetype '%s'",filetype)
+                    report_resolving("forcing filetype '%s'",filetype)
                 end
             end
         else
-            if ext == "" or not suffixmap[ext] then
-                local format_suffixes = suffixes[askedformat]
-                if format_suffixes then
-                    for i=1,#format_suffixes do
-                        wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i]
-                    end
-                end
-            end
-            filetype = askedformat
+            filetype = resolvers.formatofsuffix(filename)
             if trace_locating then
-                report_resolving("using given filetype '%s'",filetype)
+                report_resolving("using suffix based filetype '%s'",filetype)
             end
         end
-        local typespec = resolvers.variableofformat(filetype)
-        local pathlist = resolvers.expandedpathlist(typespec)
-        if not pathlist or #pathlist == 0 then
-            -- no pathlist, access check only / todo == wildcard
-            if trace_detail then
-                report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | "))
-            end
-            for k=1,#wantedfiles do
-                local fname = wantedfiles[k]
-                if fname and isreadable(fname) then
-                    filename, done = fname, true
-                    result[#result+1] = filejoin('.',fname)
-                    break
+    else
+        if ext == "" or not suffixmap[ext] then
+            local format_suffixes = suffixes[askedformat]
+            if format_suffixes then
+                for i=1,#format_suffixes do
+                    wantedfiles[#wantedfiles+1] = filename .. "." .. format_suffixes[i]
                 end
             end
-            -- this is actually 'other text files' or 'any' or 'whatever'
-            local filelist = collect_files(wantedfiles)
-            local fl = filelist and filelist[1]
-            if fl then
-                filename = fl[3]  -- not local?
-                result[#result+1] = resolvers.resolve(filename)
-                done = true
+        end
+        filetype = askedformat
+        if trace_locating then
+            report_resolving("using given filetype '%s'",filetype)
+        end
+    end
+    return filetype, wantedfiles
+end
+
+local function check_subpath(fname)
+    if isreadable(fname) then
+        if trace_detail then
+            report_resolving("found '%s' by deep scanning",fname)
+        end
+        return fname
+    end
+end
+
+local function find_intree(filename,filetype,wantedfiles,allresults)
+    local typespec = resolvers.variableofformat(filetype)
+    local pathlist = resolvers.expandedpathlist(typespec)
+    if pathlist and #pathlist > 0 then
+        -- list search
+        local filelist = collect_files(wantedfiles)
+        local dirlist = { }
+        if filelist then
+            for i=1,#filelist do
+                dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble
             end
-        else
-            -- list search
-            local filelist = collect_files(wantedfiles)
-            local dirlist = { }
+        end
+        if trace_detail then
+            report_resolving("checking filename '%s'",filename)
+        end
+        local result = { }
+        for k=1,#pathlist do
+            local path = pathlist[k]
+            local pathname = lpegmatch(inhibitstripper,path)
+            local doscan = path == pathname -- no ^!!
+            if not find (pathname,'//$') then
+                doscan = false -- we check directly on the path
+            end
+            local done = false
+            -- using file list
             if filelist then
-                for i=1,#filelist do
-                    dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble
-                end
-            end
-            if trace_detail then
-                report_resolving("checking filename '%s'",filename)
-            end
-            for k=1,#pathlist do
-                local path = pathlist[k]
-                local pathname = lpegmatch(inhibitstripper,path)
-                local doscan = path == pathname -- no ^!!
-                done = false
-                -- using file list
-                if filelist then
-                    -- compare list entries with permitted pattern -- /xx /xx//
-                    local expression = makepathexpression(pathname)
-                    if trace_detail then
-                        report_resolving("using pattern '%s' for path '%s'",expression,pathname)
-                    end
-                    for k=1,#filelist do
-                        local fl = filelist[k]
-                        local f = fl[2]
-                        local d = dirlist[k]
-                        if find(d,expression) then
-                            -- todo, test for readable
-                            result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut
-                            done = true
-                            if allresults then
-                                if trace_detail then
-                                    report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d)
-                                end
-                            else
-                                if trace_detail then
-                                    report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d)
-                                end
-                                break
+                -- compare list entries with permitted pattern -- /xx /xx//
+                local expression = makepathexpression(pathname)
+                if trace_detail then
+                    report_resolving("using pattern '%s' for path '%s'",expression,pathname)
+                end
+                for k=1,#filelist do
+                    local fl = filelist[k]
+                    local f = fl[2]
+                    local d = dirlist[k]
+                    if find(d,expression) then
+                        -- todo, test for readable
+                        result[#result+1] = resolvers.resolve(fl[3]) -- no shortcut
+                        done = true
+                        if allresults then
+                            if trace_detail then
+                                report_resolving("match to '%s' in hash for file '%s' and path '%s', continue scanning",expression,f,d)
+                            end
+                        else
+                            if trace_detail then
+                                report_resolving("match to '%s' in hash for file '%s' and path '%s', quit scanning",expression,f,d)
                             end
-                        elseif trace_detail then
-                            report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d)
+                            break
                         end
+                    elseif trace_detail then
+                        report_resolving("no match to '%s' in hash for file '%s' and path '%s'",expression,f,d)
                     end
                 end
-                if not done and doscan then
-                    -- check if on disk / unchecked / does not work at all / also zips
-                    local scheme = url.hasscheme(pathname)
-                    if not scheme or scheme == "file" then
-                        local pname = gsub(pathname,"%.%*$",'')
-                        if not find(pname,"%*") then
-                            local ppname = gsub(pname,"/+$","")
-                            if can_be_dir(ppname) then
+            end
+            if not done then
+                pathname = gsub(pathname,"/+$","")
+                pathname = resolvers.resolve(pathname)
+                local scheme = url.hasscheme(pathname)
+                if not scheme or scheme == "file" then
+                    local pname = gsub(pathname,"%.%*$",'')
+                    if not find(pname,"%*") then
+                        if can_be_dir(pname) then
+                            -- quick root scan first
+                            for k=1,#wantedfiles do
+                                local w = wantedfiles[k]
+                                local fname = check_subpath(filejoin(pname,w))
+                                if fname then
+                                    result[#result+1] = fname
+                                    done = true
+                                    if not allresults then
+                                        break
+                                    end
+                                end
+                            end
+                            if not done and doscan then
+                                -- collect files in path (and cache the result)
+                                local files = resolvers.scanfiles(pname,false,true)
                                 for k=1,#wantedfiles do
                                     local w = wantedfiles[k]
-                                    local fname = filejoin(ppname,w)
-                                    if isreadable(fname) then
-                                        if trace_detail then
-                                            report_resolving("found '%s' by scanning",fname)
+                                    local subpath = files[w]
+                                    if not subpath or subpath == "" then
+                                        -- rootscan already done
+                                    elseif type(subpath) == "string" then
+                                        local fname = check_subpath(filejoin(ppname,subpath,w))
+                                        if fname then
+                                            result[#result+1] = fname
+                                            done = true
+                                            if not allresults then
+                                                break
+                                            end
+                                        end
+                                    else
+                                        for i=1,#subpath do
+                                            local sp = subpath[i]
+                                            if sp == "" then
+                                                -- roottest already done
+                                            else
+                                                local fname = check_subpath(filejoin(ppname,sp,w))
+                                                if fname then
+                                                    result[#result+1] = fname
+                                                    done = true
+                                                    if not allresults then
+                                                        break
+                                                    end
+                                                end
+                                            end
+                                        end
+                                        if done and not allresults then
+                                            break
                                         end
-                                        result[#result+1] = fname
-                                        done = true
-                                        if not allresults then break end
                                     end
                                 end
-                            else
-                                -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
                             end
                         end
+                    else
+                        -- no access needed for non existing path, speedup (esp in large tree with lots of fake)
                     end
                 end
-                if not done and doscan then
-                    -- todo: slow path scanning ... although we now have tree:// supported in $TEXMF
-                end
-                if done and not allresults then break end
+            end
+            -- todo recursive scanning
+            if done and not allresults then
+                return #result > 0 and result
             end
         end
     end
-    for k=1,#result do
-        local rk = collapsepath(result[k])
-        result[k] = rk
-        resolvers.registerintrees(rk) -- for tracing used files
+end
+
+local function find_onpath(filename,filetype,wantedfiles,allresults)
+    if trace_detail then
+        report_resolving("checking filename '%s', filetype '%s', wanted files '%s'",filename, filetype or '?',concat(wantedfiles," | "))
     end
-    if stamp then
-        instance.found[stamp] = result
+    local result = { }
+    for k=1,#wantedfiles do
+        local fname = wantedfiles[k]
+        if fname and isreadable(fname) then
+            filename = fname
+            result[#result+1] = filejoin('.',fname)
+            if not allresults then
+                break
+            end
+        end
     end
-    return result
+    return #result > 0 and result
 end
 
--- -- -- begin of main file search routing -- -- --
-
-
-
-
-
-
-
-
+local function find_otherwise(filename,filetype,wantedfiles,allresults) -- other text files | any | whatever
+    local filelist = collect_files(wantedfiles)
+    local fl = filelist and filelist[1]
+    if fl then
+        return { resolvers.resolve(fl[3]) } -- filename
+    end
+end
 
+collect_instance_files = function(filename,askedformat,allresults) -- uses nested
+    local result, stamp, filetype, wantedfiles
+    askedformat = askedformat or ""
+    filename = collapsepath(filename)
+    if allresults then
+        -- no need for caching, only used for tracing
+        local filetype, wantedfiles = find_analyze(filename,askedformat)
+        local results = {
+            { method = "direct",    list = find_direct   (filename,stamp,true) },
+            { method = "wildcard",  list = find_wildcard (filename,true) },
+            { method = "qualified", list = find_qualified(filename,true) },
+            { method = "in tree",   list = find_intree   (filename,filetype,wantedfiles,true) },
+            { method = "on path",   list = find_onpath   (filename,filetype,wantedfiles,true) },
+            { method = "otherwise", list = find_otherwise(filename,filetype,wantedfiles,true) },
+        }
+        local result, status, done = { }, { }, { }
+        for k, r in next, results do
+            local method, list = r.method, r.list
+            if list then
+                for i=1,#list do
+                    local c = collapsepath(list[i])
+                    if not done[c] then
+                        result[#result+1] = c
+                        done[c] = true
+                    end
+                    status[#status+1] = format("%-10s: %s",method,c)
+                end
+            end
+        end
+        if trace_detail then
+            report_resolving("lookup status: %s",table.serialize(status,filename))
+        end
+        return result, status
+    else
+        if instance.remember then
+            stamp = format("%s--%s", filename, askedformat)
+            result = stamp and instance.found[stamp]
+            if result then
+                if trace_locating then
+                    report_resolving("remembered file '%s'",filename)
+                end
+                return result
+            end
+        end
+        result = find_direct   (filename,stamp) or
+                 find_wildcard (filename)       or
+                 find_qualified(filename)
+        if not result then
+            local filetype, wantedfiles = find_analyze(filename,askedformat)
+            result = find_intree   (filename,filetype,wantedfiles) or
+                     find_onpath   (filename,filetype,wantedfiles) or
+                     find_otherwise(filename,filetype,wantedfiles)
+        end
+        if result then
+            for k=1,#result do
+                local rk = collapsepath(result[k])
+                result[k] = rk
+                resolvers.registerintrees(rk) -- for tracing used files
+            end
+        else
+            result = { } -- maybe false
+        end
+        if stamp then
+            if trace_locating then
+                report_resolving("remembering file '%s'",filename)
+            end
+            instance.found[stamp] = result
+        end
+        return result
+    end
+end
 
 -- -- -- end of main file search routing -- -- --
 
+
 local function findfiles(filename,filetype,allresults)
-    local result = collect_instance_files(filename,filetype or "",allresults)
-    if #result == 0 then
+    local result, status = collect_instance_files(filename,filetype or "",allresults)
+    if not result or #result == 0 then
         local lowered = lower(filename)
         if filename ~= lowered then
-            return collect_instance_files(lowered,filetype or "",allresults)
+            result, status = collect_instance_files(lowered,filetype or "",allresults)
         end
     end
-    return result
+    return result or { }, status
 end
 
 function resolvers.findfiles(filename,filetype)
@@ -12603,6 +12775,10 @@ function resolvers.load(option)
     return files and next(files) and true
 end
 
+function resolvers.loadtime()
+    return statistics.elapsedtime(instance)
+end
+
 local function report(str)
     if trace_locating then
         report_resolving(str) -- has already verbose
@@ -12616,6 +12792,9 @@ function resolvers.dowithfilesandreport(command, files, ...) -- will move
         if trace_locating then
             report('') -- ?
         end
+        if type(files) == "string" then
+            files = { files }
+        end
         for f=1,#files do
             local file = files[f]
             local result = command(file,...)
@@ -13031,7 +13210,7 @@ end
 
 function generators.file(specification)
     local path = specification.filename
-    local content = resolvers.scanfiles(path)
+    local content = resolvers.scanfiles(path,false,true) -- scan once
     resolvers.registerfilehash(path,content,true)
 end
 
@@ -13654,9 +13833,10 @@ end
 
 function resolvers.locators.tree(specification)
     local name = specification.filename
-    if name ~= '' and lfs.isdir(name) then
+    local realname = resolvers.resolve(name) -- no shortcut
+    if realname and realname ~= '' and lfs.isdir(realname) then
         if trace_locating then
-            report_trees("locator '%s' found",name)
+            report_trees("locator '%s' found",realname)
         end
         resolvers.appendhash('tree',name,false) -- don't cache
     elseif trace_locating then
@@ -13670,6 +13850,8 @@ function resolvers.hashers.tree(specification)
         report_trees("analysing '%s'",name)
     end
     resolvers.methodhandler("hashers",name)
+
+    resolvers.generators.file(specification)
 end
 
 resolvers.concatinators.tree = resolvers.concatinators.file
@@ -14596,7 +14778,7 @@ local helpinfo = [[
 --resolve             resolve prefixed arguments
 --ctxlua              run internally (using preloaded libs)
 --internal            run script using built in libraries (same as --ctxlua)
---locate              locate given filename
+--locate              locate given filename in database (default) or system (--first --all --detail)
 
 --autotree            use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree'
 --tree=pathtotree     use given texmf tree (default file: 'setuptex.tmf')
@@ -14789,6 +14971,10 @@ function runners.execute_script(fullname,internal,nosplit)
                     environment.ownscript = result
                     dofile(result)
                 else
+                    result = string.quoted(string.unquoted(result))
+                 -- if string.match(result,' ') and not string.match(result,"^\".*\"$") then
+                 --     result = '"' .. result .. '"'
+                 -- end
                     local binary = runners.applications[file.extname(result)]
                     if binary and binary ~= "" then
                         result = binary .. " " .. result
@@ -14899,10 +15085,24 @@ function runners.resolve_string(filename)
     end
 end
 
-function runners.locate_file(filename)
-    -- differs from texmfstart where locate appends .com .exe .bat ... todo
+-- differs from texmfstart where locate appends .com .exe .bat ... todo
+
+function runners.locate_file(filename) -- was given file but only searches in tree
     if filename and filename ~= "" then
-        runners.report_location(resolvers.findgivenfile(filename))
+        if environment.argument("first") then
+            runners.report_location(resolvers.findfile(filename))
+         -- resolvers.dowithfilesandreport(resolvers.findfile,filename)
+        elseif environment.argument("all") then
+            local result, status = resolvers.findfiles(filename)
+            if status and environment.argument("detail") then
+                runners.report_location(status)
+            else
+                runners.report_location(result)
+            end
+        else
+            runners.report_location(resolvers.findgivenfile(filename))
+         -- resolvers.dowithfilesandreport(resolvers.findgivenfile,filename)
+        end
     end
 end
 
@@ -14911,12 +15111,12 @@ function runners.locate_platform()
 end
 
 function runners.report_location(result)
-    if e_verbose then
-        reportline()
-        if result and result ~= "" then
-            report(result)
-        else
-            report("not found")
+    if type(result) == "table" then
+        for i=1,#result do
+            if i > 1 then
+                io.write("\n")
+            end
+            io.write(result[i])
         end
     else
         io.write(result)
@@ -14974,8 +15174,8 @@ function resolvers.launch(str)
 end
 
 function runners.launch_file(filename)
-    instance.allresults = true
     trackers.enable("resolvers.locating")
+    local allresults = environment.arguments["all"]
     local pattern = environment.arguments["pattern"]
     if not pattern or pattern == "" then
         pattern = filename
@@ -14983,15 +15183,15 @@ function runners.launch_file(filename)
     if not pattern or pattern == "" then
         report("provide name or --pattern=")
     else
-        local t = resolvers.findfiles(pattern)
+        local t = resolvers.findfiles(pattern,nil,allresults)
         if not t or #t == 0 then
-            t = resolvers.findfiles("*/" .. pattern)
+            t = resolvers.findfiles("*/" .. pattern,nil,allresults)
         end
         if not t or #t == 0 then
-            t = resolvers.findfiles("*/" .. pattern .. "*")
+            t = resolvers.findfiles("*/" .. pattern .. "*",nil,allresults)
         end
         if t and #t > 0 then
-            if environment.arguments["all"] then
+            if allresults then
                 for _, v in pairs(t) do
                     report("launching %s", v)
                     resolvers.launch(v)
@@ -15367,7 +15567,7 @@ elseif environment.argument("resolve") then
 
 elseif environment.argument("locate") then
 
-    -- locate file
+    -- locate file (only database)
 
     runners.loadbase()
     runners.locate_file(filename)
@@ -15410,15 +15610,16 @@ elseif environment.argument("find-file") then
     -- luatools: runners.execute_ctx_script("mtx-base","--find-file",filename)
 
     resolvers.load()
+    local e_all     = environment.argument("all")
     local e_pattern = environment.argument("pattern")
-    local e_format  = environment.arguments("format")
+    local e_format  = environment.argument("format")
+    local finder    = e_all and resolvers.findfiles or resolvers.findfile
     if not e_pattern then
         runners.register_arguments(filename)
         environment.initializearguments(environment.arguments_after)
-        resolvers.dowithfilesandreport(resolvers.findfiles,environment.files,e_format)
+        resolvers.dowithfilesandreport(finder,environment.files,e_format)
     elseif type(e_pattern) == "string" then
-        instance.allresults = true -- brrrr
-        resolvers.dowithfilesandreport(resolvers.findfiles,{ e_pattern }, e_format)
+        resolvers.dowithfilesandreport(finder,{ e_pattern },e_format)
     end
 
 elseif environment.argument("find-path") then
@@ -15499,6 +15700,8 @@ elseif environment.argument("generate") then
     trackers.enable("resolvers.locating")
     resolvers.load()
 
+    e_verbose = true
+
 elseif environment.argument("make") or environment.argument("ini") or environment.argument("compile") then
 
     -- luatools: runners.execute_ctx_script("mtx-base","--make",filename)