Merge pull request #96 from phi-gamma/master

db rewrite + fixes (``texmf-fonts`` branch)

7 files changed, 595 insertions, 213 deletions

diff --git a/luaotfload-auxiliary.lua b/luaotfload-auxiliary.lua
index bb1e8f9..3597683 100644
--- a/luaotfload-auxiliary.lua
+++ b/luaotfload-auxiliary.lua
@@ -42,6 +42,21 @@ local tablecopy     = table.copy
 ---                          font patches
 -----------------------------------------------------------------------
 
+--- https://github.com/khaledhosny/luaotfload/issues/54
+
+local rewrite_fontname = function (tfmdata, specification)
+  tfmdata.name = [["]] .. specification .. [["]]
+end
+
+luatexbase.add_to_callback(
+  "luaotfload.patch_font",
+  rewrite_fontname,
+  "luaotfload.rewrite_fontname")
+
+--- as of 2.3 the compatibility hacks for TL 2013 are made optional
+
+if config.luaotfload.compatibility == true then
+
 --[[doc--
 
 The font object (tfmdata) structure has changed since version 1.x, so
@@ -101,13 +116,10 @@ local add_fontdata_fallbacks = function (fontdata)
   return fontdata
 end
 
---if config.luaotfload.compatibility == true then
-if true then
-  luatexbase.add_to_callback(
-    "luaotfload.patch_font",
-    add_fontdata_fallbacks,
-    "luaotfload.fontdata_fallbacks")
-end
+luatexbase.add_to_callback(
+  "luaotfload.patch_font",
+  add_fontdata_fallbacks,
+  "luaotfload.fontdata_fallbacks")
 
 --[[doc--
 
@@ -124,6 +136,8 @@ font.getfont() since Hans made it a harmless wrapper [1].)
 fonts.identifiers = fonts.hashes.identifiers
 fonts.ids         = fonts.hashes.identifiers
 
+end
+
 --[[doc--
 This sets two dimensions apparently relied upon by the unicode-math
 package.
diff --git a/luaotfload-database.lua b/luaotfload-database.lua
index 60e321f..b9695ad 100644
--- a/luaotfload-database.lua
+++ b/luaotfload-database.lua
@@ -1,5 +1,5 @@
 if not modules then modules = { } end modules ['luaotfload-database'] = {
-    version   = 2.2,
+    version   = 2.3,
     comment   = "companion to luaotfload.lua",
     author    = "Khaled Hosny, Elie Roux, Philipp Gesang",
     copyright = "Luaotfload Development Team",
@@ -11,14 +11,24 @@ if not modules then modules = { } end modules ['luaotfload-database'] = {
 --- difficulty with the filenames of the TEXMF tree that are referenced as
 --- relative paths...
 
+local lpeg = require "lpeg"
+
+local P, R, S, lpegmatch
+    = lpeg.P, lpeg.R, lpeg.S, lpeg.match
+
+local C, Cc, Cf, Cg, Ct
+    = lpeg.C, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Ct
+
 --- Luatex builtins
 local load                    = load
 local next                    = next
 local pcall                   = pcall
 local require                 = require
 local tonumber                = tonumber
+local unpack                  = table.unpack
 
 local fontloaderinfo          = fontloader.info
+local fontloaderopen          = fontloader.open
 local iolines                 = io.lines
 local ioopen                  = io.open
 local kpseexpand_path         = kpse.expand_path
@@ -27,6 +37,7 @@ local kpselookup              = kpse.lookup
 local kpsereadable_file       = kpse.readable_file
 local lfsisdir                = lfs.isdir
 local lfsisfile               = lfs.isfile
+local lfsattributes           = lfs.attributes
 local mathabs                 = math.abs
 local mathmin                 = math.min
 local stringfind              = string.find
@@ -71,7 +82,7 @@ fonts.definers       = fonts.definers or { }
 
 local names          = fonts.names
 
-names.version        = 2.204
+names.version        = 2.207
 names.data           = nil      --- contains the loaded database
 names.lookups        = nil      --- contains the lookup cache
 names.path           = {
@@ -131,28 +142,41 @@ This is a sketch of the luaotfload db:
         mappings        : fontentry list;
         status          : filestatus;
         version         : float;
-        // preliminary additions of v2.2:
-        basenames       : (string, int) hash;    // where int is the index in mappings
-        barenames       : (string, int) hash;    // where int is the index in mappings
+        // new in v2.3; these supersede the basenames / barenames
+        // hashes from v2.2
+        filenames       : filemap;
+    }
+    and filemap = {
+        base : (string, int) hash; // basename -> idx
+        bare : (string, int) hash; // barename -> idx
+        full : (int, string) hash; // idx -> full path
     }
     and fontentry = {
+        barename    : string;
         familyname  : string;
-        filename    : (string * int);            // int: subfont
-        fontname    : string;
-        fullname    : string;
-        names       : {
-            family     : string;
-            fullname   : string;
-            psname     : string;
-            subfamily  : string;
-        }
+        filename    : string;
+        fontname    : string; // <- metadata
+        fullname    : string; // <- metadata
+        sanitized   : {
+            family         : string;
+            fontname       : string; // <- metadata
+            fullname       : string; // <- namedata.names
+            metafamily     : string;
+            pfullname      : string;
+            prefmodifiers  : string;
+            psname         : string;
+            subfamily      : string;
+        };
         size         : int list;
         slant        : int;
+        subfont      : int;
+        texmf        : bool;
         weight       : int;
         width        : int;
-        units_per_em : int;                      // mainly 1000, but also 2048 or 256
+        units_per_em : int;        // mainly 1000, but also 2048 or 256
     }
-    and filestatus = (fullname, { index : int list; timestamp : int }) dict
+    and filestatus = (fullname,
+                      { index : int list; timestamp : int }) dict
 
 beware that this is a reconstruction and may be incomplete.
 
@@ -195,9 +219,7 @@ local fontnames_init = function (keep_cache) --- returns dbobj
     return {
         mappings        = { },
         status          = { },
-        barenames       = { },
-        basenames       = { },
---      fullnames       = { }, // -> status
+--      filenames       = { }, -- created later
         version         = names.version,
     }
 end
@@ -245,10 +267,12 @@ local flush_lookup_cache
 local font_fullinfo
 local load_names
 local load_lookups
+local read_blacklist
 local read_fonts_conf
 local reload_db
 local resolve
 local resolve_cached
+local resolve_fullpath
 local save_names
 local save_lookups
 local update_names
@@ -272,8 +296,9 @@ load_names = function (dry_run)
                                 1000*(os.gettimeofday()-starttime))
     else
         report("both", 0, "db",
-            [[Font names database not found, generating new one.
-             This can take several minutes; please be patient.]])
+            [[Font names database not found, generating new one.]])
+        report("both", 0, "db",
+             [[This can take several minutes; please be patient.]])
         data = update_names(fontnames_init(false), nil, dry_run)
         local success = save_names(data)
         if not success then
@@ -321,32 +346,47 @@ end
 
 local type1_formats = { "tfm", "ofm", }
 
---- string -> string
+local dummy_findfile = resolvers.findfile -- from basics-gen
+
+--- filemap -> string -> string -> (string | bool)
+local verbose_lookup = function (data, kind, filename)
+    local found = data[kind][filename]
+    if found ~= nil then
+        found = data.full[found]
+        if found == nil then --> texmf
+            report("info", 0, "db",
+                "crude file lookup: req=%s; hit=%s => kpse",
+                filename, kind)
+            found = dummy_findfile(filename)
+        else
+            report("info", 0, "db",
+                "crude file lookup: req=%s; hit=%s; ret=%s",
+                filename, kind, found)
+        end
+        return found
+    end
+    return false
+end
+
+--- string -> (string | string * string)
 crude_file_lookup_verbose = function (filename)
     if not names.data then names.data = load_names() end
     local data      = names.data
     local mappings  = data.mappings
+    local filenames = data.filenames
     local found
 
     --- look up in db first ...
-    found = data.barenames[filename]
-    if found and mappings[found] then
-        found = mappings[found].filename[1]
-        report("info", 0, "db",
-            "crude file lookup: req=%s; hit=bare; ret=%s",
-            filename, found)
+    found = verbose_lookup(filenames, "bare", filename)
+    if found then
         return found
     end
-    found = data.basenames[filename]
-    if found and mappings[found] then
-        found = mappings[found].filename[1]
-        report("info", 0, "db",
-            "crude file lookup: req=%s; hit=base; ret=%s",
-            filename, found)
+    found = verbose_lookup(filenames, "base", filename)
+    if found then
         return found
     end
 
-    --- ofm and tfm
+    --- ofm and tfm, returns pair
     for i=1, #type1_formats do
         local format = type1_formats[i]
         if resolvers.findfile(filename, format) then
@@ -356,16 +396,24 @@ crude_file_lookup_verbose = function (filename)
     return filename, nil
 end
 
---- string -> string
+--- string -> (string | string * string)
 crude_file_lookup = function (filename)
     if not names.data then names.data = load_names() end
     local data      = names.data
     local mappings  = data.mappings
-    local found = data.barenames[filename]
-               or data.basenames[filename]
+    local filenames = data.filenames
+
+    local found
+
+    found = filenames.base[filename]
+         or filenames.bare[filename]
+
     if found then
-        found = data.mappings[found]
-        if found then return found.filename[1] end
+        found = filenames.full[found]
+        if found == nil then
+            found = dummy_findfile(filename)
+        end
+        return found or filename
     end
     for i=1, #type1_formats do
         local format = type1_formats[i]
@@ -476,6 +524,26 @@ local add_to_match = function (
 end
 
 --[[doc--
+Existence of the resolved file name is verified differently depending
+on whether the index entry has a texmf flag set.
+--doc]]--
+
+local get_font_file = function (fullnames, entry)
+    local basename = entry.basename
+    if entry.texmf == true then
+        if kpselookup(basename) then
+            return true, basename, entry.subfont
+        end
+    else
+        local fullname = fullnames[entry.index]
+        if lfsisfile(fullname) then
+            return true, basename, entry.subfont
+        end
+    end
+    return false
+end
+
+--[[doc--
 
 Luatex-fonts, the font-loader package luaotfload imports, comes with
 basic file location facilities (see luatex-fonts-syn.lua).
@@ -494,7 +562,7 @@ the font database created by the luaotfload-tool script.
 ---     · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig }
 ---     · ???
 ---   · forced:   string
----   · lookup:   "name" | "file"
+---   · lookup:   "name"
 ---   · method:   string
 ---   · name:     string
 ---   · resolved: string
@@ -546,30 +614,41 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con
     if db_version ~= nms_version then
         report("log", 0, "db",
             [[version mismatch; expected %4.3f, got %4.3f]],
-            nms_version, db_version
-        )
-        return reload_db("version mismatch", resolve, nil, nil, specification)
+            nms_version, db_version)
+        if not fonts_reloaded then
+            return reload_db("version mismatch",
+                             resolve, nil, nil, specification)
+        end
+        return specification.name, false, false
     end
 
     if not data.mappings then
-        return reload_db("invalid database; missing font mapping",
-                         resolve, nil, nil, specification
-               )
+        if not fonts_reloaded then
+            return reload_db("invalid database; missing font mapping",
+                             resolve, nil, nil, specification)
+        end
+        return specification.name, false, false
     end
 
-    local found = { }
+    local found      = { } --> collect results
+    local fallback         --> e.g. non-matching style (fontspec is anal about this)
+    local candidates = { } --> secondary results, incomplete matches
+
     local synonym_set = style_synonyms.set
-    for _, face in next, data.mappings do
-        local family, subfamily, fullname, psname, fontname, pfullname
+    for n, face in next, data.mappings do
+        local family, subfamily, fullname, prefmodifiers
+        local psname, fontname, pfullname, metafamily
 
         local facenames = face.sanitized
         if facenames then
-            family      = facenames.family
-            subfamily   = facenames.subfamily
-            fullname    = facenames.fullname
-            psname      = facenames.psname
-            fontname    = facenames.fontname
-            pfullname   = facenames.pfullname
+            family          = facenames.family
+            subfamily       = facenames.subfamily
+            prefmodifiers   = facenames.prefmodifiers
+            fullname        = facenames.fullname
+            psname          = facenames.psname
+            fontname        = facenames.fontname
+            pfullname       = facenames.pfullname
+            metafamily      = facenames.metafamily
         end
         fontname    = fontname  or sanitize_string(face.fontname)
         pfullname   = pfullname or sanitize_string(face.fullname)
@@ -583,24 +662,28 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con
             minsize = optsize[3] and optsize[3] / 10 or dsnsize
         end
 
-        if name == family then
-            if subfamily == style then
-                local continue
-                found, continue = add_to_match(
-                    found,   optsize, dsnsize, size,
-                    minsize, maxsize, face)
-                if continue == false then break end
-            elseif synonym_set[style] and
-                   synonym_set[style][subfamily]
+        if     name == family
+            or name == metafamily
+        then
+            if     style == prefmodifiers
+                or style == subfamily
+                or synonym_set[style] and
+                    (synonym_set[style][prefmodifiers] or
+                     synonym_set[style][subfamily])
             then
                 local continue
                 found, continue = add_to_match(
                     found,   optsize, dsnsize, size,
                     minsize, maxsize, face)
                 if continue == false then break end
-            elseif subfamily == "regular" or
-                   synonym_set.regular[subfamily] then
-                found.fallback = face
+
+            elseif prefmodifiers == "regular"
+                or subfamily     == "regular"
+                --- TODO this match should be performed when building the db
+                or synonym_set.regular[prefmodifiers]
+                or synonym_set.regular[subfamily]
+            then
+                fallback = face
             elseif name == fullname
                 or name == pfullname
                 or name == fontname
@@ -611,6 +694,8 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con
                     found,   optsize, dsnsize, size,
                     minsize, maxsize, face)
                 if continue == false then break end
+            else --- mark as last straw but continue
+                candidates[#candidates+1] = face
             end
         else
             if name == fullname
@@ -626,15 +711,18 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con
         end
     end
 
+    --- this is a monster
     if #found == 1 then
         --- “found” is really synonymous with “registered in the db”.
-        local filename = found[1].filename[1]
-        if lfsisfile(filename) or kpselookup(filename) then
+        local entry = found[1]
+        local success, filename, subfont
+            = get_font_file(data.filenames.full, entry)
+        if success == true then
             report("log", 0, "resolve",
                 "font family='%s', subfamily='%s' found: %s",
                 name, style, filename
             )
-            return filename, found[1].filename[2], true
+            return filename, subfont, true
         end
     elseif #found > 1 then
         -- we found matching font(s) but not in the requested optical
@@ -650,18 +738,41 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con
                 least   = difference
             end
         end
-        local filename = closest.filename[1]
-        if lfsisfile(filename) or kpselookup(filename) then
+        local success, filename, subfont
+            = get_font_file(data.filenames.full, closest)
+        if success == true then
             report("log", 0, "resolve",
                 "font family='%s', subfamily='%s' found: %s",
                 name, style, filename
             )
-            return filename, closest.filename[2], true
+            return filename, subfont, true
+        end
+    elseif fallback then
+        local success, filename, subfont
+            = get_font_file(data.filenames.full, fallback)
+        if success == true then
+            report("log", 0, "resolve",
+                "no exact match for request %s; using fallback",
+                specification.specification
+            )
+            report("log", 0, "resolve",
+                "font family='%s', subfamily='%s' found: %s",
+                name, style, filename
+            )
+            return filename, subfont, true
+        end
+    elseif next(candidates) then
+        --- pick the first candidate encountered
+        local entry = candidates[1]
+        local success, filename, subfont
+            = get_font_file(data.filenames.full, entry)
+        if success == true then
+            report("log", 0, "resolve",
+                "font family='%s', subfamily='%s' found: %s",
+                name, style, filename
+            )
+            return filename, subfont, true
         end
-    elseif found.fallback then
-        return found.fallback.filename[1],
-               found.fallback.filename[2],
-               true
     end
 
     --- no font found so far
@@ -673,17 +784,30 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con
         )
     end
 
-    --- else, fallback to requested name
+    --- else, default to requested name
     return specification.name, false, false
 end --- resolve()
 
+resolve_fullpath = function (fontname, ext) --- getfilename()
+    if not fonts_loaded then
+        names.data = load_names()
+    end
+    local filenames = names.data.filenames
+    local idx = filenames.base[fontname]
+             or filenames.bare[fontname]
+    if idx then
+        return filenames.full[idx]
+    end
+    return ""
+end
+
 --- when reload is triggered we update the database
 --- and then re-run the caller with the arg list
 
 --- string -> ('a -> 'a) -> 'a list -> 'a
 reload_db = function (why, caller, ...)
     report("both", 1, "db", "reload initiated; reason: “%s”", why)
-    names.data = update_names()
+    names.data = update_names(names.data, false, false)
     local success = save_names()
     if success then
         fonts_reloaded = true
@@ -730,7 +854,10 @@ find_closest = function (name, limit)
     local data = names.data
 
     if type(data) ~= "table" then
-        return reload_db("no database", find_closest, name)
+        if not fonts_reloaded then
+            return reload_db("no database", find_closest, name)
+        end
+        return false
     end
     local by_distance   = { } --- (int, string list) dict
     local distances     = { } --- int list
@@ -752,7 +879,6 @@ find_closest = function (name, limit)
         --]]
         if cnames then
             local fullname, family = cnames.fullname, cnames.family
-            family = sanitize_string(family)
 
             local dist = cached[family]--- maybe already calculated
             if not dist then
@@ -805,9 +931,10 @@ The data inside an Opentype font file can be quite heterogeneous.
 Thus in order to get the relevant information, parts of the original
 table as returned by the font file reader need to be relocated.
 --doc]]--
-font_fullinfo = function (filename, subfont)
+--- string -> int -> bool -> string -> fontentry
+font_fullinfo = function (filename, subfont, texmf, basename)
     local tfmdata = { }
-    local rawfont = fontloader.open(filename, subfont)
+    local rawfont = fontloaderopen(filename, subfont)
     if not rawfont then
         report("log", 1, "error", "failed to open %s", filename)
         return
@@ -829,18 +956,19 @@ font_fullinfo = function (filename, subfont)
                 local names = {
                     --- see
                     --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html
-                    fullname  = namedata.names.compatfull
-                             or namedata.names.fullname,
-                    family    = namedata.names.preffamilyname
-                             or namedata.names.family,
-                    subfamily = tfmdata.fontstyle_name
-                             or namedata.names.prefmodifiers
-                             or namedata.names.subfamily,
-                    psname    = namedata.names.postscriptname,
-                    pfullname = metadata.fullname,
-                    fontname  = metadata.fontname,
+                    fullname      = namedata.names.compatfull
+                                 or namedata.names.fullname,
+                    family        = namedata.names.preffamilyname
+                                 or namedata.names.family,
+                    prefmodifiers = namedata.names.prefmodifiers,
+                    subfamily     = tfmdata.fontstyle_name
+                                 or namedata.names.subfamily,
+                    psname        = namedata.names.postscriptname,
+                    pfullname     = metadata.fullname,
+                    fontname      = metadata.fontname,
+                    metafamily    = metadata.familyname,
                 }
-                tfmdata.names     = names
+--              tfmdata.names     = names
                 tfmdata.sanitized = sanitize_names(names)
             end
         end
@@ -852,7 +980,6 @@ font_fullinfo = function (filename, subfont)
     tfmdata.fontname      = metadata.fontname
     tfmdata.fullname      = metadata.fullname
     tfmdata.familyname    = metadata.familyname
-    tfmdata.filename      = { filename, subfont } -- always store full path
     tfmdata.weight        = metadata.pfminfo.weight
     tfmdata.width         = metadata.pfminfo.width
     tfmdata.slant         = metadata.italicangle
@@ -865,33 +992,36 @@ font_fullinfo = function (filename, subfont)
         metadata.design_range_top    ~= 0 and metadata.design_range_top    or nil,
         metadata.design_range_bottom ~= 0 and metadata.design_range_bottom or nil,
     }
+
+    --- file location data (used to be filename field)
+    tfmdata.filename      = filename --> sys
+    tfmdata.basename      = basename --> texmf
+    tfmdata.texmf         = texmf or false
+    tfmdata.subfont       = subfont
+
     return tfmdata
 end
 
 --- we return true if the fond is new or re-indexed
 --- string -> dbobj -> dbobj -> bool
-local load_font = function (fullname, fontnames, newfontnames)
+local load_font = function (fullname, fontnames, newfontnames, texmf)
     if not fullname then
         return false
     end
 
     local newmappings   = newfontnames.mappings
-    local newstatus     = newfontnames.status
-
---  local newfullnames  = newfontnames.fullnames
-    local newbasenames  = newfontnames.basenames
-    local newbarenames  = newfontnames.barenames
+    local newstatus     = newfontnames.status --- by full path
 
     local mappings      = fontnames.mappings
     local status        = fontnames.status
---  local fullnames     = fontnames.fullnames
-    local basenames     = fontnames.basenames
-    local barenames     = fontnames.barenames
 
     local basename      = filebasename(fullname)
     local barename      = filenameonly(fullname)
 
-    local entryname     = basename
+    local entryname     = fullname
+    if texmf == true then
+        entryname = basename
+    end
 
     if names.blacklist[fullname] or names.blacklist[basename]
     then
@@ -900,37 +1030,34 @@ local load_font = function (fullname, fontnames, newfontnames)
         return false
     end
 
-    local timestamp, db_timestamp
-    db_timestamp        = status[fullname]
-                        and status[fullname].timestamp
-    timestamp           = lfs.attributes(fullname, "modification")
+    local new_timestamp, current_timestamp
+    current_timestamp   = status[fullname]
+                      and status[fullname].timestamp
+    new_timestamp       = lfsattributes(fullname, "modification")
 
-    local index_status = newstatus[fullname]
-    --- index_status: nil | false | table
-    if index_status and index_status.timestamp == timestamp then
+    local newentrystatus = newstatus[fullname]
+    --- newentrystatus: nil | false | table
+    if newentrystatus and newentrystatus.timestamp == new_timestamp then
         -- already indexed this run
         return false
     end
 
-    newstatus[fullname]           = newstatus[fullname] or { }
-    newstatus[fullname].timestamp = timestamp
-    newstatus[fullname].index     = newstatus[fullname].index or { }
+    newstatus[fullname]      = newentrystatus or { }
+    local newentrystatus     = newstatus[fullname]
+    newentrystatus.timestamp = new_timestamp
+    newentrystatus.index     = newentrystatus.index or { }
 
-    --- this test compares the modification data registered
-    --- in the database with the current one
-    if  db_timestamp == timestamp
-    and not newstatus[fullname].index[1] then
-        for _,v in next, status[fullname].index do
-            local index      = #newstatus[fullname].index
+    if  current_timestamp == new_timestamp
+    and not newentrystatus.index[1]
+    then
+        for _, v in next, status[fullname].index do
+            local index      = #newentrystatus.index
             local fullinfo   = mappings[v]
             local location   = #newmappings + 1
-            newmappings[location]               = fullinfo --- keep
-            newstatus[fullname].index[index+1]  = location --- is this actually used anywhere?
---          newfullnames[fullname]              = location
-            newbasenames[basename]              = location
-            newbarenames[barename]              = location
+            newmappings[location]          = fullinfo --- keep
+            newentrystatus.index[index+1]  = location --- is this actually used anywhere?
         end
-        report("log", 2, "db", "font “%s” already indexed", entryname)
+        report("log", 2, "db", "font “%s” already indexed", basename)
         return false
     end
 
@@ -938,38 +1065,32 @@ local load_font = function (fullname, fontnames, newfontnames)
     if info then
         if type(info) == "table" and #info > 1 then --- ttc
             for n_font = 1, #info do
-                local fullinfo = font_fullinfo(fullname, n_font-1)
+                local fullinfo = font_fullinfo(fullname, n_font-1, texmf, basename)
                 if not fullinfo then
                     return false
                 end
                 local location = #newmappings+1
-                local index    = newstatus[fullname].index[n_font]
+                local index    = newentrystatus.index[n_font]
                 if not index then index = location end
 
-                newmappings[index]                  = fullinfo
---              newfullnames[fullname]              = location
-                newbasenames[basename]              = location
-                newbarenames[barename]              = location
-                newstatus[fullname].index[n_font]   = index
+                newmappings[index]            = fullinfo
+                newentrystatus.index[n_font]  = index
             end
         else
-            local fullinfo = font_fullinfo(fullname, false)
+            local fullinfo = font_fullinfo(fullname, false, texmf, basename)
             if not fullinfo then
                 return false
             end
             local location  = #newmappings+1
-            local index     = newstatus[fullname].index[1]
+            local index     = newentrystatus.index[1]
             if not index then index = location end
 
-            newmappings[index]            = fullinfo
---          newfullnames[fullname]        = location
-            newbasenames[basename]        = location
-            newbarenames[barename]        = location
-            newstatus[fullname].index[1]  = index
+            newmappings[index]       = fullinfo
+            newentrystatus.index[1]  = index
         end
 
     else --- missing info
-        report("log", 1, "db", "failed to load “%s”", entryname)
+        report("log", 1, "db", "failed to load “%s”", basename)
         return false
     end
     return true
@@ -1021,13 +1142,80 @@ fonts.path_normalize = path_normalize
 
 names.blacklist = { }
 
-local function read_blacklist()
+local blacklist   = names.blacklist
+local p_blacklist --- prefixes of dirs
+
+--- string list -> string list
+local collapse_prefixes = function (lst)
+    --- avoid redundancies in blacklist
+    if #lst < 2 then
+        return lst
+    end
+
+    tablesort(lst)
+    local cur = lst[1]
+    local result = { cur }
+    for i=2, #lst do
+        local elm = lst[i]
+        if stringsub(elm, 1, #cur) ~= cur then
+            --- different prefix
+            cur = elm
+            result[#result+1] = cur
+        end
+    end
+    return result
+end
+
+--- string list -> string list -> (string, bool) hash_t
+local create_blacklist = function (blacklist, whitelist)
+    local result = { }
+    local dirs   = { }
+
+    report("info", 2, "db", "blacklisting “%d” files and directories",
+           #blacklist)
+    for i=1, #blacklist do
+        local entry = blacklist[i]
+        if lfsisdir(entry) then
+            dirs[#dirs+1] = entry
+        else
+            result[blacklist[i]] = true
+        end
+    end
+
+    report("info", 2, "db", "whitelisting “%d” files", #whitelist)
+    for i=1, #whitelist do
+        result[whitelist[i]] = nil
+    end
+
+    dirs = collapse_prefixes(dirs)
+
+    --- build the disjunction of the blacklisted directories
+    for i=1, #dirs do
+        local p_dir = P(dirs[i])
+        if p_blacklist then
+            p_blacklist = p_blacklist + p_dir
+        else
+            p_blacklist = p_dir
+        end
+    end
+
+    if p_blacklist == nil then
+        --- always return false
+        p_blacklist = Cc(false)
+    end
+
+    return result
+end
+
+--- unit -> unit
+read_blacklist = function ()
     local files = {
         kpselookup("luaotfload-blacklist.cnf", {all=true, format="tex"})
     }
-    local blacklist = names.blacklist
+    local blacklist = { }
     local whitelist = { }
 
+    --- TODO lpegify
     if files and type(files) == "table" then
         for _,v in next, files do
             for line in iolines(v) do
@@ -1035,23 +1223,21 @@ local function read_blacklist()
                 local first_chr = stringsub(line, 1, 1) --- faster than find
                 if first_chr == "%" or stringis_empty(line) then
                     -- comment or empty line
+                elseif first_chr == "-" then
+                    whitelist[#whitelist+1] = stringsub(line, 2, -1)
                 else
-                    --- this is highly inefficient
-                    line = stringsplit(line, "%")[1]
-                    line = stringstrip(line)
-                    if stringsub(line, 1, 1) == "-" then
-                        whitelist[stringsub(line, 2, -1)] = true
-                    else
-                        report("log", 2, "db", "blacklisted file “%s”", line)
-                        blacklist[line] = true
+                    local cmt = stringfind(line, "%%")
+                    if cmt then
+                        line = stringsub(line, 1, cmt - 1)
                     end
+                    line = stringstrip(line)
+                    report("log", 2, "db", "blacklisted file “%s”", line)
+                    blacklist[#blacklist+1] = line
                 end
             end
         end
     end
-    for _,fontname in next, whitelist do
-      blacklist[fontname] = nil
-    end
+    names.blacklist = create_blacklist(blacklist, whitelist)
 end
 
 local font_extensions = { "otf", "ttf", "ttc", "dfont" }
@@ -1072,10 +1258,15 @@ end
 
 --doc]]--
 
---- string -> dbobj -> dbobj -> bool -> (int * int)
-local scan_dir = function (dirname, fontnames, newfontnames, dry_run)
+--- string -> dbobj -> dbobj -> bool -> bool -> (int * int)
+local scan_dir = function (dirname, fontnames, newfontnames, dry_run, texmf)
+    if lpegmatch(p_blacklist, dirname) then
+        --- ignore
+        return 0, 0
+    end
+
     local n_scanned, n_new = 0, 0   --- total of fonts collected
-    report("both", 2, "db", "scanning directory %s", dirname)
+    report("both", 3, "db", "scanning directory %s", dirname)
     for _,i in next, font_extensions do
         for _,ext in next, { i, stringupper(i) } do
             local found = dirglob(stringformat("%s/**.%s$", dirname, ext))
@@ -1092,7 +1283,7 @@ local scan_dir = function (dirname, fontnames, newfontnames, dry_run)
                     report("both", 1, "db", "would have been loading “%s”", fullname)
                 else
                     report("both", 4, "db", "loading font “%s”", fullname)
-                    local new = load_font(fullname, fontnames, newfontnames)
+                    local new = load_font(fullname, fontnames, newfontnames, texmf)
                     if new == true then
                         n_new = n_new + 1
                     end
@@ -1121,7 +1312,7 @@ local scan_texmf_fonts = function (fontnames, newfontnames, dry_run)
     if not stringis_empty(fontdirs) then
         for _,d in next, filesplitpath(fontdirs) do
             report("info", 4, "db", "Entering directory %s", d)
-            local found, new = scan_dir(d, fontnames, newfontnames, dry_run)
+            local found, new = scan_dir(d, fontnames, newfontnames, dry_run, true)
             n_scanned = n_scanned + found
             n_new     = n_new     + new
         end
@@ -1145,14 +1336,6 @@ end
 local read_fonts_conf
 do --- closure for read_fonts_conf()
 
-    local lpeg = require "lpeg"
-
-    local C, Cc, Cf, Cg, Ct
-        = lpeg.C, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Ct
-
-    local P, R, S, lpegmatch
-        = lpeg.P, lpeg.R, lpeg.S, lpeg.match
-
     local alpha             = R("az", "AZ")
     local digit             = R"09"
     local tag_name          = C(alpha^1)
@@ -1356,7 +1539,7 @@ end --- read_fonts_conf closure
 
 --- TODO stuff those paths into some writable table
 --- unit -> string list
-local function get_os_dirs()
+local function get_os_dirs ()
     if os.name == 'macosx' then
         return {
             filejoin(kpseexpand_path('~'), "Library/Fonts"),
@@ -1401,10 +1584,85 @@ end
 flush_lookup_cache = function ()
     if not names.lookups then names.lookups = load_lookups() end
     names.lookups = { }
-    collectgarbage"collect"
+    collectgarbage "collect"
     return true, names.lookups
 end
 
+--- dbobj -> dbobj
+local gen_fast_lookups = function (fontnames)
+    report("both", 2, "db", "creating filename map")
+    local mappings   = fontnames.mappings
+    local nmappings  = #mappings
+    --- this is needlessly complicated due to texmf priorization
+    local filenames  = {
+        bare = { },
+        base = { },
+        full = { }, --- non-texmf
+    }
+
+    local texmf, sys = { }, { } -- quintuple list
+
+    for idx = 1, nmappings do
+        local entry    = mappings[idx]
+        local filename = entry.filename
+        local basename = entry.basename
+        local bare     = filenameonly(filename)
+        local subfont  = entry.subfont
+
+        entry.index    = idx
+---     unfortunately, the sys/texmf schism prevents us from
+---     doing away the full name, so we cannot avoid the
+---     substantial duplication
+--      entry.filename = nil
+
+        if entry.texmf == true then
+            texmf[#texmf+1] = { idx, basename, bare, true, nil }
+        else
+            sys[#sys+1] = { idx, basename, bare, false, filename }
+        end
+    end
+
+    local addmap = function (lst)
+        --- this will overwrite existing entries
+        for i=1, #lst do
+            local idx, base, bare, intexmf, full = unpack(lst[i])
+
+            local known = filenames.base[base] or filenames.bare[bare]
+            if known then --- known
+                report("both", 3, "db",
+                       "font file “%s” already indexed (%d)",
+                       base, idx)
+                report("both", 3, "db", "> old location: %s",
+                       (filenames.full[known] or "texmf"))
+                report("both", 3, "db", "> new location: %s",
+                       (intexmf and "texmf" or full))
+            end
+
+            filenames.bare[bare] = idx
+            filenames.base[base] = idx
+            if intexmf == true then
+                filenames.full[idx] = nil
+            else
+                filenames.full[idx] = full
+            end
+        end
+    end
+
+    if config.luaotfload.prioritize == "texmf" then
+        report("both", 2, "db", "preferring texmf fonts")
+        addmap(sys)
+        addmap(texmf)
+    else --- sys
+        addmap(texmf)
+        addmap(sys)
+    end
+
+    fontnames.filenames = filenames
+    texmf, sys = nil, nil
+    collectgarbage "collect"
+    return fontnames
+end
+
 --- force:      dictate rebuild from scratch
 --- dry_dun:    don’t write to the db, just scan dirs
 
@@ -1414,7 +1672,7 @@ update_names = function (fontnames, force, dry_run)
         report("info", 2, "db",
                "skipping database update")
         --- skip all db updates
-        return fontnames
+        return fontnames or names.data
     end
     local starttime = os.gettimeofday()
     local n_scanned, n_new = 0, 0
@@ -1450,6 +1708,11 @@ update_names = function (fontnames, force, dry_run)
     n_scanned = n_scanned + scanned
     n_new     = n_new     + new
 
+    --- we always generate the file lookup tables because
+    --- non-texmf entries are redirected there and the mapping
+    --- needs to be 100% consistent
+    newfontnames = gen_fast_lookups(newfontnames)
+
     --- stats:
     ---            before rewrite   | after rewrite
     ---   partial:         804 ms   |   701 ms
@@ -1652,6 +1915,8 @@ names.save                        = save_names
 names.update                      = update_names
 names.crude_file_lookup           = crude_file_lookup
 names.crude_file_lookup_verbose   = crude_file_lookup_verbose
+names.read_blacklist              = read_blacklist
+names.getfilename                 = resolve_fullpath
 
 --- font cache
 names.purge_cache    = purge_cache
@@ -1672,8 +1937,4 @@ names.find_closest      = find_closest
 -- for testing purpose
 names.read_fonts_conf = read_fonts_conf
 
---- dummy required by luatex-fonts (cf. luatex-fonts-syn.lua)
-
-fonts.names.getfilename = function (askedname,suffix) return "" end
-
 -- vim:tw=71:sw=4:ts=4:expandtab
diff --git a/luaotfload-features.lua b/luaotfload-features.lua
index a68eb04..a70dace 100644
--- a/luaotfload-features.lua
+++ b/luaotfload-features.lua
@@ -1003,7 +1003,11 @@ local prefixed          = P"name:" * ws * Cg(fontname, "name")
 --- with paths already, so we’ll add a less strict rule here.  anyways,
 --- we’ll emit a warning.
                         + P"file:" * ws * Cg(unsupported, "path")
-                        + P"file:" * ws * Cg(fontname,    "file")
+                        + P"file:" * ws * Cg(fontname, "file")
+--- EXPERIMENTAL: kpse lookup
+                        + P"kpse:" * ws * Cg(fontname, "kpse")
+--- EXPERIMENTAL: custom lookup
+                        + P"my:" * ws * Cg(fontname, "my")
 local unprefixed        = Cg(fontname, "anon")
 local path_lookup       = lbrk * Cg(C((1-rbrk)^1), "path") * rbrk
 
@@ -1069,7 +1073,7 @@ local import_values = {
     { "mode",   true },
 }
 
-local lookup_types = { "anon", "file", "name", "path" }
+local lookup_types = { "anon", "file", "kpse", "my", "name", "path" }
 
 local select_lookup = function (request)
     for i=1, #lookup_types do
@@ -1171,7 +1175,7 @@ local handle_request = function (specification)
         local keep       = import_values[n][2]
         local newvalue   = request.features[feat]
         if newvalue then
-            specification[feat]    = request.features[feat]
+            specification[feat] = request.features[feat]
             if not keep then
                 request.features[feat] = nil
             end
diff --git a/luaotfload-override.lua b/luaotfload-override.lua
index 5e642e4..caf3627 100644
--- a/luaotfload-override.lua
+++ b/luaotfload-override.lua
@@ -1,17 +1,30 @@
 if not modules then modules = { } end modules ['luat-ovr'] = {
-    version   = 2.2,
+    version   = 2.3,
     comment   = "companion to luatex-*.tex",
     author    = "Khaled Hosny, Elie Roux, Philipp Gesang",
     copyright = "Luaotfload Development Team",
     license   = "GNU GPL v2"
 }
 
-local module_name   = "luaotfload"
+--[[doc--
+The logging system is slow in general, as we always have the function
+call overhead even if we aren’t going to output anything. On the other
+hand, the more efficient approach followed by Context isn’t an option
+because we lack a user interface to toggle per-subsystem tracing.
+--doc]]--
+
+local module_name       = "luaotfload"
 
-local texiowrite_nl = texio.write_nl
-local stringformat  = string.format
-local tableconcat   = table.concat
-local type          = type
+local select            = select
+local stringformat      = string.format
+local tableconcat       = table.concat
+local texiowrite_nl     = texio.write_nl
+local texiowrite        = texio.write
+local type              = type
+
+local texio_write_nl    = texio.write_nl
+local texio_write       = texio.write
+local iowrite           = io.write
 
 --[[doc--
 We recreate the verbosity levels previously implemented in font-nms:
@@ -65,11 +78,40 @@ local log = function (category, fmt, ...)
     texiowrite_nl(logout, tableconcat(res))
 end
 
-local stdout = function (category, fmt, ...)
-    local res = { module_name, " |" }
-    if category then res[#res+1] = " " .. category end
-    if fmt      then res[#res+1] = ": " .. stringformat(fmt, ...) end
-    texiowrite_nl(tableconcat(res))
+--- with faux db update with maximum verbosity:
+---
+---     ---------   --------
+---     buffering   time (s)
+---     ---------   --------
+---     full        4.12
+---     line        4.20
+---     none        4.39
+---     ---------   --------
+---
+
+io.stdout:setvbuf "no"
+io.stderr:setvbuf "no"
+
+local writeln
+if tex and (tex.jobname or tex.formatname) then
+    --- TeX
+    writeln = texiowrite_nl
+else
+    --- Lua interpreter
+    writeln = function (str)
+        iowrite(str)
+        iowrite "\n"
+    end
+end
+
+stdout = function (category, ...)
+    local res = { module_name, "|", category, ":" }
+    if select("#", ...) == 1 then
+        res[#res+1] = select(1, ...) -- around 30% faster than unpack()
+    else
+        res[#res+1] = stringformat(...)
+    end
+    writeln(tableconcat(res, " "))
 end
 
 --- at default (zero), we aim to be quiet
diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua
index 721e3cf..e3f5a93 100755
--- a/luaotfload-tool.lua
+++ b/luaotfload-tool.lua
@@ -172,6 +172,7 @@ This tool is part of the luaotfload package. Valid options are:
   -f --force                   force re-indexing all fonts
   -l --flush-lookups           empty lookup cache of font requests
   -D --dry-run                 skip loading of fonts, just scan
+  -p --prefer-texmf            prefer fonts in the TEXMF over system fonts
 
   --find="font name"           query the database for a font name
   -F --fuzzy                   look for approximate matches if --find fails
@@ -182,6 +183,7 @@ This tool is part of the luaotfload package. Valid options are:
   --list=<criterion>           output list of entries by field <criterion>
   --list=<criterion>:<value>   restrict to entries with <criterion>=<value>
   --fields=<f1>,<f2>,…,<fn>    which fields <f> to print with --list
+  -b --show-blacklist          show blacklisted files
 
 The font database will be saved to
    %s
@@ -268,7 +270,7 @@ set.
 --]]--
 
 local action_sequence = {
-    "loglevel", "help",     "version", "cache",
+    "loglevel", "help",     "version", "blacklist", "cache",
     "flush",    "generate", "list",    "query",
 }
 local action_pending  = table.tohash(action_sequence, false)
@@ -296,6 +298,15 @@ actions.help = function (job)
     return true, false
 end
 
+actions.blacklist = function (job)
+    names.read_blacklist()
+    local n = 0
+    for n, entry in next, table.sortedkeys(fonts.names.blacklist) do
+        texiowrite_nl(stringformat("(%d %s)", n, entry))
+    end
+    return true, false
+end
+
 actions.generate = function (job)
     local fontnames, savedname
     fontnames = names.update(fontnames, job.force_reload, job.dry_run)
@@ -353,24 +364,24 @@ actions.query = function (job)
         fonts.names.resolve(nil, nil, tmpspec)
 
     if success then
-        logs.names_report(false, 1,
+        logs.names_report(false, 0,
             "resolve", "Font “%s” found!", query)
         if subfont then
-            logs.names_report(false, 1, "resolve",
+            logs.names_report(false, 0, "resolve",
                 "Resolved file name “%s”, subfont nr. “%s”",
                 foundname, subfont)
         else
-            logs.names_report(false, 1,
+            logs.names_report(false, 0,
                 "resolve", "Resolved file name “%s”", foundname)
         end
         if job.show_info then
             show_font_info(foundname)
         end
     else
-        logs.names_report(false, 1,
+        logs.names_report(false, 0,
             "resolve", "Cannot find “%s”.", query)
         if job.fuzzy == true then
-            logs.names_report(false, 1,
+            logs.names_report(false, 0,
                 "resolve", "Looking for close matches, this may take a while ...")
             local success = fonts.names.find_closest(query, job.fuzzy_limit)
         end
@@ -545,7 +556,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
         force_reload = nil,
         criterion    = "",
         query        = "",
-        log_level    = 1, --- 2 is approx. the old behavior
+        log_level    = 0, --- 2 is approx. the old behavior
     }
 
     local long_options = {
@@ -562,13 +573,15 @@ local process_cmdline = function ( ) -- unit -> jobspec
         limit              = 1,
         list               = 1,
         log                = 1,
+        ["prefer-texmf"]   = "p",
         quiet              = "q",
+        ["show-blacklist"] = "b",
         update             = "u",
         verbose            = 1  ,
         version            = "V",
     }
 
-    local short_options = "DfFilquvVh"
+    local short_options = "bDfFilpquvVh"
 
     local options, _, optarg =
         alt_getopt.get_ordered_opts (arg, short_options, long_options)
@@ -629,6 +642,10 @@ local process_cmdline = function ( ) -- unit -> jobspec
             result.cache = optarg[n]
         elseif v == "D" then
             result.dry_run = true
+        elseif v == "p" then
+            config.luaotfload.prioritize = "texmf"
+        elseif v == "b" then
+            action_pending["blacklist"] = true
         end
     end
 
diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst
index 9ea267b..06ab1cc 100644
--- a/luaotfload-tool.rst
+++ b/luaotfload-tool.rst
@@ -15,9 +15,9 @@
 SYNOPSIS
 =======================================================================
 
-**luaotfload** [ -cfFiquvVh ]
+**luaotfload** [ -bDcfFipquvVh ]
 
-**luaotfload** --update [ --force ] [ --quiet ] [ --verbose ] [ --dry-run ]
+**luaotfload** --update [ --force ] [ --quiet ] [ --verbose ] [ --prefer-texmf ] [ --dry-run ]
 
 **luaotfload** --find=FONTNAME [ --fuzzy ] [ --info ]
 
@@ -31,6 +31,8 @@ SYNOPSIS
 
 **luaotfload** --version
 
+**luaotfload** --show-blacklist
+
 DESCRIPTION
 =======================================================================
 
@@ -55,6 +57,9 @@ update mode
 --update, -u            Update the database; indexes new fonts.
 --force, -f             Force rebuilding of the database; re-indexes
                         all fonts.
+--prefer-texmf, -p      Organize the file name database in a way so
+                        that it prefer fonts in the *TEXMF* tree over
+                        system fonts if they are installed in both.
 --dry-run, -D           Don’t load fonts, scan directories only.
                         (For debugging file system related issues.)
 
@@ -68,6 +73,7 @@ query mode
                         ``--find``).
 --info, -i              Display basic information to a resolved font
                         file (requires ``--find``).
+--show-blacklist, -b    Show blacklisted files (not directories).
 --list=CRITERION        Show entries, where *CRITERION* is one of the
                         following:
 
diff --git a/luaotfload.dtx b/luaotfload.dtx
index e75a561..095ac23 100644
--- a/luaotfload.dtx
+++ b/luaotfload.dtx
@@ -1437,8 +1437,9 @@ config.luaotfload                 = config.luaotfload or { }
 config.luaotfload.resolver        = config.luaotfload.resolver         or "cached"
 config.luaotfload.definer         = config.luaotfload.definer          or "patch"
 config.luaotfload.compatibility   = config.luaotfload.compatibility    or false
-config.luaotfload.loglevel        = config.luaotfload.loglevel  or 1
+config.luaotfload.loglevel        = config.luaotfload.loglevel         or 1
 config.luaotfload.color_callback  = config.luaotfload.color_callback   or "pre_linebreak_filter"
+config.luaotfload.prioritize      = config.luaotfload.prioritize       or "sys"
 --luaotfload.prefer_merge     = config.luaotfload.prefer_merge or true
 
 luaotfload.module = {
@@ -1797,7 +1798,7 @@ loadmodule"colors.lua"     --- “font-clr”
 %    \begin{macrocode}
 
 local request_resolvers   = fonts.definers.resolvers
-local formats             = fonts.formats
+local formats             = fonts.formats -- nice table; does lowercasing ...
 formats.ofm               = "type1"
 
 %    \end{macrocode}
@@ -1821,7 +1822,7 @@ formats.ofm               = "type1"
 local resolvefile = fonts.names.crude_file_lookup
 --local resolvefile = fonts.names.crude_file_lookup_verbose
 
-function request_resolvers.file(specification)
+request_resolvers.file = function (specification)
     local name    = resolvefile(specification.name)
     local suffix  = file.suffix(name)
     if formats[suffix] then
@@ -1832,7 +1833,6 @@ function request_resolvers.file(specification)
     end
 end
 
-
 %    \end{macrocode}
 % We classify as \verb|anon:| those requests that have neither a
 % prefix nor brackets. According to Khaled\footnote{%
@@ -1917,6 +1917,44 @@ request_resolvers.path = function (specification)
 end
 
 %    \end{macrocode}
+% {\bfseries EXPERIMENTAL}:
+% \identifier{kpse}-only resolver, for those who can do without system
+% fonts.
+%
+%    \begin{macrocode}
+
+request_resolvers.kpse = function (specification)
+    local name       = specification.name
+    local suffix     = file.suffix(name)
+    if suffix and formats[suffix] then
+        name = file.removesuffix(name)
+        if resolvers.findfile(name, suffix) then
+            specification.forced = suffix
+            specification.name   = name
+            return
+        end
+    end
+    for t, format in next, formats do --- brute force
+        if kpse.find_file (name, format) then
+            specification.forced = t
+            specification.name   = name
+            return
+        end
+    end
+end
+
+%    \end{macrocode}
+% Also {\bfseries EXPERIMENTAL}:
+% custom file resolvers via callback.
+%
+%    \begin{macrocode}
+create_callback("luaotfload.resolve_font", "simple", dummy_function)
+
+request_resolvers.my = function (specification)
+    call_callback("luaotfload.resolve_font", specification)
+end
+
+%    \end{macrocode}
 % We create a callback for patching fonts on the fly, to be used by other
 % packages.
 % It initially contains the empty function that we are going to override
@@ -1943,12 +1981,12 @@ local patch_defined_font = function (specification, size, id)
         --- We need to test for the “shared” field here
         --- or else the fontspec capheight callback will
         --- operate on tfm fonts.
-        call_callback("luaotfload.patch_font", tfmdata)
+        call_callback("luaotfload.patch_font", tfmdata, specification)
     end
     return tfmdata
 end
 
-reset_callback("define_font")
+reset_callback "define_font"
 
 %    \end{macrocode}
 % Finally we register the callbacks.