From 38c8611e8af03e7da61919b3c723cc34e9a62c27 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 25 Apr 2013 15:26:07 +0200 Subject: [db] map filenames to db indices instead of paths --- luaotfload-database.lua | 137 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 93 insertions(+), 44 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e8f3d1d..acc351f 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -66,7 +66,7 @@ fonts.names = fonts.names or { } local names = fonts.names -names.version = 2.2 +names.version = 2.201 names.data = nil names.path = { basename = "luaotfload-names.lua", @@ -99,12 +99,15 @@ local sanitize_string = function (str) end --[[doc-- -This is a sketch of the db: +This is a sketch of the luaotfload db: type dbobj = { - mappings : fontentry list; - status : filestatus; - version : float; + mappings : fontentry list; + status : filestatus; + version : float; + // preliminary additions of v2.2: + basenames : (string, int) hash; // where int is the index in mappings + barenames : (string, int) hash; // where int is the index in mappings } and fontentry = { familyname : string; @@ -126,6 +129,40 @@ This is a sketch of the db: beware that this is a reconstruction and may be incomplete. +mtx-fonts has in names.tma: + + type names = { + cache_uuid : uuid; + cache_version : float; + datastate : uuid list; + fallbacks : (filetype, (basename, int) hash) hash; + families : (basename, int list) hash; + files : (filename, fullname) hash; + indices : (fullname, int) hash; + mappings : (filetype, (basename, int) hash) hash; + names : ? (empty hash) ?; + rejected : (basename, int) hash; + specifications: fontentry list; + } + and fontentry = { + designsize : int; + familyname : string; + filename : string; + fontname : string; + format : string; + fullname : string; + maxsize : int; + minsize : int; + modification : int; + rawname : string; + style : string; + subfamily : string; + variant : string; + weight : string; + width : string; + } + + --doc]]-- local fontnames_init = function ( ) @@ -134,10 +171,12 @@ local fontnames_init = function ( ) status = { }, --- adding filename mapping increases the --- size of the serialized db on my system - --- (5840 font files) by a factor of ... - barenames = { },--- incr. by 1.11 - basenames = { },--- incr. by 1.22 --- fullnames = { },--- incr. by 1.48 + --- (5840 font files) by a factor of 1.09 + --- if we store only the indices in the + --- mappings table + barenames = { }, + basenames = { }, +-- fullnames = { }, version = names.version, } end @@ -240,28 +279,38 @@ end local fonts_loaded = false local fonts_reloaded = false +--- chain: barenames -> [fullnames ->] basenames -> findfile local crude_file_lookup_verbose = function (data, filename) - local found = data.barenames[filename] - if found then + local mappings = data.mappings + local found + + --- look up in db first ... + found = data.barenames[filename] + if found and mappings[found] then + found = mappings[found].filename report("info", 0, "db", "crude file lookup: req=%s; hit=bare; ret=%s", filename, found[1]) return found end -- found = data.fullnames[filename] --- if found then --- report("info", 0, "db", +-- if found and mappings[found] then +-- found = mappings[found].filename[1] -- "crude file lookup: req=%s; hit=bare; ret=%s", -- filename, found[1]) -- return found -- end found = data.basenames[filename] - if found then + if found and mappings[found] then + found = mappings[found].filename report("info", 0, "db", "crude file lookup: req=%s; hit=bare; ret=%s", filename, found[1]) return found end + + --- now look for tfm et al.; will be superseded by proper + --- format lookup found = resolvers.findfile(filename, "tfm") if found then report("info", 0, "db", @@ -281,7 +330,10 @@ local crude_file_lookup = function (data, filename) local found = data.barenames[filename] -- or data.fullnames[filename] or data.basenames[filename] - if found then return found end + if found then + found = data.mappings[found] + if found then return found.filename end + end found = resolvers.findfile(filename, "tfm") if found then return { found, false } end found = resolvers.findfile(filename, "ofm") @@ -742,13 +794,14 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) if db_timestamp == timestamp and not newstatus[entryname].index[1] then for _,v in next, status[entryname].index do - local index = #newstatus[entryname].index - local fullinfo = mappings[v] - newmappings[#newmappings+1] = fullinfo --- keep - newstatus[entryname].index[index+1] = #newmappings --- newfullnames[fullname] = fullinfo.filename - newbasenames[basename] = fullinfo.filename - newbarenames[barename] = fullinfo.filename + local index = #newstatus[entryname].index + local fullinfo = mappings[v] + local location = #newmappings + 1 + newmappings[location] = fullinfo --- keep + newstatus[entryname].index[index+1] = location --- is this actually used anywhere? +-- newfullnames[fullname] = location + newbasenames[basename] = location + newbarenames[barename] = location end report("log", 2, "db", "font “%s” already indexed", entryname) return false @@ -757,38 +810,34 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) local info = fontloaderinfo(fullname) if info then if type(info) == "table" and #info > 1 then --- ttc - for i in next, info do - local fullinfo = font_fullinfo(fullname, i-1, texmf) + for n_font = 1, #info do + local fullinfo = font_fullinfo(fullname, n_font-1, texmf) if not fullinfo then return false end - local index = newstatus[entryname].index[i] - if newstatus[entryname].index[i] then - index = newstatus[entryname].index[i] - else - index = #newmappings+1 - end - newmappings[index] = fullinfo --- newfullnames[fullname] = fullinfo.filename - newbasenames[basename] = fullinfo.filename - newbarenames[barename] = fullinfo.filename - newstatus[entryname].index[i] = index + local location = #newmappings+1 + local index = newstatus[entryname].index[n_font] + if not index then index = location end + + newmappings[index] = fullinfo +-- newfullnames[fullname] = location + newbasenames[basename] = location + newbarenames[barename] = location + newstatus[entryname].index[n_font] = index end else local fullinfo = font_fullinfo(fullname, false, texmf) if not fullinfo then return false end - local index - if newstatus[entryname].index[1] then - index = newstatus[entryname].index[1] - else - index = #newmappings+1 - end + local location = #newmappings+1 + local index = newstatus[entryname].index[1] + if not index then index = location end + newmappings[index] = fullinfo --- newfullnames[fullname] = { fullinfo.filename[1], fullinfo.filename[2] } - newbasenames[basename] = { fullinfo.filename[1], fullinfo.filename[2] } - newbarenames[barename] = { fullinfo.filename[1], fullinfo.filename[2] } +-- newfullnames[fullname] = location + newbasenames[basename] = location + newbarenames[barename] = location newstatus[entryname].index[1] = index end -- cgit v1.2.3 From 3a645a338fc4297bc9a61dae1fece13533d2f3ee Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 25 Apr 2013 17:26:16 +0200 Subject: store preprocessed font names with db --- luaotfload-database.lua | 82 ++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 35 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index acc351f..bd664a6 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -436,12 +436,18 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con for _,face in next, data.mappings do --- TODO we really should store those in dedicated --- .sanitized field - local family = sanitize_string(face.names and face.names.family) - local subfamily = sanitize_string(face.names and face.names.subfamily) - local fullname = sanitize_string(face.names and face.names.fullname) - local psname = sanitize_string(face.names and face.names.psname) - local fontname = sanitize_string(face.fontname) - local pfullname = sanitize_string(face.fullname) + local family, subfamily, fullname, psname, fontname, pfullname + + local facenames = face.sanitized + if facenames then + family = facenames.family + subfamily = facenames.subfamily + fullname = facenames.fullname + psname = facenames.psname + end + fontname = facenames.fontname or sanitize_string(face.fontname) + pfullname = facenames.pfullname or sanitize_string(face.fullname) + local optsize, dsnsize, maxsize, minsize if #face.size > 0 then optsize = face.size @@ -483,30 +489,24 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con elseif subfamily == "regular" or synonym_set.regular[subfamily] then found.fallback = face - elseif name == fullname then - --- happens with Libertine Mono which has - --- “mono” as subfamily - found[1] = face - break end + end - else - if name == fullname - or name == pfullname - or name == fontname - or name == psname then - if optsize then - if dsnsize == size - or (size > minsize and size <= maxsize) then - found[1] = face - break - else - found[#found+1] = face - end - else + if name == fullname + or name == pfullname + or name == fontname + or name == psname then + if optsize then + if dsnsize == size + or (size > minsize and size <= maxsize) then found[1] = face break + else + found[#found+1] = face end + else + found[1] = face + break end end end @@ -623,7 +623,7 @@ find_closest = function (name, limit) for n = 1, n_fonts do local current = mappings[n] - local cnames = current.names + local cnames = current.sanitized --[[ This is simplistic but surpisingly fast. Matching is performed against the “family” name @@ -675,6 +675,14 @@ find_closest = function (name, limit) return false end --- find_closest() +local sanitize_names = function (names) + local res = { } + for idx, name in next, names do + res[idx] = sanitize_string(name) + end + return res +end + --[[doc-- The data inside an Opentype font file can be quite heterogeneous. Thus in order to get the relevant information, parts of the original @@ -701,18 +709,22 @@ font_fullinfo = function (filename, subfont, texmf) if metadata.names then for _, namedata in next, metadata.names do if namedata.lang == "English (US)" then - tfmdata.names = { + local names = { --- see --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html - fullname = namedata.names.compatfull - or namedata.names.fullname, - family = namedata.names.preffamilyname - or namedata.names.family, - subfamily= tfmdata.fontstyle_name - or namedata.names.prefmodifiers - or namedata.names.subfamily, - psname = namedata.names.postscriptname + fullname = namedata.names.compatfull + or namedata.names.fullname, + family = namedata.names.preffamilyname + or namedata.names.family, + subfamily = tfmdata.fontstyle_name + or namedata.names.prefmodifiers + or namedata.names.subfamily, + psname = namedata.names.postscriptname, + pfullname = metadata.fullname, + fontname = metadata.fontname, } + tfmdata.names = names + tfmdata.sanitized = sanitize_names(names) end end else -- cgit v1.2.3 From cd83d13014406c6e6dd8facf325c19bf0b51c432 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 25 Apr 2013 18:35:50 +0200 Subject: scan fontconfig dirs even though ``$OSFONTDIR`` is set This is essentially what this change to the TL2011 version does: https://github.com/lualatex/luaotfload/commit/31530badb2681ff3423fe31f5eccf3d05fab1956#L2R657 in response to this issue: https://github.com/lualatex/luaotfload/issues/11 --- luaotfload-database.lua | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bd664a6..1b13f69 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1188,19 +1188,16 @@ update_names = function (fontnames, force) end local newfontnames = fontnames_init() read_blacklist() - --installed_fonts_scanned = false - --scan_installed_fonts(fontnames, newfontnames) --- see fixme above - local scanned, new = scan_texmf_fonts(fontnames, newfontnames) + + local scanned, new + scanned, new = scan_texmf_fonts(fontnames, newfontnames) n_scanned = n_scanned + scanned n_new = n_new + new - --if not installed_fonts_scanned - --and stringis_empty(kpseexpand_path("$OSFONTDIR")) - if stringis_empty(kpseexpand_path("$OSFONTDIR")) - then - local scanned, new = scan_os_fonts(fontnames, newfontnames) - n_scanned = n_scanned + scanned - n_new = n_new + new - end + + scanned, new = scan_os_fonts(fontnames, newfontnames) + n_scanned = n_scanned + scanned + n_new = n_new + new + --- stats: --- before rewrite | after rewrite --- partial: 804 ms | 701 ms -- cgit v1.2.3 From b2659be825b80de3201a3df17cc1dc2d4bcf296d Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 26 Apr 2013 14:19:00 +0200 Subject: add draft lookup cache --- luaotfload-database.lua | 117 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 6 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1b13f69..55966bf 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -61,8 +61,11 @@ local tableappend = table.append local tabletohash = table.tohash --- the font loader namespace is “fonts”, same as in Context -fonts = fonts or { } -fonts.names = fonts.names or { } +--- we need to put some fallbacks into place for when running +--- as a script +fonts = fonts or { } +fonts.names = fonts.names or { } +fonts.definers = fonts.definers or { } local names = fonts.names @@ -73,6 +76,7 @@ names.path = { dir = "", path = "", } +local lookup_cache = nil --> names.data.cache; populated by load_names -- We use the cache.* of ConTeXt (see luat-basics-gen), we can -- use it safely (all checks and directory creations are already done). It @@ -108,6 +112,7 @@ This is a sketch of the luaotfload db: // preliminary additions of v2.2: basenames : (string, int) hash; // where int is the index in mappings barenames : (string, int) hash; // where int is the index in mappings + caches : lookup_cache; // see below } and fontentry = { familyname : string; @@ -165,7 +170,7 @@ mtx-fonts has in names.tma: --doc]]-- -local fontnames_init = function ( ) +local fontnames_init = function ( ) --- returns clean dbobj return { mappings = { }, status = { }, @@ -178,6 +183,10 @@ local fontnames_init = function ( ) basenames = { }, -- fullnames = { }, version = names.version, + caches = { --- new + file = { }, name = { }, + path = { }, anon = { }, + } } end @@ -242,6 +251,7 @@ load_names = function ( ) "Font names database loaded", "%s", foundname) report("info", 3, "db", "Loading took %0.f ms", 1000*(os.gettimeofday()-starttime)) + lookup_cache = data.caches else report("info", 1, "db", [[Font names database not found, generating new one. @@ -249,6 +259,7 @@ load_names = function ( ) data = update_names(fontnames_init()) save_names(data) end + fonts_loaded = true return data end @@ -341,6 +352,101 @@ local crude_file_lookup = function (data, filename) return false end +--[[doc-- +Lookups can be quite costly, more so the less specific they are. +Even if we find a matching font eventually, the next time the +user compiles Eir document E will have to stand through the delay +again. +Thus, some caching of results -- even between runs -- is in order. +We’ll just store successful lookups in the database in a record of +the respective lookup type. + +type lookup_cache = { + file : lookups; + name : lookups; + path : lookups; + anon : lookups; +} +and lookups = (string, (string * num)) dict + +TODO: + × 1) add cache to dbobj + × 2) wrap lookups in cached versions + 3) make caching optional (via the config table) for debugging + 4) make names_update() cache aware (nil if “force”) + × 5) add logging + 6) add cache control to fontdbutil + 7) incr db version + 8) ??? + n) PROFIT!!! + +--doc]]-- + +--- the resolver is called after the font request is parsed +--- this is where we insert the cache +local normal_resolve = fonts.definers.resolve +local dummyresolver = function (specification) + --- this ensures that the db is always loaded + --- before a lookup occurs + if not fonts_loaded then names.data = load_names() end + inspect(normal_resolve(specification)) + return normal_resolve(specification) +end + +--[[doc-- +The name lookup requires both the “name” and the “style” +key, so we’ll concatenate them. +--doc]]-- +--- string -> spec -> string +local to_hash_field = function (lookup, specification) + if lookup == "name" and specification.style then + return specification.name.."*"..specification.style + end + return specification.name +end + +--[[doc-- +From my reading of font-def.lua, what a resolver does is +basically rewrite the “name” field of the specification record +with the resolution. +Also, the fields “resolved”, “sub”, “force” etc. influence the outcome. + +We’ll just cache a deep copy of the entire spec as it leaves the +resolver, lest we want to worry if we caught all the details. +--doc]]-- + +local cached_resolver = function (specification) + if not lookup_cache then names.data = load_names() end + local lookup = specification.lookup + local field = to_hash_field(lookup, specification) + local cached = lookup_cache[lookup] + report("info", 4, "cache", + "looking up field “%s” in category “%s” ...", + field, lookup) + local hit = cached[field] + if hit then + report("info", 4, "cache", "found!") + return hit + end + report("info", 4, "cache", "not cached; resolving") + + --- here we add to the cache + local resolved_spec = normal_resolve(specification) + cached[field] = tablecopy(resolved_spec) --- slow for the first time + --- obviously, the updated cache needs to be stored. + --- for the moment, we write the entire db to disk + --- whenever the cache is updated. + --- TODO this should trigger a save only once the + --- document is compiled (finish_pdffile callback?) + report("info", 5, "cache", "saving updated cache") + save_names(names.data) + return resolved_spec +end + +--fonts.definers.resolve = dummyresolver +--fonts.definers.resolve = normalresolver +fonts.definers.resolve = cached_resolver + --[[doc-- Luatex-fonts, the font-loader package luaotfload imports, comes with @@ -382,10 +488,11 @@ font database created by the mkluatexfontdb script. resolve = function (_,_,specification) -- the 1st two parameters are used by ConTeXt if not fonts_loaded then names.data = load_names() - fonts_loaded = true end local data = names.data + --inspect(specification) + --os.exit() if specification.lookup == "file" then local found = crude_file_lookup(data, specification.name) --local found = crude_file_lookup_verbose(data, specification.name) @@ -607,7 +714,6 @@ find_closest = function (name, limit) if not fonts_loaded then names.data = load_names() - fonts_loaded = true end local data = names.data @@ -1234,7 +1340,6 @@ scan_external_dir = function (dir) old_names = names.data else old_names = load_names() - fonts_loaded = true end new_names = tablecopy(old_names) local n_scanned, n_new = scan_dir(dir, old_names, new_names) -- cgit v1.2.3 From 5ee12c85b552649909857ec93af8a70d982da687 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 26 Apr 2013 18:00:19 +0200 Subject: improve request caching --- luaotfload-database.lua | 202 +++++++++++++++++++++++++++--------------------- 1 file changed, 115 insertions(+), 87 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 55966bf..0c47bfd 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -76,7 +76,10 @@ names.path = { dir = "", path = "", } -local lookup_cache = nil --> names.data.cache; populated by load_names + +config = config or { } +config.luaotfload = config.luaotfload or { } +config.luaotfload.resolver = config.luaotfload.resolver or "normal" -- We use the cache.* of ConTeXt (see luat-basics-gen), we can -- use it safely (all checks and directory creations are already done). It @@ -85,8 +88,8 @@ local writable_path = caches.getwritablepath("names","") if not writable_path then error("Impossible to find a suitable writeable cache...") end -names.path.dir = writable_path -names.path.path = filejoin(writable_path, names.path.basename) +names.path.dir = writable_path +names.path.path = filejoin(writable_path, names.path.basename) --[[doc-- Auxiliary functions @@ -106,13 +109,13 @@ end This is a sketch of the luaotfload db: type dbobj = { - mappings : fontentry list; - status : filestatus; - version : float; + mappings : fontentry list; + status : filestatus; + version : float; // preliminary additions of v2.2: - basenames : (string, int) hash; // where int is the index in mappings - barenames : (string, int) hash; // where int is the index in mappings - caches : lookup_cache; // see below + basenames : (string, int) hash; // where int is the index in mappings + barenames : (string, int) hash; // where int is the index in mappings + request_cache : lookup_cache; // see below } and fontentry = { familyname : string; @@ -170,23 +173,26 @@ mtx-fonts has in names.tma: --doc]]-- -local fontnames_init = function ( ) --- returns clean dbobj +local fontnames_init = function (keep_cache) --- returns dbobj + local request_cache + if keep_cache and names.data and names.data.request_cache then + request_cache = names.data.request_cache + else + request_cache = { } + end return { - mappings = { }, - status = { }, + mappings = { }, + status = { }, --- adding filename mapping increases the --- size of the serialized db on my system --- (5840 font files) by a factor of 1.09 --- if we store only the indices in the --- mappings table - barenames = { }, - basenames = { }, --- fullnames = { }, - version = names.version, - caches = { --- new - file = { }, name = { }, - path = { }, anon = { }, - } + barenames = { }, + basenames = { }, +-- fullnames = { }, + version = names.version, + request_cache = request_cache, } end @@ -198,13 +204,8 @@ end --- is assumed to be located at an identical path, carrying the suffix --- .luc. -local code_cache = { } - --- string -> (string * table) local load_lua_file = function (path) - local code = code_cache[path] - if code then return path, code() end - local foundname = filereplacesuffix(path, "luc") local fh = ioopen(foundname, "rb") -- try bin first @@ -225,8 +226,6 @@ local load_lua_file = function (path) end if not code then return nil, nil end - - code_cache[path] = code --- insert into memo return foundname, code() end @@ -241,6 +240,10 @@ local save_names local scan_external_dir local update_names +--- state of the database +local fonts_loaded = false +local fonts_reloaded = false + --- unit -> dbobj load_names = function ( ) local starttime = os.gettimeofday() @@ -251,12 +254,11 @@ load_names = function ( ) "Font names database loaded", "%s", foundname) report("info", 3, "db", "Loading took %0.f ms", 1000*(os.gettimeofday()-starttime)) - lookup_cache = data.caches else report("info", 1, "db", [[Font names database not found, generating new one. This can take several minutes; please be patient.]]) - data = update_names(fontnames_init()) + data = update_names(fontnames_init(false)) save_names(data) end fonts_loaded = true @@ -286,10 +288,6 @@ do end end ---- state of the database -local fonts_loaded = false -local fonts_reloaded = false - --- chain: barenames -> [fullnames ->] basenames -> findfile local crude_file_lookup_verbose = function (data, filename) local mappings = data.mappings @@ -361,19 +359,13 @@ Thus, some caching of results -- even between runs -- is in order. We’ll just store successful lookups in the database in a record of the respective lookup type. -type lookup_cache = { - file : lookups; - name : lookups; - path : lookups; - anon : lookups; -} -and lookups = (string, (string * num)) dict +type lookup_cache = (string, (string * num)) dict TODO: × 1) add cache to dbobj × 2) wrap lookups in cached versions - 3) make caching optional (via the config table) for debugging - 4) make names_update() cache aware (nil if “force”) + × 3) make caching optional (via the config table) for debugging + × 4) make names_update() cache aware (nil if “force”) × 5) add logging 6) add cache control to fontdbutil 7) incr db version @@ -384,26 +376,39 @@ TODO: --- the resolver is called after the font request is parsed --- this is where we insert the cache -local normal_resolve = fonts.definers.resolve -local dummyresolver = function (specification) +local normal_resolver = fonts.definers.resolve +local dummy_resolver = function (specification) --- this ensures that the db is always loaded --- before a lookup occurs - if not fonts_loaded then names.data = load_names() end - inspect(normal_resolve(specification)) - return normal_resolve(specification) + if not names.data then names.data = load_names() end + --inspect(specification) + local resolved = normal_resolver(specification) + --inspect(resolved) + return resolved end --[[doc-- -The name lookup requires both the “name” and the “style” -key, so we’ll concatenate them. +The name lookup requires both the “name” and some other +keys, so we’ll concatenate them. +The spec is modified in place (ugh), so we’ll have to catalogue what +fields actually influence its behavior. + +Idk what the “spec” resolver is for. + + lookup inspects modifies + file: name forced, name + name:* name, style, sub, resolved, sub, name, forced + optsize, size + spec: name, sub resolved, sub, name, forced + +* name: contains both the name resolver from luatex-fonts and resolve() + below + +The following fields of a resolved spec need to be cached: --doc]]-- ---- string -> spec -> string -local to_hash_field = function (lookup, specification) - if lookup == "name" and specification.style then - return specification.name.."*"..specification.style - end - return specification.name -end +local cache_fields = { + "forced", "hash", "lookup", "name", "resolved", "sub", +} --[[doc-- From my reading of font-def.lua, what a resolver does is @@ -415,37 +420,54 @@ We’ll just cache a deep copy of the entire spec as it leaves the resolver, lest we want to worry if we caught all the details. --doc]]-- +--- spec -> spec local cached_resolver = function (specification) - if not lookup_cache then names.data = load_names() end - local lookup = specification.lookup - local field = to_hash_field(lookup, specification) - local cached = lookup_cache[lookup] + if not names.data then names.data = load_names() end + local request_cache = names.data.request_cache + local request = specification.specification report("info", 4, "cache", - "looking up field “%s” in category “%s” ...", - field, lookup) - local hit = cached[field] - if hit then + "looking for “%s” in cache ...", + request) + local found = names.data.request_cache[request] + if found then --- replay fields from cache hit report("info", 4, "cache", "found!") - return hit + for i=1, #cache_fields do + local f = cache_fields[i] + if found[f] then specification[f] = found[f] end + end + return specification end report("info", 4, "cache", "not cached; resolving") - --- here we add to the cache - local resolved_spec = normal_resolve(specification) - cached[field] = tablecopy(resolved_spec) --- slow for the first time + --- first we resolve normally ... + local resolved_spec = normal_resolver(specification) + --- ... then we add the fields to the cache + local entry = { } + for i=1, #cache_fields do + local f = cache_fields[i] + entry[f] = resolved_spec[f] + end + report("info", 4, "cache", "new entry: %s", request) + names.data.request_cache[request] = entry + --- obviously, the updated cache needs to be stored. --- for the moment, we write the entire db to disk --- whenever the cache is updated. --- TODO this should trigger a save only once the --- document is compiled (finish_pdffile callback?) report("info", 5, "cache", "saving updated cache") - save_names(names.data) + save_names() return resolved_spec end ---fonts.definers.resolve = dummyresolver ---fonts.definers.resolve = normalresolver -fonts.definers.resolve = cached_resolver +local resolvers = { + dummy = dummy_resolver, + normal = normal_resolver, + cached = cached_resolver, +} + +fonts.definers.resolve = resolvers[config.luaotfload.resolver] +--fonts.definers.resolve = resolvers.cached --[[doc-- @@ -484,15 +506,16 @@ font database created by the mkluatexfontdb script. --- successful lookup as this cannot be inferred from the other --- values. --- ---- + resolve = function (_,_,specification) -- the 1st two parameters are used by ConTeXt if not fonts_loaded then + print("=============") + print(names.data) names.data = load_names() + --os.exit() end local data = names.data - --inspect(specification) - --os.exit() if specification.lookup == "file" then local found = crude_file_lookup(data, specification.name) --local found = crude_file_lookup_verbose(data, specification.name) @@ -1281,18 +1304,18 @@ update_names = function (fontnames, force) .. (force and " forcefully" or "")) if force then - fontnames = fontnames_init() + fontnames = fontnames_init(false) else if not fontnames then fontnames = load_names() end if fontnames.version ~= names.version then - fontnames = fontnames_init() + fontnames = fontnames_init(true) report("log", 1, "db", "No font names database or old " .. "one found; generating new one") end end - local newfontnames = fontnames_init() + local newfontnames = fontnames_init(true) read_blacklist() local scanned, new @@ -1317,21 +1340,26 @@ end --- dbobj -> unit save_names = function (fontnames) + if not fontnames then fontnames = names.data end local path = names.path.dir if not lfs.isdir(path) then dirmkdirs(path) end - path = filejoin(path, names.path.basename) if fileiswritable(path) then - local luaname, lucname = make_name(path) - tabletofile(luaname, fontnames, true) - caches.compile(fontnames,luaname,lucname) - report("info", 0, "db", "Font names database saved") - return path - else - report("info", 0, "db", "Failed to save names database") - return nil + local luaname, lucname = make_name(names.path.path) + if luaname then + --tabletofile(luaname, data, true, { reduce=true }) + table.tofile(luaname, fontnames, true) + if lucname and type(caches.compile) == "function" then + os.remove(lucname) + caches.compile(fontnames, luaname, lucname) + report("info", 0, "db", "Font names database saved") + return names.path.path + end + end end + report("info", 0, "db", "Failed to save names database") + return nil end scan_external_dir = function (dir) -- cgit v1.2.3 From 05e014ba69d57c18c460e0d0e12a7802d9c58225 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 26 Apr 2013 18:21:06 +0200 Subject: add ``flush-cache`` option to fontdbutil --- luaotfload-database.lua | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 0c47bfd..7c6e657 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -69,7 +69,7 @@ fonts.definers = fonts.definers or { } local names = fonts.names -names.version = 2.201 +names.version = 2.202 names.data = nil names.path = { basename = "luaotfload-names.lua", @@ -231,6 +231,7 @@ end --- define locals in scope local find_closest +local flush_cache local font_fullinfo local load_names local read_fonts_conf @@ -367,9 +368,10 @@ TODO: × 3) make caching optional (via the config table) for debugging × 4) make names_update() cache aware (nil if “force”) × 5) add logging - 6) add cache control to fontdbutil - 7) incr db version - 8) ??? + × 6) add cache control to fontdbutil + × 7) incr db version + 8) wishlist: save cache only at the end of a run + 9) ??? n) PROFIT!!! --doc]]-- @@ -508,12 +510,7 @@ font database created by the mkluatexfontdb script. --- resolve = function (_,_,specification) -- the 1st two parameters are used by ConTeXt - if not fonts_loaded then - print("=============") - print(names.data) - names.data = load_names() - --os.exit() - end + if not fonts_loaded then names.data = load_names() end local data = names.data if specification.lookup == "file" then @@ -1291,6 +1288,13 @@ local function scan_os_fonts(fontnames, newfontnames) return n_scanned, n_new end +flush_cache = function (fontnames) + if not fontnames then fontnames = load_names() end + fontnames.request_cache = { } + collectgarbage"collect" + return true, fontnames +end + --- dbobj -> bool -> dbobj update_names = function (fontnames, force) local starttime = os.gettimeofday() @@ -1376,10 +1380,11 @@ scan_external_dir = function (dir) end --- export functionality to the namespace “fonts.names” -names.scan = scan_external_dir -names.load = load_names -names.update = update_names -names.save = save_names +names.flush_cache = flush_cache +names.load = load_names +names.save = save_names +names.scan = scan_external_dir +names.update = update_names names.resolve = resolve --- replace the resolver from luatex-fonts names.resolvespec = resolve -- cgit v1.2.3 From 7c283ff361da9c8477e4f1af3798d54371f81100 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 26 Apr 2013 18:48:32 +0200 Subject: have flush_cache operate on names.data --- luaotfload-database.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 7c6e657..4980766 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1288,11 +1288,11 @@ local function scan_os_fonts(fontnames, newfontnames) return n_scanned, n_new end -flush_cache = function (fontnames) - if not fontnames then fontnames = load_names() end - fontnames.request_cache = { } +flush_cache = function () + if not names.data then names.data = load_names() end + names.data.request_cache = { } collectgarbage"collect" - return true, fontnames + return true, names.data end --- dbobj -> bool -> dbobj @@ -1352,8 +1352,8 @@ save_names = function (fontnames) if fileiswritable(path) then local luaname, lucname = make_name(names.path.path) if luaname then - --tabletofile(luaname, data, true, { reduce=true }) - table.tofile(luaname, fontnames, true) + --tabletofile(luaname, fontnames, true, { reduce=true }) + tabletofile(luaname, fontnames, true) if lucname and type(caches.compile) == "function" then os.remove(lucname) caches.compile(fontnames, luaname, lucname) -- cgit v1.2.3 From d16a93579dfec03a0dab4281b45f63d877fc4c88 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 26 Apr 2013 22:33:37 +0200 Subject: install the new file: lookup in luaotfload.lua --- luaotfload-database.lua | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 4980766..576971e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -230,6 +230,8 @@ local load_lua_file = function (path) end --- define locals in scope +local crude_file_lookup +local crude_file_lookup_verbose local find_closest local flush_cache local font_fullinfo @@ -290,8 +292,12 @@ do end --- chain: barenames -> [fullnames ->] basenames -> findfile -local crude_file_lookup_verbose = function (data, filename) - local mappings = data.mappings + +--- string -> (string * bool | int) +crude_file_lookup_verbose = function (filename) + if not names.data then names.data = names_reload() end + local data = names.data + local mappings = data.mappings local found --- look up in db first ... @@ -336,7 +342,11 @@ local crude_file_lookup_verbose = function (data, filename) return false end -local crude_file_lookup = function (data, filename) +--- string -> (string * bool | int) +crude_file_lookup = function (filename) + if not names.data then names.data = names_reload() end + local data = names.data + local mappings = data.mappings local found = data.barenames[filename] -- or data.fullnames[filename] or data.basenames[filename] @@ -470,6 +480,7 @@ local resolvers = { fonts.definers.resolve = resolvers[config.luaotfload.resolver] --fonts.definers.resolve = resolvers.cached +fonts.definers.resolve = resolvers.dummy --[[doc-- @@ -513,12 +524,6 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con if not fonts_loaded then names.data = load_names() end local data = names.data - if specification.lookup == "file" then - local found = crude_file_lookup(data, specification.name) - --local found = crude_file_lookup_verbose(data, specification.name) - if found then return found[1], found[2], true end - end - local name = sanitize_string(specification.name) local style = sanitize_string(specification.style) or "regular" @@ -1380,11 +1385,13 @@ scan_external_dir = function (dir) end --- export functionality to the namespace “fonts.names” -names.flush_cache = flush_cache -names.load = load_names -names.save = save_names -names.scan = scan_external_dir -names.update = update_names +names.flush_cache = flush_cache +names.load = load_names +names.save = save_names +names.scan = scan_external_dir +names.update = update_names +names.crude_file_lookup = crude_file_lookup +names.crude_file_lookup_verbose = crude_file_lookup_verbose names.resolve = resolve --- replace the resolver from luatex-fonts names.resolvespec = resolve -- cgit v1.2.3 From 8331000eb1d76381fe5f03fa6c090ae936a110b6 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 27 Apr 2013 02:31:08 +0200 Subject: drop scanning fonts.conf in favor of `fc-cat` --- luaotfload-database.lua | 140 ++++++++++++++++-------------------------------- 1 file changed, 45 insertions(+), 95 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 4980766..e23902a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -21,6 +21,7 @@ local tonumber = tonumber local fontloaderinfo = fontloader.info local iolines = io.lines local ioopen = io.open +local iopopen = io.popen local kpseexpand_path = kpse.expand_path local kpseexpand_var = kpse.expand_var local kpselookup = kpse.lookup @@ -234,7 +235,6 @@ local find_closest local flush_cache local font_fullinfo local load_names -local read_fonts_conf local reload_db local resolve local save_names @@ -1141,6 +1141,29 @@ local function scan_texmf_fonts(fontnames, newfontnames) return n_scanned, n_new end +--- fc-cat outputs every directory it finds fonts in, +--- so we have to remove all subdirectories from the +--- list. + +--- dir list -> dir list +local collapse_paths = function (dir_lst) + tablesort(dir_lst) + local last = dir_lst[1] + local res = { last } + local ndirs = #dir_lst + if ndirs > 1 then + for i=2, ndirs do + local dir = dir_lst[i] + if stringsub(dir, 1, #last) ~= last then + --- new prefix + res[#res+1] = dir + last = dir + end + end + end + return res +end + --[[ For the OS fonts, there are several options: - if OSFONTDIR is set (which is the case under windows by default but @@ -1148,99 +1171,32 @@ end in the scan_texmf_fonts. - if not: - under Windows and Mac OSX, we take a look at some hardcoded directories - - under Unix, we read /etc/fonts/fonts.conf and read the directories in it + - under Unix, we parse the output of “fc-cat -v” This means that if you have fonts in fancy directories, you need to set them in OSFONTDIR. ]] ---- (string -> tab -> tab -> tab) -read_fonts_conf = function (path, results, passed_paths) - --[[ - This function parses /etc/fonts/fonts.conf and returns all the dir - it finds. The code is minimal, please report any error it may - generate. - - TODO fonts.conf are some kind of XML so in theory the following - is totally inappropriate. Maybe a future version of the - lualibs will include the lxml-* files from Context so we - can write something presentable instead. - ]] - local fh = ioopen(path) - passed_paths[#passed_paths+1] = path - passed_paths_set = tabletohash(passed_paths, true) - if not fh then - report("log", 2, "db", "cannot open file %s", path) - return results - end - local incomments = false - for line in fh:lines() do - while line and line ~= "" do - -- spaghetti code... hmmm... - if incomments then - local tmp = stringfind(line, '-->') --- wtf? - if tmp then - incomments = false - line = stringsub(line, tmp+3) - else - line = nil - end - else - local tmp = stringfind(line, '') --- wtf? + if tmp then + incomments = false + line = stringsub(line, tmp+3) + else + line = nil + end + else + local tmp = stringfind(line, '" + + ---> header specifica + local xml_declaration = P"")^0 * P"?>" + local xml_doctype = P"")^0 * P">" + local header = xml_declaration^-1 + * (xml_doctype + comment + ws)^0 + + ---> enforce root node + local root_start = P"<" * ws^-1 * P"fontconfig" * ws^-1 * P">" + local root_stop = P"" + + local dquote, squote = P[["]], P"'" + local xml_namestartchar = S":_" + alpha --- ascii only, funk the rest + local xml_namechar = S":._" + alpha + digit + local xml_name = ws^-1 + * C(xml_namestartchar * xml_namechar^0) + local xml_attvalue = dquote * C((1 - S[[%&"]])^1) * dquote * ws^-1 + + squote * C((1 - S[[%&']])^1) * squote * ws^-1 + local xml_attr = Cg(xml_name * P"=" * xml_attvalue) + local xml_attr_list = Cf(Ct"" * xml_attr^1, rawset) + + --[[doc-- + scan_node creates a parser for a given xml tag. + --doc]]-- + --- string -> bool -> lpeg_t + local scan_node = function (tag) + --- Node attributes go into a table with the index “attributes” + --- (relevant for “prefix="xdg"” and the likes). + local p_tag = P(tag) + local with_attributes = P"<" * p_tag + * Cg(xml_attr_list, "attributes")^-1 + * ws^-1 + * P">" + local plain = P"<" * p_tag * ws^-1 * P">" + local node_start = plain + with_attributes + local node_stop = P"" + --- there is no nesting, the earth is flat ... + local node = node_start + * Cc(tag) * C(comment + (1 - node_stop)^1) + * node_stop + return Ct(node) -- returns {string, string [, attributes = { key = val }] } end - local incomments = false - for line in fh:lines() do - while line and line ~= "" do - -- spaghetti code... hmmm... - if incomments then - local tmp = stringfind(line, '-->') --- wtf? - if tmp then - incomments = false - line = stringsub(line, tmp+3) - else - line = nil + + --[[doc-- + At the moment, the interesting tags are “dir” for + directory declarations, and “include” for including + further configuration files. + + spec: http://freedesktop.org/software/fontconfig/fontconfig-user.html + --doc]]-- + local include_node = scan_node"include" + local dir_node = scan_node"dir" + + local element = dir_node + + include_node + + comment --> ignore + + P(1-root_stop) --> skip byte + + local root = root_start * Ct(element^0) * root_stop + local p_cheapxml = header * root + + --lpeg.print(p_cheapxml) ---> 757 rules with v0.10 + + --[[doc-- + fonts_conf_scanner() handles configuration files. + It is called on an abolute path to a config file (e.g. + /home/luser/.config/fontconfig/fonts.conf) and returns a list + of the nodes it managed to extract from the file. + --doc]]-- + --- string -> path list + local fonts_conf_scanner = function (path) + local fh = ioopen(path, "r") + if not fh then + report("both", 3, "db", "cannot open fontconfig file %s", path) + return + end + local raw = fh:read"*all" + fh:close() + + local confdata = lpegmatch(p_cheapxml, raw) + if not confdata then + report("both", 3, "db", "cannot scan fontconfig file %s", path) + return + end + return confdata + end + + --[[doc-- + read_fonts_conf_indeed() is called with six arguments; the + latter three are tables that represent the state and are + always returned. + The first three are + · the path to the file + · the expanded $HOME + · the expanded $XDG_CONFIG_DIR + --doc]]-- + --- string -> string -> string -> tab -> tab -> (tab * tab * tab) + local read_fonts_conf_indeed + read_fonts_conf_indeed = function (start, home, xdg_home, + acc, done, dirs_done) + + local paths = fonts_conf_scanner(start) + if not paths then --- nothing to do + return acc, done, dirs_done + end + + for i=1, #paths do + local pathobj = paths[i] + local kind, path = pathobj[1], pathobj[2] + local attributes = pathobj.attributes + if attributes and attributes.prefix == "xdg" then + --- this prepends the xdg root (usually ~/.config) + path = filejoin(xdg_home, path) + end + + if kind == "dir" then + if stringsub(path, 1, 1) == "~" then + path = filejoin(home, stringsub(path, 2)) end - else - local tmp = stringfind(line, ' list: paths collected + local done = { } ---> set: files inspected + local dirs_done = { } ---> set: dirs in list + for i=1, #path_list do --- we keep the state between files + acc, done, dirs_done = read_fonts_conf_indeed( + path_list[i], home, xdg_home, + acc, done, dirs_done) + end + return acc + end +end --- read_fonts_conf closure --- TODO stuff those paths into some writable table local function get_os_dirs() @@ -1258,14 +1380,12 @@ local function get_os_dirs() elseif os.type == "windows" or os.type == "msdos" or os.name == "cygwin" then local windir = os.getenv("WINDIR") return { filejoin(windir, 'Fonts') } - else - local passed_paths = {} - local os_dirs = {} - for _,p in next, {"/usr/local/etc/fonts/fonts.conf", "/etc/fonts/fonts.conf"} do - if lfs.isfile(p) then - read_fonts_conf(p, os_dirs, passed_paths) - end - end + else + local fonts_conves = { --- plural, much? + "/usr/local/etc/fonts/fonts.conf", + "/etc/fonts/fonts.conf", + } + local os_dirs = read_fonts_conf(fonts_conves) return os_dirs end return {} @@ -1275,7 +1395,8 @@ local function scan_os_fonts(fontnames, newfontnames) local n_scanned, n_new = 0, 0 --[[ This function scans the OS fonts through - - fontcache for Unix (reads the fonts.conf file and scans the directories) + - fontcache for Unix (reads the fonts.conf file and scans the + directories) - a static set of directories for Windows and MacOSX ]] report("info", 2, "db", "Scanning OS fonts...") @@ -1390,6 +1511,9 @@ names.resolve = resolve --- replace the resolver from luatex-fonts names.resolvespec = resolve names.find_closest = find_closest +-- for testing purpose +names.read_fonts_conf = read_fonts_conf + --- dummy required by luatex-fonts (cf. luatex-fonts-syn.lua) fonts.names.getfilename = function (askedname,suffix) return "" end -- cgit v1.2.3 From 9fceb52298c05c31666521b4226362322b18c178 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 27 Apr 2013 18:26:21 +0200 Subject: update fontconfig test --- luaotfload-database.lua | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index edc3a5b..3085b63 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -86,12 +86,19 @@ config.luaotfload.resolver = config.luaotfload.resolver or "normal" -- We use the cache.* of ConTeXt (see luat-basics-gen), we can -- use it safely (all checks and directory creations are already done). It -- uses TEXMFCACHE or TEXMFVAR as starting points. -local writable_path = caches.getwritablepath("names","") -if not writable_path then - error("Impossible to find a suitable writeable cache...") +local writable_path +if caches then + writable_path = caches.getwritablepath("names","") + if not writable_path then + error("Impossible to find a suitable writeable cache...") + end + names.path.dir = writable_path + names.path.path = filejoin(writable_path, names.path.basename) +else --- running as script, inject some dummies + caches = { } + logs = { report = function () end } end -names.path.dir = writable_path -names.path.path = filejoin(writable_path, names.path.basename) + --[[doc-- Auxiliary functions -- cgit v1.2.3 From a1ab3fe5a146cf741d517be62985b97268de624b Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 28 Apr 2013 12:59:51 +0200 Subject: add fallback for specification name --- luaotfload-database.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 576971e..b678f51 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -437,7 +437,7 @@ local cached_resolver = function (specification) if not names.data then names.data = load_names() end local request_cache = names.data.request_cache local request = specification.specification - report("info", 4, "cache", + report("log", 4, "cache", "looking for “%s” in cache ...", request) local found = names.data.request_cache[request] @@ -449,7 +449,7 @@ local cached_resolver = function (specification) end return specification end - report("info", 4, "cache", "not cached; resolving") + report("log", 4, "cache", "not cached; resolving") --- first we resolve normally ... local resolved_spec = normal_resolver(specification) @@ -459,7 +459,7 @@ local cached_resolver = function (specification) local f = cache_fields[i] entry[f] = resolved_spec[f] end - report("info", 4, "cache", "new entry: %s", request) + report("log", 4, "cache", "new entry: %s", request) names.data.request_cache[request] = entry --- obviously, the updated cache needs to be stored. @@ -467,7 +467,7 @@ local cached_resolver = function (specification) --- whenever the cache is updated. --- TODO this should trigger a save only once the --- document is compiled (finish_pdffile callback?) - report("info", 5, "cache", "saving updated cache") + report("log", 5, "cache", "saving updated cache") save_names() return resolved_spec end @@ -480,7 +480,7 @@ local resolvers = { fonts.definers.resolve = resolvers[config.luaotfload.resolver] --fonts.definers.resolve = resolvers.cached -fonts.definers.resolve = resolvers.dummy +--fonts.definers.resolve = resolvers.dummy --[[doc-- @@ -1268,7 +1268,7 @@ local function get_os_dirs() local os_dirs = {} for _,p in next, {"/usr/local/etc/fonts/fonts.conf", "/etc/fonts/fonts.conf"} do if lfs.isfile(p) then - read_fonts_conf(p, os_dirs, passed_paths) + paths = read_fonts_conf(p, os_dirs, passed_paths) end end return os_dirs -- cgit v1.2.3 From 74ebd14b94432c9de82614627454dfa3bfcb9de0 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 28 Apr 2013 16:39:26 +0200 Subject: add lookups ``file:``, ``path:``, ``name:``, and ``anon:`` --- luaotfload-database.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index b678f51..eb8271d 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -295,7 +295,7 @@ end --- string -> (string * bool | int) crude_file_lookup_verbose = function (filename) - if not names.data then names.data = names_reload() end + if not names.data then names.data = load_names() end local data = names.data local mappings = data.mappings local found @@ -344,7 +344,7 @@ end --- string -> (string * bool | int) crude_file_lookup = function (filename) - if not names.data then names.data = names_reload() end + if not names.data then names.data = load_names() end local data = names.data local mappings = data.mappings local found = data.barenames[filename] -- cgit v1.2.3 From 189085f9a7212320183d5fed0c4aa31df1602d5f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 28 Apr 2013 19:08:11 +0200 Subject: catch tfm/ofm before db update; remove ``kpse.lookup()`` as criterion for resolved font --- luaotfload-database.lua | 84 ++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 46 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index eb8271d..e5065bd 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -291,7 +291,7 @@ do end end ---- chain: barenames -> [fullnames ->] basenames -> findfile +local type1_formats = { "tfm", "ofm", } --- string -> (string * bool | int) crude_file_lookup_verbose = function (filename) @@ -320,26 +320,19 @@ crude_file_lookup_verbose = function (filename) if found and mappings[found] then found = mappings[found].filename report("info", 0, "db", - "crude file lookup: req=%s; hit=bare; ret=%s", + "crude file lookup: req=%s; hit=base; ret=%s", filename, found[1]) return found end - --- now look for tfm et al.; will be superseded by proper - --- format lookup - found = resolvers.findfile(filename, "tfm") - if found then - report("info", 0, "db", - "crude file lookup: req=tfm; hit=bare; ret=%s", found) - return { found, false } - end - found = resolvers.findfile(filename, "ofm") - if found then - report("info", 0, "db", - "crude file lookup: req=ofm; hit=bare; ret=%s", found) - return { found, false } + --- ofm and tfm + for i=1, #type1_formats do + local format = type1_formats[i] + if resolvers.findfile(filename, format) then + return { file.addsuffix(filename, format), false }, format + end end - return false + return { filename, false }, nil end --- string -> (string * bool | int) @@ -354,11 +347,13 @@ crude_file_lookup = function (filename) found = data.mappings[found] if found then return found.filename end end - found = resolvers.findfile(filename, "tfm") - if found then return { found, false } end - found = resolvers.findfile(filename, "ofm") - if found then return { found, false } end - return false + for i=1, #type1_formats do + local format = type1_formats[i] + if resolvers.findfile(filename, format) then + return { file.addsuffix(filename, format), false }, format + end + end + return { filename, false }, nil end --[[doc-- @@ -644,13 +639,14 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con end if #found == 1 then - if kpselookup(found[1].filename[1]) then - report("log", 0, "resolve", - "font family='%s', subfamily='%s' found: %s", - name, style, found[1].filename[1] - ) - return found[1].filename[1], found[1].filename[2], true - end + --- Since we do the file resolving ourselves, we don’t need the + --- kpathsea lookup here any longer. + --- “found” is really synonymous with “registered in the db”. + report("log", 0, "resolve", + "font family='%s', subfamily='%s' found: %s", + name, style, found[1].filename[1] + ) + return found[1].filename[1], found[1].filename[2], true elseif #found > 1 then -- we found matching font(s) but not in the requested optical -- sizes, so we loop through the matches to find the one with @@ -665,13 +661,11 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con least = difference end end - if kpselookup(closest.filename[1]) then - report("log", 0, "resolve", - "font family='%s', subfamily='%s' found: %s", - name, style, closest.filename[1] - ) - return closest.filename[1], closest.filename[2], true - end + report("log", 0, "resolve", + "font family='%s', subfamily='%s' found: %s", + name, style, closest.filename[1] + ) + return closest.filename[1], closest.filename[2], true elseif found.fallback then return found.fallback.filename[1], found.fallback.filename[2], true end @@ -680,7 +674,7 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con if not fonts_reloaded then --- last straw: try reloading the database return reload_db( - "unresoled font name: “" .. name .. "”", + "unresolved font name: “" .. name .. "”", resolve, nil, nil, specification ) end @@ -737,9 +731,7 @@ find_closest = function (name, limit) local name = sanitize_string(name) limit = limit or fuzzy_limit - if not fonts_loaded then - names.data = load_names() - end + if not fonts_loaded then names.data = load_names() end local data = names.data @@ -866,10 +858,10 @@ font_fullinfo = function (filename, subfont, texmf) tfmdata.fontname = metadata.fontname tfmdata.fullname = metadata.fullname tfmdata.familyname = metadata.familyname - tfmdata.filename = { - texmf and filebasename(filename) or filename, - subfont - } + if texmf then + filename = filebasename(filename) + end + tfmdata.filename = { filename, subfont } tfmdata.weight = metadata.pfminfo.weight tfmdata.width = metadata.pfminfo.width tfmdata.slant = metadata.italicangle @@ -1328,9 +1320,9 @@ update_names = function (fontnames, force) read_blacklist() local scanned, new - scanned, new = scan_texmf_fonts(fontnames, newfontnames) - n_scanned = n_scanned + scanned - n_new = n_new + new +-- scanned, new = scan_texmf_fonts(fontnames, newfontnames) +-- n_scanned = n_scanned + scanned +-- n_new = n_new + new scanned, new = scan_os_fonts(fontnames, newfontnames) n_scanned = n_scanned + scanned -- cgit v1.2.3 From 1c8a1d0a59de24f0d1f456c4fad29024e00ef23b Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 28 Apr 2013 19:13:56 +0200 Subject: test for file existence again, but try with ``lfs.isfile()`` first --- luaotfload-database.lua | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e5065bd..8caa35b 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -25,6 +25,7 @@ local kpseexpand_path = kpse.expand_path local kpseexpand_var = kpse.expand_var local kpselookup = kpse.lookup local kpsereadable_file = kpse.readable_file +local lfsisfile = lfs.isfile local mathabs = math.abs local mathmin = math.min local stringfind = string.find @@ -639,14 +640,15 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con end if #found == 1 then - --- Since we do the file resolving ourselves, we don’t need the - --- kpathsea lookup here any longer. --- “found” is really synonymous with “registered in the db”. - report("log", 0, "resolve", - "font family='%s', subfamily='%s' found: %s", - name, style, found[1].filename[1] - ) - return found[1].filename[1], found[1].filename[2], true + local filename = found[1].filename[1] + if lfsisfile(filename) or kpselookup(filename) then + report("log", 0, "resolve", + "font family='%s', subfamily='%s' found: %s", + name, style, filename + ) + return filename, found[1].filename[2], true + end elseif #found > 1 then -- we found matching font(s) but not in the requested optical -- sizes, so we loop through the matches to find the one with @@ -661,13 +663,18 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con least = difference end end - report("log", 0, "resolve", - "font family='%s', subfamily='%s' found: %s", - name, style, closest.filename[1] - ) - return closest.filename[1], closest.filename[2], true + local filename = closest.filename[1] + if lfsisfile(filename) or kpselookup(filename) then + report("log", 0, "resolve", + "font family='%s', subfamily='%s' found: %s", + name, style, filename + ) + return filename, closest.filename[2], true + end elseif found.fallback then - return found.fallback.filename[1], found.fallback.filename[2], true + return found.fallback.filename[1], + found.fallback.filename[2], + true end --- no font found so far -- cgit v1.2.3 From 1a76cea6357f50d7dae752765173af335ba95beb Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 29 Apr 2013 13:57:35 +0200 Subject: add handler for XeTeX slashed notation --- luaotfload-database.lua | 2 -- 1 file changed, 2 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 8caa35b..4aa966a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -562,8 +562,6 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con local found = { } local synonym_set = style_synonyms.set for _,face in next, data.mappings do - --- TODO we really should store those in dedicated - --- .sanitized field local family, subfamily, fullname, psname, fontname, pfullname local facenames = face.sanitized -- cgit v1.2.3 From e79f34d859d48be485589f19fc6905afa5872a53 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 29 Apr 2013 14:52:53 +0200 Subject: only cache ``name:`` lookups --- luaotfload-database.lua | 72 ++++++++++++++++++------------------------------- 1 file changed, 26 insertions(+), 46 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 4aa966a..abf06a2 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -240,6 +240,7 @@ local load_names local read_fonts_conf local reload_db local resolve +local resolve_cached local save_names local scan_external_dir local update_names @@ -380,22 +381,6 @@ TODO: 9) ??? n) PROFIT!!! ---doc]]-- - ---- the resolver is called after the font request is parsed ---- this is where we insert the cache -local normal_resolver = fonts.definers.resolve -local dummy_resolver = function (specification) - --- this ensures that the db is always loaded - --- before a lookup occurs - if not names.data then names.data = load_names() end - --inspect(specification) - local resolved = normal_resolver(specification) - --inspect(resolved) - return resolved -end - ---[[doc-- The name lookup requires both the “name” and some other keys, so we’ll concatenate them. The spec is modified in place (ugh), so we’ll have to catalogue what @@ -428,33 +413,29 @@ We’ll just cache a deep copy of the entire spec as it leaves the resolver, lest we want to worry if we caught all the details. --doc]]-- ---- spec -> spec -local cached_resolver = function (specification) +--- 'a -> 'a -> table -> (string * int|boolean * boolean) +resolve_cached = function (_, _, specification) if not names.data then names.data = load_names() end local request_cache = names.data.request_cache local request = specification.specification - report("log", 4, "cache", - "looking for “%s” in cache ...", + report("log", 4, "cache", "looking for “%s” in cache ...", request) + local found = names.data.request_cache[request] + + --- case 1) cache positive ---------------------------------------- if found then --- replay fields from cache hit report("info", 4, "cache", "found!") - for i=1, #cache_fields do - local f = cache_fields[i] - if found[f] then specification[f] = found[f] end - end - return specification + return found[1], found[2], true end report("log", 4, "cache", "not cached; resolving") + --- case 2) cache negative ---------------------------------------- --- first we resolve normally ... - local resolved_spec = normal_resolver(specification) - --- ... then we add the fields to the cache - local entry = { } - for i=1, #cache_fields do - local f = cache_fields[i] - entry[f] = resolved_spec[f] - end + local filename, subfont, success = resolve(nil, nil, specification) + if not success then return filename, subfont, false end + --- ... then we add the fields to the cache ... ... + local entry = { filename, subfont } report("log", 4, "cache", "new entry: %s", request) names.data.request_cache[request] = entry @@ -463,21 +444,13 @@ local cached_resolver = function (specification) --- whenever the cache is updated. --- TODO this should trigger a save only once the --- document is compiled (finish_pdffile callback?) + --- TODO we should speed up writing by separating + --- the cache from the db report("log", 5, "cache", "saving updated cache") save_names() - return resolved_spec + return filename, subfont, true end -local resolvers = { - dummy = dummy_resolver, - normal = normal_resolver, - cached = cached_resolver, -} - -fonts.definers.resolve = resolvers[config.luaotfload.resolver] ---fonts.definers.resolve = resolvers.cached ---fonts.definers.resolve = resolvers.dummy - --[[doc-- Luatex-fonts, the font-loader package luaotfload imports, comes with @@ -1390,9 +1363,16 @@ names.update = update_names names.crude_file_lookup = crude_file_lookup names.crude_file_lookup_verbose = crude_file_lookup_verbose -names.resolve = resolve --- replace the resolver from luatex-fonts -names.resolvespec = resolve -names.find_closest = find_closest +--- replace the resolver from luatex-fonts +if config.luaotfload.resolver == "cached" then + report("info", 0, "cache", "caching of name: lookups active") + names.resolve = resolve_cached + names.resolvespec = resolve_cached +else + names.resolve = resolve + names.resolvespec = resolve +end +names.find_closest = find_closest --- dummy required by luatex-fonts (cf. luatex-fonts-syn.lua) -- cgit v1.2.3 From 57fab947bfac698ff9f5aada24c0704e7068e224 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 29 Apr 2013 15:38:24 +0200 Subject: remove exceptional handling of texmf fonts (``name:`` always resolves to absolute paths) --- luaotfload-database.lua | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index d4613ba..a750804 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -542,7 +542,7 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con local found = { } local synonym_set = style_synonyms.set - for _,face in next, data.mappings do + for _, face in next, data.mappings do local family, subfamily, fullname, psname, fontname, pfullname local facenames = face.sanitized @@ -797,7 +797,7 @@ The data inside an Opentype font file can be quite heterogeneous. Thus in order to get the relevant information, parts of the original table as returned by the font file reader need to be relocated. --doc]]-- -font_fullinfo = function (filename, subfont, texmf) +font_fullinfo = function (filename, subfont) local tfmdata = { } local rawfont = fontloader.open(filename, subfont) if not rawfont then @@ -844,10 +844,7 @@ font_fullinfo = function (filename, subfont, texmf) tfmdata.fontname = metadata.fontname tfmdata.fullname = metadata.fullname tfmdata.familyname = metadata.familyname - if texmf then - filename = filebasename(filename) - end - tfmdata.filename = { filename, subfont } + tfmdata.filename = { filename, subfont } -- always store full path tfmdata.weight = metadata.pfminfo.weight tfmdata.width = metadata.pfminfo.width tfmdata.slant = metadata.italicangle @@ -862,7 +859,7 @@ end --- we return true if the fond is new or re-indexed --- string -> dbobj -> dbobj -> bool -> bool -local load_font = function (fullname, fontnames, newfontnames, texmf) +local load_font = function (fullname, fontnames, newfontnames) local newmappings = newfontnames.mappings local newstatus = newfontnames.status @@ -879,11 +876,7 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) local basename = filebasename(fullname) local barename = filenameonly(fullname) - --- entryname is apparently the identifier a font is - --- loaded by; it is different for files in the texmf - --- (due to kpse? idk.) - --- entryname = texmf : true -> basename | false -> fullname - local entryname = texmf and basename or fullname + local entryname = basename if not fullname then return false end @@ -899,8 +892,7 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) and status[entryname].timestamp timestamp = lfs.attributes(fullname, "modification") - local index_status = newstatus[entryname] - or (not texmf and newstatus[basename]) + local index_status = newstatus[entryname] or newstatus[basename] local teststat = newstatus[entryname] --- index_status: nil | false | table if index_status and index_status.timestamp == timestamp then @@ -932,7 +924,7 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) if info then if type(info) == "table" and #info > 1 then --- ttc for n_font = 1, #info do - local fullinfo = font_fullinfo(fullname, n_font-1, texmf) + local fullinfo = font_fullinfo(fullname, n_font-1) if not fullinfo then return false end @@ -947,7 +939,7 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) newstatus[entryname].index[n_font] = index end else - local fullinfo = font_fullinfo(fullname, false, texmf) + local fullinfo = font_fullinfo(fullname, false) if not fullinfo then return false end @@ -1070,13 +1062,13 @@ for key, value in next, font_extensions do end --- string -> dbobj -> dbobj -> bool -> (int * int) -local scan_dir = function (dirname, fontnames, newfontnames, texmf) +local scan_dir = function (dirname, fontnames, newfontnames) --[[ This function scans a directory and populates the list of fonts with all the fonts it finds. - dirname is the name of the directory to scan - names is the font database to fill -> no such term!!! - - texmf is a boolean saying if we are scanning a texmf directory + - texmf used to be a boolean saying if we are scanning a texmf directory ]] local n_scanned, n_new = 0, 0 --- total of fonts collected report("log", 2, "db", "scanning", "%s", dirname) @@ -1092,7 +1084,7 @@ local scan_dir = function (dirname, fontnames, newfontnames, texmf) local fullname = found[j] fullname = path_normalize(fullname) report("log", 2, "db", "loading font “%s”", fullname) - local new = load_font(fullname, fontnames, newfontnames, texmf) + local new = load_font(fullname, fontnames, newfontnames) if new then n_new = n_new + 1 end end end @@ -1382,8 +1374,9 @@ local function scan_os_fonts(fontnames, newfontnames) ]] report("info", 2, "db", "Scanning OS fonts...") report("info", 3, "db", "Searching in static system directories...") + print"~~~~" for _,d in next, get_os_dirs() do - local found, new = scan_dir(d, fontnames, newfontnames, false) + local found, new = scan_dir(d, fontnames, newfontnames) n_scanned = n_scanned + found n_new = n_new + new end -- cgit v1.2.3 From dae29b04d0bb7e21bb4d3d19b9781b59e44ec5cb Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 29 Apr 2013 16:26:45 +0200 Subject: use ``fullname`` in status entries This suppresses redundand database updates in the case where a font is found in multiple directories with different timestamps. Also removed references to ``fontdbutil``. --- luaotfload-database.lua | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index a750804..037ca07 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -383,7 +383,7 @@ TODO: × 3) make caching optional (via the config table) for debugging × 4) make names_update() cache aware (nil if “force”) × 5) add logging - × 6) add cache control to fontdbutil + × 6) add cache control to luaotfload-tool × 7) incr db version 8) wishlist: save cache only at the end of a run 9) ??? @@ -858,8 +858,10 @@ font_fullinfo = function (filename, subfont) end --- we return true if the fond is new or re-indexed ---- string -> dbobj -> dbobj -> bool -> bool +--- string -> dbobj -> dbobj -> bool local load_font = function (fullname, fontnames, newfontnames) + if not fullname then return false end + local newmappings = newfontnames.mappings local newstatus = newfontnames.status @@ -878,40 +880,39 @@ local load_font = function (fullname, fontnames, newfontnames) local entryname = basename - if not fullname then return false end - - if names.blacklist[fullname] - or names.blacklist[basename] + if names.blacklist[fullname] or names.blacklist[basename] then report("log", 2, "db", "ignoring blacklisted font “%s”", fullname) return false end + local timestamp, db_timestamp - db_timestamp = status[entryname] - and status[entryname].timestamp + db_timestamp = status[fullname] + and status[fullname].timestamp timestamp = lfs.attributes(fullname, "modification") - local index_status = newstatus[entryname] or newstatus[basename] - local teststat = newstatus[entryname] + local index_status = newstatus[fullname] --- index_status: nil | false | table if index_status and index_status.timestamp == timestamp then -- already indexed this run return false end - newstatus[entryname] = newstatus[entryname] or { } - newstatus[entryname].timestamp = timestamp - newstatus[entryname].index = newstatus[entryname].index or { } + newstatus[fullname] = newstatus[fullname] or { } + newstatus[fullname].timestamp = timestamp + newstatus[fullname].index = newstatus[fullname].index or { } + --- this test compares the modification data registered + --- in the database with the current one if db_timestamp == timestamp - and not newstatus[entryname].index[1] then - for _,v in next, status[entryname].index do - local index = #newstatus[entryname].index + and not newstatus[fullname].index[1] then + for _,v in next, status[fullname].index do + local index = #newstatus[fullname].index local fullinfo = mappings[v] local location = #newmappings + 1 newmappings[location] = fullinfo --- keep - newstatus[entryname].index[index+1] = location --- is this actually used anywhere? + newstatus[fullname].index[index+1] = location --- is this actually used anywhere? -- newfullnames[fullname] = location newbasenames[basename] = location newbarenames[barename] = location @@ -929,14 +930,14 @@ local load_font = function (fullname, fontnames, newfontnames) return false end local location = #newmappings+1 - local index = newstatus[entryname].index[n_font] + local index = newstatus[fullname].index[n_font] if not index then index = location end newmappings[index] = fullinfo -- newfullnames[fullname] = location newbasenames[basename] = location newbarenames[barename] = location - newstatus[entryname].index[n_font] = index + newstatus[fullname].index[n_font] = index end else local fullinfo = font_fullinfo(fullname, false) @@ -944,14 +945,14 @@ local load_font = function (fullname, fontnames, newfontnames) return false end local location = #newmappings+1 - local index = newstatus[entryname].index[1] + local index = newstatus[fullname].index[1] if not index then index = location end newmappings[index] = fullinfo -- newfullnames[fullname] = location newbasenames[basename] = location newbarenames[barename] = location - newstatus[entryname].index[1] = index + newstatus[fullname].index[1] = index end else --- missing info @@ -1108,7 +1109,7 @@ local function scan_texmf_fonts(fontnames, newfontnames) fontdirs = fontdirs .. stringgsub(kpseexpand_path("$TTFONTS"), "^%.", "") if not stringis_empty(fontdirs) then for _,d in next, filesplitpath(fontdirs) do - local found, new = scan_dir(d, fontnames, newfontnames, true) + local found, new = scan_dir(d, fontnames, newfontnames) n_scanned = n_scanned + found n_new = n_new + new end @@ -1374,7 +1375,6 @@ local function scan_os_fonts(fontnames, newfontnames) ]] report("info", 2, "db", "Scanning OS fonts...") report("info", 3, "db", "Searching in static system directories...") - print"~~~~" for _,d in next, get_os_dirs() do local found, new = scan_dir(d, fontnames, newfontnames) n_scanned = n_scanned + found @@ -1409,9 +1409,9 @@ update_names = function (fontnames, force) fontnames = load_names() end if fontnames.version ~= names.version then - fontnames = fontnames_init(true) report("log", 1, "db", "No font names database or old " .. "one found; generating new one") + fontnames = fontnames_init(true) end end local newfontnames = fontnames_init(true) -- cgit v1.2.3