From eb297696edfc77eda080fc9db6bdbfc0b10f4874 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 24 Apr 2013 14:16:29 +0200 Subject: eliminate loop in db updater --- luaotfload-database.lua | 115 ++++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 53 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 19b04db..4ccf4e5 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -121,10 +121,41 @@ local sanitize_string = function (str) return nil end +--[[doc-- +This is a sketch of the db: + + type dbobj = { + mappings : fontentry list; + status : filestatus; + version : float; + } + and fontentry = { + familyname : string; + filename : (string * bool); + fontname : string; + fullname : string; + names : { + family : string; + fullname : string; + psname : string; + subfamily : string; + } + size : int list; + slant : int; + weight : int; + width : int; + } + and filestatus = (fullname, { index : int list; timestamp : int }) dict + +beware that this is a reconstruction and may be incomplete. + +--doc]]-- + local fontnames_init = function ( ) return { mappings = { }, status = { }, + filenames = { }, --- maybe overkill version = names.version, } end @@ -204,10 +235,6 @@ do regular = { "normal", "roman", "plain", "book", "medium", }, - --- TODO note from Élie Roux - --- boldregular was for old versions of Linux Libertine, is it still useful? - --- semibold is in new versions of Linux Libertine, but there is also a bold, - --- not sure it's useful here... bold = { "demi", "demibold", "semibold", "boldregular",}, italic = { "regularitalic", "normalitalic", @@ -741,66 +768,41 @@ for key, value in next, font_extensions do font_extensions_set[value] = true end ---local installed_fonts_scanned = false --- ugh - ---- we already have scan_os_fonts don’t we? - ---local function scan_installed_fonts(fontnames, newfontnames) --- --- Try to query and add font list from operating system. --- --- This uses the lualatex-platform module. --- --- what for? why can’t we do this in Lua? --- report("info", 0, "Scanning fonts known to operating system...") --- local fonts = get_installed_fonts() --- if fonts and #fonts > 0 then --- installed_fonts_scanned = true --- report("log", 2, "operating system fonts found", "%d", #fonts) --- for key, value in next, fonts do --- local file = value.path --- if file then --- local ext = fileextname(file) --- if ext and font_extensions_set[ext] then --- file = path_normalize(file) --- report("log", 1, "loading font", "%s", file) --- load_font(file, fontnames, newfontnames, false) --- end --- end --- end --- else --- report("log", 2, "Could not retrieve list of installed fonts") --- end ---end - -local function scan_dir(dirname, fontnames, newfontnames, texmf) +--- string -> dbobj -> dbobj -> bool -> int +local scan_dir = function (dirname, fontnames, newfontnames, texmf) --[[ This function scans a directory and populates the list of fonts with all the fonts it finds. - dirname is the name of the directory to scan - names is the font database to fill - texmf is a boolean saying if we are scanning a texmf directory + + srsly guys, calling a variable “list” is just lazy! ]] - local list, found = { }, { } - local nbfound = 0 + --- list: string list + local n_found = 0 --- total of fonts collected report("log", 2, "db", "scanning", "%s", dirname) for _,i in next, font_extensions do for _,ext in next, { i, stringupper(i) } do - found = dirglob(stringformat("%s/**.%s$", dirname, ext)) - -- note that glob fails silently on broken symlinks, which happens - -- sometimes in TeX Live. + local found = dirglob(stringformat("%s/**.%s$", dirname, ext)) + local n_new = #found + --- note that glob fails silently on broken symlinks, which happens + --- sometimes in TeX Live. report("log", 2, "db", "fonts found", "%s '%s' fonts found", #found, ext) - nbfound = nbfound + #found - tableappend(list, found) + n_found = n_found + n_new + for j=1, n_new do + local file = found[j] + file = path_normalize(file) + report("log", 1, "db", "loading font", "%s", file) + load_font(file, fontnames, newfontnames, texmf) + end + --tableappend(list, found) end end report("log", 2, "db", - "fonts found", "%d fonts found in '%s'", nbfound, dirname) - - for _,file in next, list do - file = path_normalize(file) - report("log", 1, "db", - "loading font", "%s", file) - load_font(file, fontnames, newfontnames, texmf) - end + "fonts found", "%d fonts found in '%s'", n_found, dirname) + return n_found end local function scan_texmf_fonts(fontnames, newfontnames) @@ -942,8 +944,6 @@ local function get_os_dirs() else local passed_paths = {} local os_dirs = {} - -- what about ~/config/fontconfig/fonts.conf etc? - -- Answer: they should be included by the others, please report if it's not for _,p in next, {"/usr/local/etc/fonts/fonts.conf", "/etc/fonts/fonts.conf"} do if lfs.isfile(p) then read_fonts_conf(p, os_dirs, passed_paths) @@ -967,13 +967,16 @@ local function scan_os_fonts(fontnames, newfontnames) end end +--- dbobj -> bool -> dbobj update_names = function (fontnames, force) + local starttime = os.gettimeofday() --[[ The main function, scans everything - - fontnames is the final table to return + - “newfontnames” is the final table to return - force is whether we rebuild it from scratch or not ]] - report("info", 1, "db", "Updating the font names database") + report("info", 1, "db", "Updating the font names database" + .. (force and " forcefully" or "")) if force then fontnames = fontnames_init() @@ -998,9 +1001,15 @@ update_names = function (fontnames, force) then scan_os_fonts(fontnames, newfontnames) end + --- stats before rewrite: + --- partial: 1144 ms + --- forced: 45384 ms + report("info", 1, "db", "Rebuilt in %0.f ms", + 1000*(os.gettimeofday()-starttime)) return newfontnames end +--- dbobj -> unit save_names = function (fontnames) local path = names.path.dir if not lfs.isdir(path) then -- cgit v1.2.3 From 29204c74484a36bdc8208860e8e1cbae51a2f002 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 24 Apr 2013 14:58:06 +0200 Subject: pick path normalizer only once per run --- luaotfload-database.lua | 123 +++++++++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 48 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 4ccf4e5..2741ab0 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -45,6 +45,7 @@ local utf8lower = unicode.utf8.lower local dirglob = dir.glob local dirmkdirs = dir.mkdirs local filebasename = file.basename +local filedirname = file.dirname local filecollapsepath = file.collapsepath or file.collapse_path local fileextname = file.extname local fileiswritable = file.iswritable @@ -212,11 +213,14 @@ local scan_external_dir local update_names load_names = function ( ) + local starttime = os.gettimeofday() local foundname, data = load_lua_file(names.path.path) if data then report("info", 1, "db", "Font names database loaded", "%s", foundname) + report("info", 1, "db", "Loading took %0.f ms", + 1000*(os.gettimeofday()-starttime)) else report("info", 0, "db", [[Font names database not found, generating new one. @@ -622,6 +626,7 @@ font_fullinfo = function (filename, subfont, texmf) return tfmdata end +----- load_font(file, fontnames, newfontnames, texmf) local load_font = function (filename, fontnames, newfontnames, texmf) local newmappings = newfontnames.mappings local newstatus = newfontnames.status @@ -632,7 +637,7 @@ local load_font = function (filename, fontnames, newfontnames, texmf) if filename then if names.blacklist[filename] or names.blacklist[basename] then - report("log", 2, "db", "ignoring font", "%s", filename) + report("log", 2, "db", "ignoring blacklisted font “%s”", filename) return end local timestamp, db_timestamp @@ -655,7 +660,7 @@ local load_font = function (filename, fontnames, newfontnames, texmf) newmappings[#newmappings+1] = mappings[v] newstatus[basefile].index[index+1] = #newmappings end - report("log", 1, "db", "font already indexed", "%s", basefile) + report("log", 2, "db", "font “%s” already indexed", basefile) return end local info = fontloader.info(filename) @@ -690,39 +695,66 @@ local load_font = function (filename, fontnames, newfontnames, texmf) newstatus[basefile].index[1] = index end else - report("log", 1, "db", "failed to load", "%s", basefile) + report("log", 1, "db", "failed to load “%s”", basefile) end end end -local function path_normalize(path) - --[[ - path normalization: - - a\b\c -> a/b/c - - a/../b -> b - - /cygdrive/a/b -> a:/b - - reading symlinks under non-Win32 - - using kpse.readable_file on Win32 - ]] - if os.type == "windows" or os.type == "msdos" or os.name == "cygwin" then - path = stringgsub(path, '\\', '/') - path = stringlower(path) - path = stringgsub(path, '^/cygdrive/(%a)/', '%1:/') - end - if os.type ~= "windows" and os.type ~= "msdos" then - local dest = lfs.readlink(path) - if dest then - if kpsereadable_file(dest) then - path = dest - elseif kpsereadable_file(filejoin(file.dirname(path), dest)) then - path = filejoin(file.dirname(path), dest) - else - -- broken symlink? +local path_normalize +do + --- os.type and os.name are constants so we + --- choose a normalization function in advance + --- instead of testing with every call + local os_type, os_name = os.type, os.name + local filecollapsepath = filecollapsepath + local lfsreadlink = lfs.readlink + + --- windows and dos + if os_type == "windows" or os_type == "msdos" then + --- ms platfom specific stuff + path_normalize = function (path) + path = stringgsub(path, '\\', '/') + path = stringlower(path) + path = stringgsub(path, '^/cygdrive/(%a)/', '%1:/') + path = filecollapsepath(path) + return path + end + + elseif os_name == "cygwin" then -- union of ms + unix + path_normalize = function (path) + path = stringgsub(path, '\\', '/') + path = stringlower(path) + path = stringgsub(path, '^/cygdrive/(%a)/', '%1:/') + local dest = lfsreadlink(path) + if dest then + if kpsereadable_file(dest) then + path = dest + elseif kpsereadable_file(filejoin(filedirname(path), dest)) then + path = filejoin(file.dirname(path), dest) + else + -- broken symlink? + end + end + path = filecollapsepath(path) + return path + end + + else -- posix + path_normalize = function (path) + local dest = lfsreadlink(path) + if dest then + if kpsereadable_file(dest) then + path = dest + elseif kpsereadable_file(filejoin(filedirname(path), dest)) then + path = filejoin(file.dirname(path), dest) + else + -- broken symlink? + end end + path = filecollapsepath(path) + return path end end - path = filecollapsepath(path) - return path end fonts.path_normalize = path_normalize @@ -750,7 +782,7 @@ local function read_blacklist() if stringsub(line, 1, 1) == "-" then whitelist[stringsub(line, 2, -1)] = true else - report("log", 2, "db", "blacklisted file", "%s", line) + report("log", 2, "db", "blacklisted file “%s”", line) blacklist[line] = true end end @@ -774,34 +806,28 @@ local scan_dir = function (dirname, fontnames, newfontnames, texmf) This function scans a directory and populates the list of fonts with all the fonts it finds. - dirname is the name of the directory to scan - - names is the font database to fill + - names is the font database to fill -> no such term!!! - texmf is a boolean saying if we are scanning a texmf directory - - srsly guys, calling a variable “list” is just lazy! ]] - --- list: string list local n_found = 0 --- total of fonts collected report("log", 2, "db", "scanning", "%s", dirname) for _,i in next, font_extensions do for _,ext in next, { i, stringupper(i) } do local found = dirglob(stringformat("%s/**.%s$", dirname, ext)) local n_new = #found - --- note that glob fails silently on broken symlinks, which happens - --- sometimes in TeX Live. - report("log", 2, "db", - "fonts found", "%s '%s' fonts found", #found, ext) + --- note that glob fails silently on broken symlinks, which + --- happens sometimes in TeX Live. + report("log", 2, "db", "%s '%s' fonts found", n_new, ext) n_found = n_found + n_new for j=1, n_new do - local file = found[j] - file = path_normalize(file) - report("log", 1, "db", "loading font", "%s", file) - load_font(file, fontnames, newfontnames, texmf) + local filename = found[j] + filename = path_normalize(filename) + report("log", 2, "db", "loading font “%s”", filename) + load_font(filename, fontnames, newfontnames, texmf) end - --tableappend(list, found) end end - report("log", 2, "db", - "fonts found", "%d fonts found in '%s'", n_found, dirname) + report("log", 2, "db", "%d fonts found in '%s'", n_found, dirname) return n_found end @@ -853,7 +879,7 @@ read_fonts_conf = function (path, results, passed_paths) passed_paths[#passed_paths+1] = path passed_paths_set = tabletohash(passed_paths, true) if not fh then - report("log", 2, "db", "cannot open file", "%s", path) + report("log", 2, "db", "cannot open file %s", path) return results end local incomments = false @@ -1001,9 +1027,10 @@ update_names = function (fontnames, force) then scan_os_fonts(fontnames, newfontnames) end - --- stats before rewrite: - --- partial: 1144 ms - --- forced: 45384 ms + --- stats: + --- before rewrite | after rewrite + --- partial: 804 ms | 701 ms + --- forced: 45384 ms | 44714 ms report("info", 1, "db", "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) return newfontnames -- cgit v1.2.3 From a70a14dcbcdaf570bbc235a45c7fc5dc842efb11 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 24 Apr 2013 16:07:38 +0200 Subject: collect stats about new additions --- luaotfload-database.lua | 193 ++++++++++++++++++++++++++++-------------------- 1 file changed, 114 insertions(+), 79 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 2741ab0..50f3e08 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -18,6 +18,7 @@ local pcall = pcall local require = require local tonumber = tonumber +local fontloaderinfo = fontloader.info local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -156,7 +157,7 @@ local fontnames_init = function ( ) return { mappings = { }, status = { }, - filenames = { }, --- maybe overkill + filenames = { }, --- (basename, fullname) hash; maybe overkill version = names.version, } end @@ -626,78 +627,95 @@ font_fullinfo = function (filename, subfont, texmf) return tfmdata end ------ load_font(file, fontnames, newfontnames, texmf) -local load_font = function (filename, fontnames, newfontnames, texmf) - local newmappings = newfontnames.mappings - local newstatus = newfontnames.status - local mappings = fontnames.mappings - local status = fontnames.status - local basename = filebasename(filename) - local basefile = texmf and basename or filename - if filename then - if names.blacklist[filename] or - names.blacklist[basename] then - report("log", 2, "db", "ignoring blacklisted font “%s”", filename) - return - end - local timestamp, db_timestamp - db_timestamp = status[basefile] and status[basefile].timestamp - timestamp = lfs.attributes(filename, "modification") - - local index_status = newstatus[basefile] or (not texmf and newstatus[basename]) - if index_status and index_status.timestamp == timestamp then - -- already indexed this run - return - end +--- we return true if the fond is new or re-indexed +--- string -> dbobj -> dbobj -> bool -> bool +local load_font = function (fullname, fontnames, newfontnames, texmf) + local newmappings = newfontnames.mappings + local newstatus = newfontnames.status + local mappings = fontnames.mappings + local status = fontnames.status + local filenames = fontnames.filenames + local basename = filebasename(fullname) + --- entryname is apparently the identifier a font is + --- loaded by; it is different for files in the texmf + --- (due to kpse? idk.) + --- entryname = texmf : true -> basename | false -> fullname + local entryname = texmf and basename or fullname + + if not fullname then return false end + + if names.blacklist[fullname] + or names.blacklist[basename] + then + report("log", 2, "db", + "ignoring blacklisted font “%s”", fullname) + return false + end + local timestamp, db_timestamp + db_timestamp = status[entryname] + and status[entryname].timestamp + timestamp = lfs.attributes(fullname, "modification") + + local index_status = newstatus[entryname] + or (not texmf and newstatus[basename]) + local teststat = newstatus[entryname] + --- index_status: nil | false | table + if index_status and index_status.timestamp == timestamp then + -- already indexed this run + return false + end - newstatus[basefile] = newstatus[basefile] or { } - newstatus[basefile].timestamp = timestamp - newstatus[basefile].index = newstatus[basefile].index or { } + newstatus[entryname] = newstatus[entryname] or { } + newstatus[entryname].timestamp = timestamp + newstatus[entryname].index = newstatus[entryname].index or { } - if db_timestamp == timestamp and not newstatus[basefile].index[1] then - for _,v in next, status[basefile].index do - local index = #newstatus[basefile].index - newmappings[#newmappings+1] = mappings[v] - newstatus[basefile].index[index+1] = #newmappings - end - report("log", 2, "db", "font “%s” already indexed", basefile) - return + if db_timestamp == timestamp and not newstatus[entryname].index[1] then + for _,v in next, status[entryname].index do + local index = #newstatus[entryname].index + newmappings[#newmappings+1] = mappings[v] + newstatus[entryname].index[index+1] = #newmappings end - local info = fontloader.info(filename) - if info then - if type(info) == "table" and #info > 1 then - for i in next, info do - local fullinfo = font_fullinfo(filename, i-1, texmf) - if not fullinfo then - return - end - local index = newstatus[basefile].index[i] - if newstatus[basefile].index[i] then - index = newstatus[basefile].index[i] - else - index = #newmappings+1 - end - newmappings[index] = fullinfo - newstatus[basefile].index[i] = index - end - else - local fullinfo = font_fullinfo(filename, false, texmf) + report("log", 2, "db", "font “%s” already indexed", entryname) + return false + end + local info = fontloaderinfo(fullname) + + if info then + if type(info) == "table" and #info > 1 then + for i in next, info do + local fullinfo = font_fullinfo(fullname, i-1, texmf) if not fullinfo then - return + return false end - local index - if newstatus[basefile].index[1] then - index = newstatus[basefile].index[1] + local index = newstatus[entryname].index[i] + if newstatus[entryname].index[i] then + index = newstatus[entryname].index[i] else index = #newmappings+1 end newmappings[index] = fullinfo - newstatus[basefile].index[1] = index + newstatus[entryname].index[i] = index end else - report("log", 1, "db", "failed to load “%s”", basefile) + local fullinfo = font_fullinfo(fullname, false, texmf) + if not fullinfo then + return false + end + local index + if newstatus[entryname].index[1] then + index = newstatus[entryname].index[1] + else + index = #newmappings+1 + end + newmappings[index] = fullinfo + newstatus[entryname].index[1] = index end + + else --- missing info + report("log", 1, "db", "failed to load “%s”", entryname) + return false end + return true end local path_normalize @@ -800,7 +818,7 @@ for key, value in next, font_extensions do font_extensions_set[value] = true end ---- string -> dbobj -> dbobj -> bool -> int +--- string -> dbobj -> dbobj -> bool -> (int * int) local scan_dir = function (dirname, fontnames, newfontnames, texmf) --[[ This function scans a directory and populates the list of fonts @@ -809,29 +827,31 @@ local scan_dir = function (dirname, fontnames, newfontnames, texmf) - names is the font database to fill -> no such term!!! - texmf is a boolean saying if we are scanning a texmf directory ]] - local n_found = 0 --- total of fonts collected + local n_scanned, n_new = 0, 0 --- total of fonts collected report("log", 2, "db", "scanning", "%s", dirname) for _,i in next, font_extensions do for _,ext in next, { i, stringupper(i) } do local found = dirglob(stringformat("%s/**.%s$", dirname, ext)) - local n_new = #found + local n_found = #found --- note that glob fails silently on broken symlinks, which --- happens sometimes in TeX Live. - report("log", 2, "db", "%s '%s' fonts found", n_new, ext) - n_found = n_found + n_new - for j=1, n_new do - local filename = found[j] - filename = path_normalize(filename) - report("log", 2, "db", "loading font “%s”", filename) - load_font(filename, fontnames, newfontnames, texmf) + report("log", 2, "db", "%s '%s' fonts found", n_found, ext) + n_scanned = n_scanned + n_found + for j=1, n_found do + local fullname = found[j] + fullname = path_normalize(fullname) + report("log", 2, "db", "loading font “%s”", fullname) + local new = load_font(fullname, fontnames, newfontnames, texmf) + if new then n_new = n_new + 1 end end end end - report("log", 2, "db", "%d fonts found in '%s'", n_found, dirname) - return n_found + report("log", 2, "db", "%d fonts found in '%s'", n_scanned, dirname) + return n_scanned, n_new end local function scan_texmf_fonts(fontnames, newfontnames) + local n_scanned, n_new = 0, 0 --[[ This function scans all fonts in the texmf tree, through kpathsea variables OPENTYPEFONTS and TTFONTS of texmf.cnf @@ -845,9 +865,12 @@ local function scan_texmf_fonts(fontnames, newfontnames) fontdirs = fontdirs .. stringgsub(kpseexpand_path("$TTFONTS"), "^%.", "") if not stringis_empty(fontdirs) then for _,d in next, filesplitpath(fontdirs) do - scan_dir(d, fontnames, newfontnames, true) + local found, new = scan_dir(d, fontnames, newfontnames, true) + n_scanned = n_scanned + found + n_new = n_new + new end end + return n_scanned, n_new end --[[ @@ -981,6 +1004,7 @@ local function get_os_dirs() end local function scan_os_fonts(fontnames, newfontnames) + local n_scanned, n_new = 0, 0 --[[ This function scans the OS fonts through - fontcache for Unix (reads the fonts.conf file and scans the directories) @@ -989,13 +1013,17 @@ local function scan_os_fonts(fontnames, newfontnames) report("info", 1, "db", "Scanning OS fonts...") report("info", 2, "db", "Searching in static system directories...") for _,d in next, get_os_dirs() do - scan_dir(d, fontnames, newfontnames, false) + local found, new = scan_dir(d, fontnames, newfontnames, false) + n_scanned = n_scanned + found + n_new = n_new + new end + return n_scanned, n_new end --- dbobj -> bool -> dbobj update_names = function (fontnames, force) local starttime = os.gettimeofday() + local n_scanned, n_new = 0, 0 --[[ The main function, scans everything - “newfontnames” is the final table to return @@ -1020,19 +1048,25 @@ update_names = function (fontnames, force) read_blacklist() --installed_fonts_scanned = false --scan_installed_fonts(fontnames, newfontnames) --- see fixme above - scan_texmf_fonts(fontnames, newfontnames) + local scanned, new = scan_texmf_fonts(fontnames, newfontnames) + n_scanned = n_scanned + scanned + n_new = n_new + new --if not installed_fonts_scanned --and stringis_empty(kpseexpand_path("$OSFONTDIR")) if stringis_empty(kpseexpand_path("$OSFONTDIR")) then - scan_os_fonts(fontnames, newfontnames) + local scanned, new = scan_os_fonts(fontnames, newfontnames) + n_scanned = n_scanned + scanned + n_new = n_new + new end --- stats: --- before rewrite | after rewrite --- partial: 804 ms | 701 ms --- forced: 45384 ms | 44714 ms - report("info", 1, "db", "Rebuilt in %0.f ms", - 1000*(os.gettimeofday()-starttime)) + report("info", 1, "db", + "Scanned %d font files; %d new entries.", n_scanned, n_new) + report("info", 1, "db", + "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) return newfontnames end @@ -1064,8 +1098,9 @@ scan_external_dir = function (dir) fonts_loaded = true end new_names = tablecopy(old_names) - scan_dir(dir, old_names, new_names) + local n_scanned, n_new = scan_dir(dir, old_names, new_names) names.data = new_names + return n_scanned, n_new end --- export functionality to the namespace “fonts.names” -- cgit v1.2.3 From 24a3ecb19d3ca204ac4432e8f2f0b2bf53954b05 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 24 Apr 2013 18:58:02 +0200 Subject: catch irregular ``file:`` lookups we now index base names and extensionless base names of font files as well so as to work around quirks of the Xetex compatibility layer. this will probably get removed after the syntax parser is redone. --- luaotfload-database.lua | 144 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 101 insertions(+), 43 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 50f3e08..b8a993a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -46,6 +46,7 @@ local utf8lower = unicode.utf8.lower local dirglob = dir.glob local dirmkdirs = dir.mkdirs local filebasename = file.basename +local filenameonly = file.nameonly local filedirname = file.dirname local filecollapsepath = file.collapsepath or file.collapse_path local fileextname = file.extname @@ -83,32 +84,6 @@ end names.path.dir = writable_path names.path.path = filejoin(writable_path, names.path.basename) - ----- ---- ---- these lines load some binary module called “lualatex-platform” ---- that doesn’t appear to build with Lua 5.2. I’m going ahead and ---- disable it for the time being until someone clarifies what it ---- is supposed to do and whether we should care to fix it. ---- ---local success = pcall(require, "luatexbase.modutils") ---if success then --- success = pcall(luatexbase.require_module, --- "lualatex-platform", "2011/03/30") --- print(success) ---end - ---local get_installed_fonts ---if success then --- get_installed_fonts = lualatex.platform.get_installed_fonts ---else --- function get_installed_fonts() --- end ---end ----- - -local get_installed_fonts = nil - --[[doc-- Auxiliary functions --doc]]-- @@ -157,7 +132,12 @@ local fontnames_init = function ( ) return { mappings = { }, status = { }, - filenames = { }, --- (basename, fullname) hash; maybe overkill + --- adding filename mapping increases the + --- size of the serialized db on my system + --- (5840 font files) by a factor of ... + barenames = { },--- incr. by 1.11 + basenames = { },--- incr. by 1.22 +-- fullnames = { },--- incr. by 1.48 version = names.version, } end @@ -259,6 +239,55 @@ end local fonts_loaded = false local fonts_reloaded = false +local crude_file_lookup_verbose = function (data, filename) + local found = data.barenames[filename] + if found then + report("info", 0, "db", + "crude file lookup: req=%s; hit=bare; ret=%s", + filename, found[1]) + return found + end +-- found = data.fullnames[filename] +-- if found then +-- report("info", 0, "db", +-- "crude file lookup: req=%s; hit=bare; ret=%s", +-- filename, found[1]) +-- return found +-- end + found = data.basenames[filename] + if found then + report("info", 0, "db", + "crude file lookup: req=%s; hit=bare; ret=%s", + filename, found[1]) + return found + end + found = resolvers.findfile(filename, "tfm") + if found then + report("info", 0, "db", + "crude file lookup: req=tfm; hit=bare; ret=%s", found) + return { found, false } + end + found = resolvers.findfile(filename, "ofm") + if found then + report("info", 0, "db", + "crude file lookup: req=ofm; hit=bare; ret=%s", found) + return { found, false } + end + return false +end + +local crude_file_lookup = function (data, filename) + local found = data.barenames[filename] +-- or data.fullnames[filename] + or data.basenames[filename] + if found then return found end + found = resolvers.findfile(filename, "tfm") + if found then return { found, false } end + found = resolvers.findfile(filename, "ofm") + if found then return { found, false } end + return false +end + --[[doc-- Luatex-fonts, the font-loader package luaotfload imports, comes with @@ -285,8 +314,10 @@ font database created by the mkluatexfontdb script. --- · specification: string (== ":" ) --- · sub: string --- ---- the return value of “resolve” is the file name of the requested ---- font +--- the first return value of “resolve” is the file name of the +--- requested font (string) +--- the second is of type bool or string and indicates the subfont of a +--- ttc --- --- 'a -> 'a -> table -> (string * string | bool * bool) --- @@ -296,6 +327,18 @@ font database created by the mkluatexfontdb script. --- --- resolve = function (_,_,specification) -- the 1st two parameters are used by ConTeXt + if not fonts_loaded then + names.data = load_names() + fonts_loaded = true + end + local data = names.data + + if specification.lookup == "file" then + local found = crude_file_lookup(data, specification.name) + --local found = crude_file_lookup_verbose(data, specification.name) + if found then return found[1], found[2], true end + end + local name = sanitize_string(specification.name) local style = sanitize_string(specification.style) or "regular" @@ -306,12 +349,6 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con size = specification.size / 65536 end - if not fonts_loaded then - names.data = load_names() - fonts_loaded = true - end - - local data = names.data if type(data) == "table" then local db_version, nms_version = data.version, names.version if data.version ~= names.version then @@ -632,10 +669,20 @@ end local load_font = function (fullname, fontnames, newfontnames, texmf) local newmappings = newfontnames.mappings local newstatus = newfontnames.status + +-- local newfullnames = newfontnames.fullnames + local newbasenames = newfontnames.basenames + local newbarenames = newfontnames.barenames + local mappings = fontnames.mappings local status = fontnames.status - local filenames = fontnames.filenames +-- local fullnames = fontnames.fullnames + local basenames = fontnames.basenames + local barenames = fontnames.barenames + local basename = filebasename(fullname) + local barename = filenameonly(fullname) + --- entryname is apparently the identifier a font is --- loaded by; it is different for files in the texmf --- (due to kpse? idk.) @@ -669,19 +716,24 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) newstatus[entryname].timestamp = timestamp newstatus[entryname].index = newstatus[entryname].index or { } - if db_timestamp == timestamp and not newstatus[entryname].index[1] then + if db_timestamp == timestamp + and not newstatus[entryname].index[1] then for _,v in next, status[entryname].index do - local index = #newstatus[entryname].index - newmappings[#newmappings+1] = mappings[v] + local index = #newstatus[entryname].index + local fullinfo = mappings[v] + newmappings[#newmappings+1] = fullinfo --- keep newstatus[entryname].index[index+1] = #newmappings +-- newfullnames[fullname] = fullinfo.filename + newbasenames[basename] = fullinfo.filename + newbarenames[barename] = fullinfo.filename end report("log", 2, "db", "font “%s” already indexed", entryname) return false end - local info = fontloaderinfo(fullname) + local info = fontloaderinfo(fullname) if info then - if type(info) == "table" and #info > 1 then + if type(info) == "table" and #info > 1 then --- ttc for i in next, info do local fullinfo = font_fullinfo(fullname, i-1, texmf) if not fullinfo then @@ -693,7 +745,10 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) else index = #newmappings+1 end - newmappings[index] = fullinfo + newmappings[index] = fullinfo +-- newfullnames[fullname] = fullinfo.filename + newbasenames[basename] = fullinfo.filename + newbarenames[barename] = fullinfo.filename newstatus[entryname].index[i] = index end else @@ -707,7 +762,10 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) else index = #newmappings+1 end - newmappings[index] = fullinfo + newmappings[index] = fullinfo +-- newfullnames[fullname] = { fullinfo.filename[1], fullinfo.filename[2] } + newbasenames[basename] = { fullinfo.filename[1], fullinfo.filename[2] } + newbarenames[barename] = { fullinfo.filename[1], fullinfo.filename[2] } newstatus[entryname].index[1] = index end -- cgit v1.2.3 From 988854282eb83a9c8bdcc0f97ac85937bb8f9ff8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 24 Apr 2013 22:40:07 +0200 Subject: make db reloading require a reason; decrease nesting depth of conditionals --- luaotfload-database.lua | 373 +++++++++++++++++++++++++----------------------- 1 file changed, 194 insertions(+), 179 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index b8a993a..d731038 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -193,6 +193,7 @@ local save_names local scan_external_dir local update_names +--- unit -> dbobj load_names = function ( ) local starttime = os.gettimeofday() local foundname, data = load_lua_file(names.path.path) @@ -349,146 +350,162 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con size = specification.size / 65536 end - if type(data) == "table" then - local db_version, nms_version = data.version, names.version - if data.version ~= names.version then - report("log", 0, "db", - [[version mismatch; expected %4.3f, got %4.3f]], - nms_version, db_version - ) - return reload_db(resolve, nil, nil, specification) + if type(data) ~= "table" then + --- this catches a case where load_names() doesn’t + --- return a database object, which can happen only + --- in case there is valid Lua code in the database, + --- but it’s not a table, e.g. it contains an integer. + if not fonts_reloaded then + return reload_db("invalid database; not a table", + resolve, nil, nil, specification + ) end - if data.mappings then - local found = { } - local synonym_set = style_synonyms.set - for _,face in next, data.mappings do - --- TODO we really should store those in dedicated - --- .sanitized field - local family = sanitize_string(face.names and face.names.family) - local subfamily = sanitize_string(face.names and face.names.subfamily) - local fullname = sanitize_string(face.names and face.names.fullname) - local psname = sanitize_string(face.names and face.names.psname) - local fontname = sanitize_string(face.fontname) - local pfullname = sanitize_string(face.fullname) - local optsize, dsnsize, maxsize, minsize - if #face.size > 0 then - optsize = face.size - dsnsize = optsize[1] and optsize[1] / 10 - -- can be nil - maxsize = optsize[2] and optsize[2] / 10 or dsnsize - minsize = optsize[3] and optsize[3] / 10 or dsnsize - end - if name == family then - if subfamily == style then - if optsize then - if dsnsize == size - or (size > minsize and size <= maxsize) then - found[1] = face - break - else - found[#found+1] = face - end - else - found[1] = face - break - end - elseif synonym_set[style] and - synonym_set[style][subfamily] then - if optsize then - if dsnsize == size - or (size > minsize and size <= maxsize) then - found[1] = face - break - else - found[#found+1] = face - end - else - found[1] = face - break - end - elseif subfamily == "regular" or - synonym_set.regular[subfamily] then - found.fallback = face + --- unsucessfully reloaded; bail + return specification.name, false, false + end + + local db_version, nms_version = data.version, names.version + if db_version ~= nms_version then + report("log", 0, "db", + [[version mismatch; expected %4.3f, got %4.3f]], + nms_version, db_version + ) + return reload_db("version mismatch", resolve, nil, nil, specification) + end + + if not data.mappings then + return reload_db("invalid database; missing font mapping", + resolve, nil, nil, specification + ) + end + + local found = { } + local synonym_set = style_synonyms.set + for _,face in next, data.mappings do + --- TODO we really should store those in dedicated + --- .sanitized field + local family = sanitize_string(face.names and face.names.family) + local subfamily = sanitize_string(face.names and face.names.subfamily) + local fullname = sanitize_string(face.names and face.names.fullname) + local psname = sanitize_string(face.names and face.names.psname) + local fontname = sanitize_string(face.fontname) + local pfullname = sanitize_string(face.fullname) + local optsize, dsnsize, maxsize, minsize + if #face.size > 0 then + optsize = face.size + dsnsize = optsize[1] and optsize[1] / 10 + -- can be nil + maxsize = optsize[2] and optsize[2] / 10 or dsnsize + minsize = optsize[3] and optsize[3] / 10 or dsnsize + end + if name == family then + if subfamily == style then + if optsize then + if dsnsize == size + or (size > minsize and size <= maxsize) then + found[1] = face + break + else + found[#found+1] = face end else - if name == fullname - or name == pfullname - or name == fontname - or name == psname then - if optsize then - if dsnsize == size - or (size > minsize and size <= maxsize) then - found[1] = face - break - else - found[#found+1] = face - end - else - found[1] = face - break - end + found[1] = face + break + end + elseif synonym_set[style] and + synonym_set[style][subfamily] then + if optsize then + if dsnsize == size + or (size > minsize and size <= maxsize) then + found[1] = face + break + else + found[#found+1] = face end + else + found[1] = face + break end + elseif subfamily == "regular" or + synonym_set.regular[subfamily] then + found.fallback = face end - if #found == 1 then - if kpselookup(found[1].filename[1]) then - report("log", 0, "resolve", - "font family='%s', subfamily='%s' found: %s", - name, style, found[1].filename[1] - ) - return found[1].filename[1], found[1].filename[2], true - end - elseif #found > 1 then - -- we found matching font(s) but not in the requested optical - -- sizes, so we loop through the matches to find the one with - -- least difference from the requested size. - local closest - local least = math.huge -- initial value is infinity - for i,face in next, found do - local dsnsize = face.size[1]/10 - local difference = mathabs(dsnsize-size) - if difference < least then - closest = face - least = difference + else + if name == fullname + or name == pfullname + or name == fontname + or name == psname then + if optsize then + if dsnsize == size + or (size > minsize and size <= maxsize) then + found[1] = face + break + else + found[#found+1] = face end + else + found[1] = face + break end - if kpselookup(closest.filename[1]) then - report("log", 0, "resolve", - "font family='%s', subfamily='%s' found: %s", - name, style, closest.filename[1] - ) - return closest.filename[1], closest.filename[2], true - end - elseif found.fallback then - return found.fallback.filename[1], found.fallback.filename[2], true end - --- no font found so far - if not fonts_reloaded then - --- last straw: try reloading the database - return reload_db(resolve, nil, nil, specification) - else - --- else, fallback to requested name - --- specification.name is empty with absolute paths, looks - --- like a bug in the specification parser 1 then + -- we found matching font(s) but not in the requested optical + -- sizes, so we loop through the matches to find the one with + -- least difference from the requested size. + local closest + local least = math.huge -- initial value is infinity + for i,face in next, found do + local dsnsize = face.size[1]/10 + local difference = mathabs(dsnsize-size) + if difference < least then + closest = face + least = difference end end - else --- no db or outdated; reload names and retry - if not fonts_reloaded then - return reload_db(resolve, nil, nil, specification) - else --- unsucessfully reloaded; bail - return specification.name, false, false + if kpselookup(closest.filename[1]) then + report("log", 0, "resolve", + "font family='%s', subfamily='%s' found: %s", + name, style, closest.filename[1] + ) + return closest.filename[1], closest.filename[2], true end + elseif found.fallback then + return found.fallback.filename[1], found.fallback.filename[2], true + end + + --- no font found so far + if not fonts_reloaded then + --- last straw: try reloading the database + return reload_db( + "unresoled font name: “" .. name .. "”", + resolve, nil, nil, specification + ) end + + --- else, fallback to requested name + --- specification.name is empty with absolute paths, looks + --- like a bug in the specification parser 'a) -> 'a list -> 'a -reload_db = function (caller, ...) - report("log", 1, "db", "reload initiated") +--- string -> ('a -> 'a) -> 'a list -> 'a +reload_db = function (why, caller, ...) + report("log", 1, "db", "reload initiated; reason: “%s”", why) names.data = update_names() save_names(names.data) fonts_reloaded = true @@ -535,67 +552,65 @@ find_closest = function (name, limit) local data = names.data - if type(data) == "table" then - local by_distance = { } --- (int, string list) dict - local distances = { } --- int list - local cached = { } --- (string, int) dict - local mappings = data.mappings - local n_fonts = #mappings - - for n = 1, n_fonts do - local current = mappings[n] - local cnames = current.names - --[[ - This is simplistic but surpisingly fast. - Matching is performed against the “family” name - of a db record. We then store its “fullname” at - it edit distance. - We should probably do some weighting over all the - font name categories as well as whatever agrep - does. - --]] - if cnames then - local fullname, family = cnames.fullname, cnames.family - family = sanitize_string(family) - - local dist = cached[family]--- maybe already calculated - if not dist then - dist = iterative_levenshtein(name, family) - cached[family] = dist - end - local namelst = by_distance[dist] - if not namelst then --- first entry - namelst = { fullname } - distances[#distances+1] = dist - else --- append - namelst[#namelst+1] = fullname - end - by_distance[dist] = namelst + if type(data) ~= "table" then + return reload_db("no database", find_closest, name) + end + local by_distance = { } --- (int, string list) dict + local distances = { } --- int list + local cached = { } --- (string, int) dict + local mappings = data.mappings + local n_fonts = #mappings + + for n = 1, n_fonts do + local current = mappings[n] + local cnames = current.names + --[[ + This is simplistic but surpisingly fast. + Matching is performed against the “family” name + of a db record. We then store its “fullname” at + it edit distance. + We should probably do some weighting over all the + font name categories as well as whatever agrep + does. + --]] + if cnames then + local fullname, family = cnames.fullname, cnames.family + family = sanitize_string(family) + + local dist = cached[family]--- maybe already calculated + if not dist then + dist = iterative_levenshtein(name, family) + cached[family] = dist end - end - - --- print the matches according to their distance - local n_distances = #distances - if n_distances > 0 then --- got some data - tablesort(distances) - limit = mathmin(n_distances, limit) - report(false, 1, "query", - "displaying %d distance levels", limit) - - for i = 1, limit do - local dist = distances[i] - local namelst = by_distance[dist] - report(false, 0, "query", - "distance from “" .. name .. "”: " .. dist - .. "\n " .. tableconcat(namelst, "\n ") - ) + local namelst = by_distance[dist] + if not namelst then --- first entry + namelst = { fullname } + distances[#distances+1] = dist + else --- append + namelst[#namelst+1] = fullname end + by_distance[dist] = namelst + end + end - return true + --- print the matches according to their distance + local n_distances = #distances + if n_distances > 0 then --- got some data + tablesort(distances) + limit = mathmin(n_distances, limit) + report(false, 1, "query", + "displaying %d distance levels", limit) + + for i = 1, limit do + local dist = distances[i] + local namelst = by_distance[dist] + report(false, 0, "query", + "distance from “" .. name .. "”: " .. dist + .. "\n " .. tableconcat(namelst, "\n ") + ) end - return false - else --- need reload - return reload_db(find_closest, name) + + return true end return false end --- find_closest() -- cgit v1.2.3 From c22e7fa7a129c4f3496753657d36b651ba6d1b39 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 25 Apr 2013 04:25:53 +0200 Subject: accommodate weird subfamily type of Libertine Mono --- luaotfload-database.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index d731038..666172a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -398,6 +398,7 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con maxsize = optsize[2] and optsize[2] / 10 or dsnsize minsize = optsize[3] and optsize[3] / 10 or dsnsize end + if name == family then if subfamily == style then if optsize then @@ -413,7 +414,8 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con break end elseif synonym_set[style] and - synonym_set[style][subfamily] then + synonym_set[style][subfamily] + then if optsize then if dsnsize == size or (size > minsize and size <= maxsize) then @@ -427,9 +429,11 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con break end elseif subfamily == "regular" or + subfamily == "mono" or --> Libertine Mono O synonym_set.regular[subfamily] then found.fallback = face end + else if name == fullname or name == pfullname -- cgit v1.2.3 From 4d0d2c19ab36d4918a72041a087fbcb451ac8c52 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 25 Apr 2013 10:35:05 +0200 Subject: more reliable guard for Libertine Mono --- luaotfload-database.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 666172a..4044e8e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -429,9 +429,13 @@ resolve = function (_,_,specification) -- the 1st two parameters are used by Con break end elseif subfamily == "regular" or - subfamily == "mono" or --> Libertine Mono O synonym_set.regular[subfamily] then found.fallback = face + elseif name == fullname then + --- happens with Libertine Mono which has + --- “mono” as subfamily + found[1] = face + break end else -- cgit v1.2.3 From 7bf46ad63839d6e38a9c1e11ec7d4fba27d1bc51 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Thu, 25 Apr 2013 11:25:18 +0200 Subject: add ``--alias`` option to fontdbutil --- luaotfload-database.lua | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 4044e8e..e8f3d1d 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -201,10 +201,10 @@ load_names = function ( ) if data then report("info", 1, "db", "Font names database loaded", "%s", foundname) - report("info", 1, "db", "Loading took %0.f ms", + report("info", 3, "db", "Loading took %0.f ms", 1000*(os.gettimeofday()-starttime)) else - report("info", 0, "db", + report("info", 1, "db", [[Font names database not found, generating new one. This can take several minutes; please be patient.]]) data = update_names(fontnames_init()) @@ -938,9 +938,9 @@ local function scan_texmf_fonts(fontnames, newfontnames) variables OPENTYPEFONTS and TTFONTS of texmf.cnf ]] if stringis_empty(kpseexpand_path("$OSFONTDIR")) then - report("info", 1, "db", "Scanning TEXMF fonts...") + report("info", 2, "db", "Scanning TEXMF fonts...") else - report("info", 1, "db", "Scanning TEXMF and OS fonts...") + report("info", 2, "db", "Scanning TEXMF and OS fonts...") end local fontdirs = stringgsub(kpseexpand_path("$OPENTYPEFONTS"), "^%.", "") fontdirs = fontdirs .. stringgsub(kpseexpand_path("$TTFONTS"), "^%.", "") @@ -1091,8 +1091,8 @@ local function scan_os_fonts(fontnames, newfontnames) - fontcache for Unix (reads the fonts.conf file and scans the directories) - a static set of directories for Windows and MacOSX ]] - report("info", 1, "db", "Scanning OS fonts...") - report("info", 2, "db", "Searching in static system directories...") + report("info", 2, "db", "Scanning OS fonts...") + report("info", 3, "db", "Searching in static system directories...") for _,d in next, get_os_dirs() do local found, new = scan_dir(d, fontnames, newfontnames, false) n_scanned = n_scanned + found @@ -1110,7 +1110,7 @@ update_names = function (fontnames, force) - “newfontnames” is the final table to return - force is whether we rebuild it from scratch or not ]] - report("info", 1, "db", "Updating the font names database" + report("info", 2, "db", "Updating the font names database" .. (force and " forcefully" or "")) if force then @@ -1144,9 +1144,9 @@ update_names = function (fontnames, force) --- before rewrite | after rewrite --- partial: 804 ms | 701 ms --- forced: 45384 ms | 44714 ms - report("info", 1, "db", + report("info", 3, "db", "Scanned %d font files; %d new entries.", n_scanned, n_new) - report("info", 1, "db", + report("info", 3, "db", "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) return newfontnames end -- cgit v1.2.3