From 08135c42930576980e9bdae7a82208cb9dcf42f9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 19:40:48 +0200 Subject: [db] new structure --- luaotfload-database.lua | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 755ad52..067b48c 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -318,11 +318,19 @@ mtx-fonts has in names.tma: local fontnames_init = function (formats) --- returns dbobj return { - mappings = { }, - status = { }, + families = { + texmf = { }, + system = { }, + }, + index = { }, -- was: status; map abspath -> mapping + mappings = { }, -- TODO: check if still necessary after rewrite + names = { }, -- filenames = { }, -- created later - version = names.version, - formats = formats, + meta = { + formats = formats, + statistics = { }, + version = names.version, + }, } end @@ -2191,8 +2199,14 @@ local gen_fast_lookups = function (fontnames) local nmappings = #mappings --- this is needlessly complicated due to texmf priorization local filenames = { - bare = { }, - base = { }, + bare = { + system = { }, --- mapped to mapping format -> index in full + texmf = { }, --- mapped to mapping format -> “true” + }, + base = { + system = { }, --- mapped to index in “full” + texmf = { }, --- set; all values are “true” + }, full = { }, --- non-texmf } -- cgit v1.2.3 From 4f04f33f36933aaec7626154e055687b2abe903c Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 19:59:24 +0200 Subject: [db] document new db layout for rewrite --- luaotfload-database.lua | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 067b48c..6e49556 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -238,17 +238,41 @@ end This is a sketch of the luaotfload db: type dbobj = { - formats : string list; // { "otf", "ttf", "ttc", "dfont" } mappings : fontentry list; - status : filestatus; - version : float; - // new in v2.3; these supersede the basenames / barenames - // hashes from v2.2 + index : filestatus; + families : familytable; + names : namedata; // TODO: check for relevance after db is finalized + meta : metadata; filenames : filemap; } + and familytable = { + texmf : (format, familyentry) hash; + system : (format, familyentry) hash; + } + and familyentry = { + regular : sizes; + italic : sizes; + bold : sizes; + bolditalic : sizes; + } + and sizes = { + default = int; // points into mappings or names + other = (int, int) list; // design size -> index entry + } + and metadata = { + formats : string list; // { "otf", "ttf", "ttc", "dfont" } + statistics : TODO; + version : float; + } and filemap = { - base : (string, int) hash; // basename -> idx - bare : (string, int) hash; // barename -> idx + base : { + texmf : (string, int) hash; // basename -> idx + system : (string, int) hash; + }; + bare : { + texmf : (string, (string, int) hash) hash; // format -> (barename -> idx) + system : (string, (string, int) hash) hash; + }; full : (int, string) hash; // idx -> full path } and fontentry = { -- cgit v1.2.3 From d66597559f15ce654845a958e2f905bd502a1ed8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:00:52 +0200 Subject: [db] bump db version number --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 6e49556..dedcd01 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -102,7 +102,7 @@ if config.luaotfload.update_live ~= false then config.luaotfload.update_live = true end -names.version = 2.210 +names.version = 2.4 names.data = nil --- contains the loaded database names.lookups = nil --- contains the lookup cache -- cgit v1.2.3 From 55ba70c6d8f76706f083818e7dd38006e88dcfb7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:06:40 +0200 Subject: =?UTF-8?q?[db]=20introduce=20=E2=80=9Clocal=E2=80=9D=20as=20third?= =?UTF-8?q?=20location=20category?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- luaotfload-database.lua | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index dedcd01..5d662e0 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -238,14 +238,15 @@ end This is a sketch of the luaotfload db: type dbobj = { - mappings : fontentry list; - index : filestatus; - families : familytable; - names : namedata; // TODO: check for relevance after db is finalized - meta : metadata; - filenames : filemap; + families : familytable; + filenames : filemap; + index : filestatus; + mappings : fontentry list; + meta : metadata; + names : namedata; // TODO: check for relevance after db is finalized } and familytable = { + local : (format, familyentry) hash; // specified with include dir texmf : (format, familyentry) hash; system : (format, familyentry) hash; } @@ -256,8 +257,8 @@ This is a sketch of the luaotfload db: bolditalic : sizes; } and sizes = { - default = int; // points into mappings or names - other = (int, int) list; // design size -> index entry + default : int; // points into mappings or names + optical : (int, int) list; // design size -> index entry } and metadata = { formats : string list; // { "otf", "ttf", "ttc", "dfont" } @@ -266,12 +267,14 @@ This is a sketch of the luaotfload db: } and filemap = { base : { - texmf : (string, int) hash; // basename -> idx + local : (string, int) hash; // basename -> idx system : (string, int) hash; + texmf : (string, int) hash; }; bare : { - texmf : (string, (string, int) hash) hash; // format -> (barename -> idx) + local : (string, (string, int) hash) hash; // format -> (barename -> idx) system : (string, (string, int) hash) hash; + texmf : (string, (string, int) hash) hash; }; full : (int, string) hash; // idx -> full path } @@ -295,7 +298,7 @@ This is a sketch of the luaotfload db: size : int list; slant : int; subfont : int; - texmf : bool; + location : local | system | texmf; weight : int; width : int; units_per_em : int; // mainly 1000, but also 2048 or 256 @@ -343,8 +346,9 @@ mtx-fonts has in names.tma: local fontnames_init = function (formats) --- returns dbobj return { families = { - texmf = { }, - system = { }, + ["local"] = { }, + system = { }, + texmf = { }, }, index = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite -- cgit v1.2.3 From 4ae6dab28fdf719fd10ba165d1b2414bef6794d8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:40:33 +0200 Subject: [db] refactor texmf and system font scanning --- luaotfload-database.lua | 74 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 25 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 5d662e0..0139d89 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -240,7 +240,7 @@ This is a sketch of the luaotfload db: type dbobj = { families : familytable; filenames : filemap; - index : filestatus; + status : filestatus; mappings : fontentry list; meta : metadata; names : namedata; // TODO: check for relevance after db is finalized @@ -301,10 +301,11 @@ This is a sketch of the luaotfload db: location : local | system | texmf; weight : int; width : int; - units_per_em : int; // mainly 1000, but also 2048 or 256 + units_per_em : int; // mainly 1000, but also 2048 or 256 } - and filestatus = (fullname, - { index : int list; timestamp : int }) dict + and filestatus = (string, // fullname + { index : int list; // pointer into mappings + timestamp : int; }) dict beware that this is a reconstruction and may be incomplete. @@ -350,7 +351,7 @@ local fontnames_init = function (formats) --- returns dbobj system = { }, texmf = { }, }, - index = { }, -- was: status; map abspath -> mapping + status = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite names = { }, -- filenames = { }, -- created later @@ -1404,17 +1405,22 @@ local loaders = { pfa = t1_fullinfo, } ---- we return true if the fond is new or re-indexed +--- we return true if the font is new or re-indexed --- string -> dbobj -> dbobj -> bool -local load_font = function (fullname, fontnames, newfontnames, texmf) + +local read_font_names = function (fullname, + fontnames, + newfontnames, + texmf) + local newmappings = newfontnames.mappings local newstatus = newfontnames.status --- by full path local mappings = fontnames.mappings local status = fontnames.status - local basename = filebasename(fullname) - local barename = filenameonly(fullname) + local basename = filebasename (fullname) + local barename = filenameonly (fullname) local format = stringlower (filesuffix (basename)) @@ -1438,7 +1444,7 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) local newentrystatus = newstatus[fullname] --- newentrystatus: nil | false | table if newentrystatus and newentrystatus.timestamp == new_timestamp then - -- already indexed this run + -- already statused this run return false end @@ -1824,6 +1830,7 @@ end --doc]]-- --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) + local scan_dir = function (dirname, fontnames, newfontnames, dry_run, texmf) if lpegmatch (p_blacklist, dirname) then @@ -1849,12 +1856,13 @@ local scan_dir = function (dirname, fontnames, newfontnames, local new if dry_run == true then report ("both", 1, "db", - "Would have been loading %q", fullname) + "Would have been extracting metadata from %q", + fullname) else report ("both", 4, "db", - "Loading font %q", fullname) - local new = load_font (fullname, fontnames, - newfontnames, texmf) + "Extracting metadata from font %q", fullname) + local new = read_font_names (fullname, fontnames, + newfontnames, texmf) if new == true then n_new = n_new + 1 end @@ -1890,6 +1898,7 @@ local path_separator = ostype == "windows" and ";" or ":" --doc]]-- --- dbobj -> dbobj -> bool? -> (int * int) + local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) local n_scanned, n_new, fontdirs = 0, 0 @@ -2301,6 +2310,24 @@ local gen_fast_lookups = function (fontnames) return fontnames end +local retrieve_namedata = function (fontnames, newfontnames, dry_run) + local n_rawnames, n_new = 0, 0 + + local rawnames, new = scan_texmf_fonts (fontnames, + newfontnames, + dry_run) + + n_rawnames = n_rawnames + rawnames + n_new = n_new + new + + rawnames, new = scan_os_fonts (fontnames, newfontnames, dry_run) + + n_rawnames = n_rawnames + rawnames + n_new = n_new + new + + return n_rawnames, n_new +end + --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs @@ -2314,8 +2341,8 @@ update_names = function (fontnames, force, dry_run) return fontnames or names.data end - local starttime = os.gettimeofday() - local n_scanned, n_new = 0, 0 + local starttime = os.gettimeofday() + local n_rawnames, n_new = 0, 0 --[[ The main function, scans everything @@ -2340,14 +2367,11 @@ update_names = function (fontnames, force, dry_run) local newfontnames = fontnames_init (get_font_filter ()) read_blacklist () - local scanned, new - scanned, new = scan_texmf_fonts (fontnames, newfontnames, dry_run) - n_scanned = n_scanned + scanned - n_new = n_new + new - - scanned, new = scan_os_fonts (fontnames, newfontnames, dry_run) - n_scanned = n_scanned + scanned - n_new = n_new + new + local rawnames, new = retrieve_namedata (fontnames, + newfontnames, + dry_run) + n_rawnames = n_rawnames + rawnames + n_new = n_new + new --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping @@ -2359,7 +2383,7 @@ update_names = function (fontnames, force, dry_run) --- partial: 804 ms | 701 ms --- forced: 45384 ms | 44714 ms report("info", 3, "db", - "Scanned %d font files; %d new entries.", n_scanned, n_new) + "Scanned %d font files; %d new entries.", n_rawnames, n_new) report("info", 3, "db", "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) names.data = newfontnames -- cgit v1.2.3 From ef4e74ea5906f81b8b12ddab7bb8417a90635d50 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:46:56 +0200 Subject: [db] choose more meaningful identifiers in name extractor --- luaotfload-database.lua | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 0139d89..bc654fa 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1409,22 +1409,22 @@ local loaders = { --- string -> dbobj -> dbobj -> bool local read_font_names = function (fullname, - fontnames, - newfontnames, + currentnames, + targetnames, texmf) - local newmappings = newfontnames.mappings - local newstatus = newfontnames.status --- by full path + local targetmappings = targetnames.mappings + local targetstatus = targetnames.status --- by full path - local mappings = fontnames.mappings - local status = fontnames.status + local currentmappings = currentnames.mappings + local currentstatus = currentnames.status - local basename = filebasename (fullname) - local barename = filenameonly (fullname) + local basename = filebasename (fullname) + local barename = filenameonly (fullname) - local format = stringlower (filesuffix (basename)) + local format = stringlower (filesuffix (basename)) - local entryname = fullname + local entryname = fullname if texmf == true then entryname = basename end @@ -1437,30 +1437,30 @@ local read_font_names = function (fullname, end local new_timestamp, current_timestamp - current_timestamp = status[fullname] - and status[fullname].timestamp + current_timestamp = currentstatus[fullname] + and currentstatus[fullname].timestamp new_timestamp = lfsattributes(fullname, "modification") - local newentrystatus = newstatus[fullname] + local newentrystatus = targetstatus[fullname] --- newentrystatus: nil | false | table if newentrystatus and newentrystatus.timestamp == new_timestamp then -- already statused this run return false end - newstatus[fullname] = newentrystatus or { } - local newentrystatus = newstatus[fullname] + targetstatus[fullname] = newentrystatus or { } + local newentrystatus = targetstatus[fullname] newentrystatus.timestamp = new_timestamp newentrystatus.index = newentrystatus.index or { } if current_timestamp == new_timestamp and not newentrystatus.index[1] then - for _, v in next, status[fullname].index do + for _, v in next, currentstatus[fullname].index do local index = #newentrystatus.index - local fullinfo = mappings[v] - local location = #newmappings + 1 - newmappings[location] = fullinfo --- keep + local fullinfo = currentmappings[v] + local location = #targetmappings + 1 + targetmappings[location] = fullinfo --- keep newentrystatus.index[index+1] = location --- is this actually used anywhere? end report("log", 2, "db", "Font %q already indexed", basename) @@ -1482,11 +1482,11 @@ local read_font_names = function (fullname, if not fullinfo then return false end - local location = #newmappings+1 + local location = #targetmappings + 1 local index = newentrystatus.index[n_font] if not index then index = location end - newmappings[index] = fullinfo + targetmappings[index] = fullinfo newentrystatus.index[n_font] = index end else @@ -1494,11 +1494,11 @@ local read_font_names = function (fullname, if not fullinfo then return false end - local location = #newmappings+1 + local location = #targetmappings + 1 local index = newentrystatus.index[1] if not index then index = location end - newmappings[index] = fullinfo + targetmappings[index] = fullinfo newentrystatus.index[1] = index end @@ -2364,7 +2364,9 @@ update_names = function (fontnames, force, dry_run) fontnames = fontnames_init (get_font_filter ()) end end + local newfontnames = fontnames_init (get_font_filter ()) + read_blacklist () local rawnames, new = retrieve_namedata (fontnames, -- cgit v1.2.3 From 599a9652b02520c6dfcd162c2f90bb1919177517 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:10:51 +0200 Subject: [db] store location (texmf/sys) as string, not bool --- luaotfload-database.lua | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bc654fa..23058cb 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -615,7 +615,7 @@ on whether the index entry has a texmf flag set. local get_font_file = function (fullnames, entry) local basename = entry.basename - if entry.texmf == true then + if entry.location == "texmf" then if kpselookup(basename) then return true, basename, entry.subfont end @@ -1224,7 +1224,7 @@ table as returned by the font file reader need to be relocated. --doc]]-- --- string -> int -> bool -> string -> fontentry -ot_fullinfo = function (filename, subfont, texmf, basename) +ot_fullinfo = function (filename, subfont, location, basename) local namedata = { } local metadata = load_font_file (filename, subfont) @@ -1306,7 +1306,7 @@ ot_fullinfo = function (filename, subfont, texmf, basename) --- file location data (used to be filename field) namedata.filename = filename --> sys namedata.basename = basename --> texmf - namedata.texmf = texmf or false + namedata.location = location or "system" --- TODO or “local”?? namedata.subfont = subfont return namedata @@ -1323,7 +1323,7 @@ end --doc]]-- --- string -> int -> bool -> string -> fontentry -t1_fullinfo = function (filename, _subfont, texmf, basename) +t1_fullinfo = function (filename, _subfont, location, basename) local namedata = { } local metadata = load_font_file (filename) @@ -1390,8 +1390,9 @@ t1_fullinfo = function (filename, _subfont, texmf, basename) namedata.filename = filename --> sys namedata.basename = basename --> texmf - namedata.texmf = texmf or false + namedata.location = location or "system" namedata.subfont = false + return namedata end @@ -1411,7 +1412,7 @@ local loaders = { local read_font_names = function (fullname, currentnames, targetnames, - texmf) + location) local targetmappings = targetnames.mappings local targetstatus = targetnames.status --- by full path @@ -1425,7 +1426,7 @@ local read_font_names = function (fullname, local format = stringlower (filesuffix (basename)) local entryname = fullname - if texmf == true then + if location == "texmf" then entryname = basename end @@ -1475,10 +1476,11 @@ local read_font_names = function (fullname, end local info = fontloaderinfo(fullname) + if info then if type(info) == "table" and #info > 1 then --- ttc for n_font = 1, #info do - local fullinfo = loader (fullname, n_font-1, texmf, basename) + local fullinfo = loader (fullname, n_font-1, location, basename) if not fullinfo then return false end @@ -1490,7 +1492,7 @@ local read_font_names = function (fullname, newentrystatus.index[n_font] = index end else - local fullinfo = loader (fullname, false, texmf, basename) + local fullinfo = loader (fullname, false, location, basename) if not fullinfo then return false end @@ -1832,14 +1834,14 @@ end --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) local scan_dir = function (dirname, fontnames, newfontnames, - dry_run, texmf) + dry_run, location) if lpegmatch (p_blacklist, dirname) then report ("both", 3, "db", "Skipping blacklisted directory %s", dirname) --- ignore return 0, 0 end - local found = find_font_files (dirname, texmf ~= true) + local found = find_font_files (dirname, location ~= "texmf") if not found then report ("both", 3, "db", "No such directory: %q; skipping.", dirname) @@ -1862,7 +1864,7 @@ local scan_dir = function (dirname, fontnames, newfontnames, report ("both", 4, "db", "Extracting metadata from font %q", fullname) local new = read_font_names (fullname, fontnames, - newfontnames, texmf) + newfontnames, location) if new == true then n_new = n_new + 1 end @@ -1928,7 +1930,7 @@ local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) "Initiating scan of %d directories.", #tasks) for _, d in next, tasks do local found, new = scan_dir (d, fontnames, newfontnames, - dry_run, true) + dry_run, "texmf") n_scanned = n_scanned + found n_new = n_new + new end @@ -2092,6 +2094,10 @@ do --- closure for read_fonts_conf() --- We exclude paths with texmf in them, as they should be --- found anyway; also duplicates are ignored by checking --- if they are elements of dirs_done. + --- + --- FIXME does this mean we cannot access paths from + --- distributions (e.g. Context minimals) installed + --- separately? if not (stringfind(path, "texmf") or dirs_done[path]) then acc[#acc+1] = path dirs_done[path] = true @@ -2230,7 +2236,7 @@ flush_lookup_cache = function () end --- dbobj -> dbobj -local gen_fast_lookups = function (fontnames) +local gen_fast_lookups = function (fontnames) --- this will become obsolete report("both", 2, "db", "Creating filename map") local mappings = fontnames.mappings local nmappings = #mappings @@ -2262,7 +2268,7 @@ local gen_fast_lookups = function (fontnames) --- substantial duplication -- entry.filename = nil - if entry.texmf == true then + if entry.location == "texmf" then texmf[#texmf+1] = { idx, basename, bare, true, nil } else sys[#sys+1] = { idx, basename, bare, false, filename } @@ -2378,6 +2384,7 @@ update_names = function (fontnames, force, dry_run) --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent + newfontnames = gen_fast_lookups(newfontnames) --- stats: -- cgit v1.2.3 From a4c38dbe7fa31d7dbd2336a3290d366237621e46 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:34:32 +0200 Subject: [db] refactor timestamp comparison --- luaotfload-database.lua | 118 ++++++++++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 44 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 23058cb..c46956d 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1406,6 +1406,52 @@ local loaders = { pfa = t1_fullinfo, } +--- not side-effect free! + +local compare_timestamps = function (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) + + local currenttimestamp = currententrystatus + and currententrystatus.timestamp + local targettimestamp = lfsattributes (fullname, "modification") + + if targetentrystatus ~= nil + and targetentrystatus.timestamp == targettimestamp then + report ("log", 3, "db", "Font %q already read.", basename) + return false + end + + targetentrystatus.timestamp = targettimestamp + targetentrystatus.index = targetentrystatus.index or { } + + if currenttimestamp == targettimestamp + and not targetentrystatus.index [1] + then + + --- copy old namedata into new + + for _, currentindex in next, currententrystatus.index do + + local targetindex = #targetentrystatus.index + local fullinfo = currentmappings [currentindex] + local location = #targetmappings + 1 + + targetmappings [location] = fullinfo + targetentrystatus.index [targetindex + 1] = location + end + + report ("log", 3, "db", "Font %q already indexed.", basename) + + return false + end +end + + --- we return true if the font is new or re-indexed --- string -> dbobj -> dbobj -> bool @@ -1414,18 +1460,25 @@ local read_font_names = function (fullname, targetnames, location) - local targetmappings = targetnames.mappings - local targetstatus = targetnames.status --- by full path + local targetmappings = targetnames.mappings + local targetstatus = targetnames.status --- by full path + local targetentrystatus = targetstatus [fullname] - local currentmappings = currentnames.mappings - local currentstatus = currentnames.status + if targetentrystatus == nil then + targetentrystatus = { } + targetstatus [fullname] = targetentrystatus + end - local basename = filebasename (fullname) - local barename = filenameonly (fullname) + local currentmappings = currentnames.mappings + local currentstatus = currentnames.status + local currententrystatus = currentstatus [fullname] - local format = stringlower (filesuffix (basename)) + local basename = filebasename (fullname) + local barename = filenameonly (fullname) + local entryname = fullname + + local format = stringlower (filesuffix (basename)) - local entryname = fullname if location == "texmf" then entryname = basename end @@ -1437,36 +1490,13 @@ local read_font_names = function (fullname, return false end - local new_timestamp, current_timestamp - current_timestamp = currentstatus[fullname] - and currentstatus[fullname].timestamp - new_timestamp = lfsattributes(fullname, "modification") - - local newentrystatus = targetstatus[fullname] - --- newentrystatus: nil | false | table - if newentrystatus and newentrystatus.timestamp == new_timestamp then - -- already statused this run - return false - end - - targetstatus[fullname] = newentrystatus or { } - local newentrystatus = targetstatus[fullname] - newentrystatus.timestamp = new_timestamp - newentrystatus.index = newentrystatus.index or { } - - if current_timestamp == new_timestamp - and not newentrystatus.index[1] - then - for _, v in next, currentstatus[fullname].index do - local index = #newentrystatus.index - local fullinfo = currentmappings[v] - local location = #targetmappings + 1 - targetmappings[location] = fullinfo --- keep - newentrystatus.index[index+1] = location --- is this actually used anywhere? - end - report("log", 2, "db", "Font %q already indexed", basename) - return false - end + local changed = compare_timestamps (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) local loader = loaders[format] --- ot_fullinfo, t1_fullinfo if not loader then @@ -1485,11 +1515,11 @@ local read_font_names = function (fullname, return false end local location = #targetmappings + 1 - local index = newentrystatus.index[n_font] + local index = targetentrystatus.index[n_font] if not index then index = location end - targetmappings[index] = fullinfo - newentrystatus.index[n_font] = index + targetmappings[index] = fullinfo + targetentrystatus.index[n_font] = index end else local fullinfo = loader (fullname, false, location, basename) @@ -1497,11 +1527,11 @@ local read_font_names = function (fullname, return false end local location = #targetmappings + 1 - local index = newentrystatus.index[1] + local index = targetentrystatus.index[1] if not index then index = location end - targetmappings[index] = fullinfo - newentrystatus.index[1] = index + targetmappings[index] = fullinfo + targetentrystatus.index[1] = index end else --- missing info -- cgit v1.2.3 From c39f4e5ea942a88a319321b8cd546532300700dc Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:39:42 +0200 Subject: [db] correctly abort if timestamps match --- luaotfload-database.lua | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index c46956d..6202079 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -431,7 +431,7 @@ load_names = function (dry_run) report ("info", 3, "db", "Loading took %0.f ms", 1000*(os.gettimeofday()-starttime)) - local db_version, nms_version = data.version, names.version + local db_version, nms_version = data.meta.version, names.version if db_version ~= nms_version then report ("both", 0, "db", [[Version mismatch; expected %4.3f, got %4.3f]], @@ -1449,6 +1449,8 @@ local compare_timestamps = function (fullname, return false end + + return true end @@ -1490,13 +1492,16 @@ local read_font_names = function (fullname, return false end - local changed = compare_timestamps (fullname, - currentstatus, - currententrystatus, - currentmappings, - targetstatus, - targetentrystatus, - targetmappings) + if not compare_timestamps (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) + then + return false + end local loader = loaders[format] --- ot_fullinfo, t1_fullinfo if not loader then -- cgit v1.2.3 From 698b5178cbb9d8d9c9cf5f429478770f1645cb43 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:58:40 +0200 Subject: [db] refactor font info extraction --- luaotfload-database.lua | 99 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 33 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 6202079..3ffc1c4 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1453,6 +1453,36 @@ local compare_timestamps = function (fullname, return true end +local insert_fullinfo = function (fullname, + basename, + n_font, + loader, + location, + targetmappings, + targetentrystatus) + + local subfont = n_font and n_font - 1 or false + + local fullinfo = loader (fullname, subfont, + location, basename) + + if not fullinfo then + return false + end + + local index = targetentrystatus.index [n_font] + + if not index then + index = #targetmappings + 1 + end + + targetmappings [index] = fullinfo + targetentrystatus.index [n_font] = index + + return true +end + + --- we return true if the font is new or re-indexed --- string -> dbobj -> dbobj -> bool @@ -1479,19 +1509,20 @@ local read_font_names = function (fullname, local barename = filenameonly (fullname) local entryname = fullname - local format = stringlower (filesuffix (basename)) - if location == "texmf" then entryname = basename end - if names.blacklist[fullname] or names.blacklist[basename] - then + --- 1) skip if blacklisted + + if names.blacklist[fullname] or names.blacklist[basename] then report("log", 2, "db", "Ignoring blacklisted font %q", fullname) return false end + --- 2) skip if known with same timestamp + if not compare_timestamps (fullname, currentstatus, currententrystatus, @@ -1503,47 +1534,49 @@ local read_font_names = function (fullname, return false end - local loader = loaders[format] --- ot_fullinfo, t1_fullinfo + --- 3) new font; choose a loader, abort if unknown + + local format = stringlower (filesuffix (basename)) + local loader = loaders [format] --- ot_fullinfo, t1_fullinfo + if not loader then report ("both", 0, "db", "Unknown format: %q, skipping.", format) return false end - local info = fontloaderinfo(fullname) + --- 4) get basic info, abort if fontloader can’t read it - if info then - if type(info) == "table" and #info > 1 then --- ttc - for n_font = 1, #info do - local fullinfo = loader (fullname, n_font-1, location, basename) - if not fullinfo then - return false - end - local location = #targetmappings + 1 - local index = targetentrystatus.index[n_font] - if not index then index = location end + local info = fontloaderinfo (fullname) + + if not info then + report ("log", 1, "db", + "Failed to read basic information from %q", basename) + return false + end - targetmappings[index] = fullinfo - targetentrystatus.index[n_font] = index - end - else - local fullinfo = loader (fullname, false, location, basename) - if not fullinfo then - return false - end - local location = #targetmappings + 1 - local index = targetentrystatus.index[1] - if not index then index = location end - targetmappings[index] = fullinfo - targetentrystatus.index[1] = index + --- 5) check for subfonts and process each of them + + if type (info) == "table" and #info > 1 then --- ttc + + local success = false --- true if at least one subfont got read + + for n_font = 1, #info do + if insert_fullinfo (fullname, basename, n_font, + loader, location, + targetmappings, targetentrystatus) + then + success = true + end end - else --- missing info - report("log", 1, "db", "Failed to load %q", basename) - return false + return success end - return true + + return insert_fullinfo (fullname, basename, false, + loader, location, + targetmappings, targetentrystatus) end local path_normalize -- cgit v1.2.3 From 1167ef911872ca57f9deacde1c4b1d5eb71309ad Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 12:04:41 +0200 Subject: [db] reorganize mappings record; refactor metadata extraction --- luaotfload-database.lua | 262 ++++++++++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 107 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 3ffc1c4..a85079b 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -272,7 +272,7 @@ This is a sketch of the luaotfload db: texmf : (string, int) hash; }; bare : { - local : (string, (string, int) hash) hash; // format -> (barename -> idx) + local : (string, (string, int) hash) hash; // location -> (barename -> idx) system : (string, (string, int) hash) hash; texmf : (string, (string, int) hash) hash; }; @@ -1217,21 +1217,31 @@ local load_font_file = function (filename, subfont) return metadata end ---[[doc-- -The data inside an Opentype font file can be quite heterogeneous. -Thus in order to get the relevant information, parts of the original -table as returned by the font file reader need to be relocated. ---doc]]-- +--- rawdata -> (int * int * int | bool) ---- string -> int -> bool -> string -> fontentry -ot_fullinfo = function (filename, subfont, location, basename) - local namedata = { } +local get_size_info = function (metadata) + local design_size = metadata.design_size + local design_range_top = metadata.design_range_top + local design_range_bottom = metadata.design_range_bottom - local metadata = load_font_file (filename, subfont) - if not metadata then - return nil + local fallback_size = design_size ~= 0 and design_size + or design_range_bottom ~= 0 and design_range_bottom + or design_range_top ~= 0 and design_range_top + + if fallback_size then + design_size = (design_size or fallback_size) / 10 + design_range_top = (design_range_top or fallback_size) / 10 + design_range_bottom = (design_range_bottom or fallback_size) / 10 + return { + design_size, design_range_top, design_range_bottom, + } end + return false +end + +local extract_namedata = function (metadata, basename) + local english_names if metadata.names then @@ -1241,11 +1251,11 @@ ot_fullinfo = function (filename, subfont, location, basename) end end else - -- no names table, propably a broken font + -- no names table, probably a broken font report("log", 1, "db", "Broken font %s rejected due to missing names table.", basename) - return + return nil end local fontnames = { @@ -1272,44 +1282,56 @@ ot_fullinfo = function (filename, subfont, location, basename) end end - namedata.sanitized = sanitize_names (fontnames) - namedata.fontname = metadata.fontname - namedata.fullname = metadata.fullname - namedata.familyname = metadata.familyname - namedata.weight = metadata.pfminfo.weight - namedata.width = metadata.pfminfo.width - namedata.slant = metadata.italicangle - --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details - namedata.units_per_em = metadata.units_per_em - namedata.version = metadata.version - -- don't waste the space with zero values + return { + sanitized = sanitize_names (fontnames), + fontname = metadata.fontname, + fullname = metadata.fullname, + familyname = metadata.familyname, + } - local design_size = metadata.design_size - local design_range_top = metadata.design_range_top - local design_range_bottom = metadata.design_range_bottom +end - local fallback_size = design_size ~= 0 and design_size - or design_range_bottom ~= 0 and design_range_bottom - or design_range_top ~= 0 and design_range_top - if fallback_size then - design_size = (design_size or fallback_size) / 10 - design_range_top = (design_range_top or fallback_size) / 10 - design_range_bottom = (design_range_bottom or fallback_size) / 10 - namedata.size = { - design_size, design_range_top, design_range_bottom, - } - else - namedata.size = false +--[[doc-- +The data inside an Opentype font file can be quite heterogeneous. +Thus in order to get the relevant information, parts of the original +table as returned by the font file reader need to be relocated. +--doc]]-- + +--- string -> int -> bool -> string -> fontentry + +ot_fullinfo = function (filename, subfont, location, basename, format) + local styles = { } + + local metadata = load_font_file (filename, subfont) + if not metadata then + return nil end - --- file location data (used to be filename field) - namedata.filename = filename --> sys - namedata.basename = basename --> texmf - namedata.location = location or "system" --- TODO or “local”?? - namedata.subfont = subfont + local namedata = extract_namedata (metadata, basename) - return namedata + + local style = { + size = get_size_info (metadata), + weight = metadata.pfminfo.weight, + width = metadata.pfminfo.width, + slant = metadata.italicangle, + --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details + units_per_em = metadata.units_per_em, + version = metadata.version, + } + + return { + file = { base = basename, + full = filename, + subfont = subfont, + location = location or "system" }, + format = format, + names = namedata, + style = style, + units_per_em = metadata.units_per_em, + version = metadata.version, + } end --[[doc-- @@ -1323,7 +1345,8 @@ end --doc]]-- --- string -> int -> bool -> string -> fontentry -t1_fullinfo = function (filename, _subfont, location, basename) + +t1_fullinfo = function (filename, _subfont, location, basename, format) local namedata = { } local metadata = load_font_file (filename) @@ -1390,6 +1413,7 @@ t1_fullinfo = function (filename, _subfont, location, basename) namedata.filename = filename --> sys namedata.basename = basename --> texmf + namedata.format = format namedata.location = location or "system" namedata.subfont = false @@ -1457,6 +1481,7 @@ local insert_fullinfo = function (fullname, basename, n_font, loader, + format, location, targetmappings, targetentrystatus) @@ -1464,7 +1489,8 @@ local insert_fullinfo = function (fullname, local subfont = n_font and n_font - 1 or false local fullinfo = loader (fullname, subfont, - location, basename) + location, basename, + format) if not fullinfo then return false @@ -1564,7 +1590,7 @@ local read_font_names = function (fullname, for n_font = 1, #info do if insert_fullinfo (fullname, basename, n_font, - loader, location, + loader, format, location, targetmappings, targetentrystatus) then success = true @@ -1575,7 +1601,7 @@ local read_font_names = function (fullname, end return insert_fullinfo (fullname, basename, false, - loader, location, + loader, format, location, targetmappings, targetentrystatus) end @@ -2303,85 +2329,107 @@ flush_lookup_cache = function () return true, names.lookups end ---- dbobj -> dbobj -local gen_fast_lookups = function (fontnames) --- this will become obsolete - report("both", 2, "db", "Creating filename map") - local mappings = fontnames.mappings + +--- fontentry list -> filemap + +local generate_filedata = function (mappings) + + report ("both", 2, "db", "Creating filename map") + local nmappings = #mappings - --- this is needlessly complicated due to texmf priorization + local filenames = { bare = { - system = { }, --- mapped to mapping format -> index in full - texmf = { }, --- mapped to mapping format -> “true” + ["local"] = { }, + system = { }, --- mapped to mapping format -> index in full + texmf = { }, --- mapped to mapping format -> “true” }, base = { - system = { }, --- mapped to index in “full” - texmf = { }, --- set; all values are “true” + ["local"] = { }, + system = { }, --- mapped to index in “full” + texmf = { }, --- set; all values are “true” }, full = { }, --- non-texmf } - local texmf, sys = { }, { } -- quintuple list + local base = filenames.base + local bare = filenames.bare + local full = filenames.full + + local conflicts = { + basenames = 0, + barenames = 0, + } + + for index = 1, nmappings do + local entry = mappings [index] + + local filedata = entry.file + + local format = entry.format --- otf, afm, ... + local location = filedata.location --- texmf, system, ... - for idx = 1, nmappings do - local entry = mappings[idx] - local filename = entry.filename - local basename = entry.basename - local bare = filenameonly(filename) - local subfont = entry.subfont + local filename = filedata.full + local basename = filedata.base + local barename = filenameonly (filename) + local subfont = filedata.subfont - entry.index = idx ---- unfortunately, the sys/texmf schism prevents us from ---- doing away the full name, so we cannot avoid the ---- substantial duplication --- entry.filename = nil + entry.index = index - if entry.location == "texmf" then - texmf[#texmf+1] = { idx, basename, bare, true, nil } + --- 1) add to basename table + + local inbase = base [location] --- no format since the suffix is known + + if inbase then + if inbase [basename] then + report ("both", 3, "db", + "Conflicting basename: %q already indexed \z + in category %s, ignoring.", + barename, location) + conflicts.basenames = conflicts.basenames + 1 + else + inbase [basename] = index + end else - sys[#sys+1] = { idx, basename, bare, false, filename } + inbase = { basename = index } + base [location] = inbase end - end - local addmap = function (lst) - --- this will overwrite existing entries - for i=1, #lst do - local idx, base, bare, intexmf, full = unpack(lst[i]) + --- 2) add to barename table - local known = filenames.base[base] or filenames.bare[bare] - if known then --- known - report("both", 3, "db", - "Font file %q already indexed (%d)", - base, idx) - report("both", 3, "db", "> old location: %s", - (filenames.full[known] or "texmf")) - report("both", 3, "db", "> new location: %s", - (intexmf and "texmf" or full)) - end + local inbare = bare [location] [format] - filenames.bare[bare] = idx - filenames.base[base] = idx - if intexmf == true then - filenames.full[idx] = nil + if inbare then + if inbare [barename] then + report ("both", 3, "db", + "Conflicting barename: %q already indexed \z + in category %s/%s, ignoring.", + barename, location, format) + conflicts.barenames = conflicts.barenames + 1 else - filenames.full[idx] = full + inbare [barename] = index end + else + inbare = { barename = index } + bare [location] [format] = inbare end - end - if config.luaotfload.prioritize == "texmf" then - report("both", 2, "db", "Preferring texmf fonts") - addmap(sys) - addmap(texmf) - else --- sys - addmap(texmf) - addmap(sys) + --- 3) add to fullname map + + full [index] = fullname end - fontnames.filenames = filenames - texmf, sys = nil, nil - collectgarbage "collect" - return fontnames + --- TODO adapt to new mechanism! +-- if config.luaotfload.prioritize == "texmf" then +-- report("both", 2, "db", "Preferring texmf fonts") +-- addmap(sys) +-- addmap(texmf) +-- else --- sys +-- addmap(texmf) +-- addmap(sys) +-- end + + return filenames end local retrieve_namedata = function (fontnames, newfontnames, dry_run) @@ -2453,7 +2501,7 @@ update_names = function (fontnames, force, dry_run) --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - newfontnames = gen_fast_lookups(newfontnames) + newfontnames.files = generate_filedata (newfontnames.mappings) --- stats: --- before rewrite | after rewrite -- cgit v1.2.3 From 1b5783af6e22335afa514712e738273d323bc76a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 12:18:17 +0200 Subject: [db] include information about conflicting file names in record --- luaotfload-database.lua | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index a85079b..946c773 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2369,9 +2369,9 @@ local generate_filedata = function (mappings) local format = entry.format --- otf, afm, ... local location = filedata.location --- texmf, system, ... - local filename = filedata.full + local fullname = filedata.full local basename = filedata.base - local barename = filenameonly (filename) + local barename = filenameonly (fullname) local subfont = filedata.subfont entry.index = index @@ -2381,12 +2381,23 @@ local generate_filedata = function (mappings) local inbase = base [location] --- no format since the suffix is known if inbase then - if inbase [basename] then + local present = inbase [basename] + if present then report ("both", 3, "db", "Conflicting basename: %q already indexed \z in category %s, ignoring.", barename, location) conflicts.basenames = conflicts.basenames + 1 + + --- track conflicts per font + local conflictdata = entry.conflicts + + if not conflictdata then + entry.conflicts = { basename = present } + else -- some conflicts already detected + conflictdata.basename = present + end + else inbase [basename] = index end @@ -2400,12 +2411,23 @@ local generate_filedata = function (mappings) local inbare = bare [location] [format] if inbare then - if inbare [barename] then + local present = inbare [barename] + if present then report ("both", 3, "db", "Conflicting barename: %q already indexed \z in category %s/%s, ignoring.", barename, location, format) conflicts.barenames = conflicts.barenames + 1 + + --- track conflicts per font + local conflictdata = entry.conflicts + + if not conflictdata then + entry.conflicts = { barename = present } + else -- some conflicts already detected + conflictdata.barename = present + end + else inbare [barename] = index end -- cgit v1.2.3 From 0a60f38e18e916ef44a71454032471f1b2173bd6 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 13:22:10 +0200 Subject: [db] cleanup: use more consistent identifiers --- luaotfload-database.lua | 122 ++++++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 57 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 946c773..2a9a3e9 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -44,6 +44,7 @@ local lfscurrentdir = lfs.currentdir local lfsdir = lfs.dir local mathabs = math.abs local mathmin = math.min +local osgettimeofday = os.gettimeofday local osremove = os.remove local stringfind = string.find local stringformat = string.format @@ -344,7 +345,7 @@ mtx-fonts has in names.tma: --doc]]-- -local fontnames_init = function (formats) --- returns dbobj +local initialize_namedata = function (formats) --- returns dbobj return { families = { ["local"] = { }, @@ -422,14 +423,14 @@ local fuzzy_limit = 1 --- display closest only --- bool? -> dbobj load_names = function (dry_run) - local starttime = os.gettimeofday () + local starttime = osgettimeofday () local foundname, data = load_lua_file (names.path.index.lua) if data then report ("both", 2, "db", "Font names database loaded", "%s", foundname) report ("info", 3, "db", "Loading took %0.f ms", - 1000*(os.gettimeofday()-starttime)) + 1000 * (osgettimeofday () - starttime)) local db_version, nms_version = data.meta.version, names.version if db_version ~= nms_version then @@ -450,7 +451,7 @@ load_names = function (dry_run) [[Font names database not found, generating new one.]]) report ("both", 0, "db", [[This can take several minutes; please be patient.]]) - data = update_names (fontnames_init (get_font_filter ()), + data = update_names (initialize_namedata (get_font_filter ()), nil, dry_run) if not success then report ("both", 0, "db", "Database creation unsuccessful.") @@ -1918,16 +1919,16 @@ end scan_dir() scans a directory and populates the list of fonts with all the fonts it finds. - · dirname : name of the directory to scan - · fontnames : current font db object - · newnames : font db object to fill - · dry_run : don’t touch anything + · dirname : name of the directory to scan + · currentnames : current font db object + · targetnames : font db object to fill + · dry_run : don’t touch anything --doc]]-- --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) -local scan_dir = function (dirname, fontnames, newfontnames, +local scan_dir = function (dirname, currentnames, targetnames, dry_run, location) if lpegmatch (p_blacklist, dirname) then report ("both", 3, "db", @@ -1957,8 +1958,8 @@ local scan_dir = function (dirname, fontnames, newfontnames, else report ("both", 4, "db", "Extracting metadata from font %q", fullname) - local new = read_font_names (fullname, fontnames, - newfontnames, location) + local new = read_font_names (fullname, currentnames, + targetnames, location) if new == true then n_new = n_new + 1 end @@ -1995,7 +1996,7 @@ local path_separator = ostype == "windows" and ";" or ":" --- dbobj -> dbobj -> bool? -> (int * int) -local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) +local scan_texmf_fonts = function (currentnames, targetnames, dry_run) local n_scanned, n_new, fontdirs = 0, 0 local osfontdir = kpseexpand_path "$OSFONTDIR" @@ -2023,7 +2024,7 @@ local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) report ("info", 3, "db", "Initiating scan of %d directories.", #tasks) for _, d in next, tasks do - local found, new = scan_dir (d, fontnames, newfontnames, + local found, new = scan_dir (d, currentnames, targetnames, dry_run, "texmf") n_scanned = n_scanned + found n_new = n_new + new @@ -2303,7 +2304,8 @@ end --doc]]-- --- dbobj -> dbobj -> bool? -> (int * int) -local scan_os_fonts = function (fontnames, newfontnames, +local scan_os_fonts = function (currentnames, + targetnames, dry_run) local n_scanned, n_new = 0, 0 @@ -2312,8 +2314,8 @@ local scan_os_fonts = function (fontnames, newfontnames, "Searching in static system directories...") for _, d in next, get_os_dirs () do - local found, new = scan_dir (d, fontnames, - newfontnames, dry_run) + local found, new = scan_dir (d, currentnames, + targetnames, dry_run) n_scanned = n_scanned + found n_new = n_new + new end @@ -2454,86 +2456,90 @@ local generate_filedata = function (mappings) return filenames end -local retrieve_namedata = function (fontnames, newfontnames, dry_run) - local n_rawnames, n_new = 0, 0 - - local rawnames, new = scan_texmf_fonts (fontnames, - newfontnames, +local retrieve_namedata = function (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) + local rawnames, new = scan_texmf_fonts (currentnames, + targetnames, dry_run) n_rawnames = n_rawnames + rawnames - n_new = n_new + new + n_newnames = n_newnames + new - rawnames, new = scan_os_fonts (fontnames, newfontnames, dry_run) + rawnames, new = scan_os_fonts (currentnames, targetnames, dry_run) n_rawnames = n_rawnames + rawnames - n_new = n_new + new + n_newnames = n_newnames + new - return n_rawnames, n_new + return n_rawnames, n_newnames end --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs --- dbobj? -> bool? -> bool? -> dbobj -update_names = function (fontnames, force, dry_run) +update_names = function (currentnames, force, dry_run) if config.luaotfload.update_live == false then - report("info", 2, "db", - "Skipping database update") + report ("info", 2, "db", + "Skipping database update") --- skip all db updates - return fontnames or names.data + return currentnames or names.data end - local starttime = os.gettimeofday() - local n_rawnames, n_new = 0, 0 + local starttime = osgettimeofday () + local n_rawnames, n_newnames = 0, 0 --[[ The main function, scans everything - - “newfontnames” is the final table to return + - “targetnames” is the final table to return - force is whether we rebuild it from scratch or not ]] - report("both", 2, "db", "Updating the font names database" - .. (force and " forcefully" or "")) + report ("both", 2, "db", "Updating the font names database" + .. (force and " forcefully" or "")) if force then - fontnames = fontnames_init (get_font_filter ()) + currentnames = initialize_namedata (get_font_filter ()) else - if not fontnames then - fontnames = load_names (dry_run) + if not currentnames then + currentnames = load_names (dry_run) end - if fontnames.version ~= names.version then + if currentnames.meta.version ~= names.version then report ("both", 1, "db", "No font names database or old " .. "one found; generating new one") - fontnames = fontnames_init (get_font_filter ()) + currentnames = initialize_namedata (get_font_filter ()) end end - local newfontnames = fontnames_init (get_font_filter ()) + local targetnames = initialize_namedata (get_font_filter ()) read_blacklist () - local rawnames, new = retrieve_namedata (fontnames, - newfontnames, - dry_run) - n_rawnames = n_rawnames + rawnames - n_new = n_new + new + local n_rawnames, n_newnames = retrieve_namedata (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - newfontnames.files = generate_filedata (newfontnames.mappings) + targetnames.files = generate_filedata (targetnames.mappings) --- stats: --- before rewrite | after rewrite --- partial: 804 ms | 701 ms --- forced: 45384 ms | 44714 ms - report("info", 3, "db", - "Scanned %d font files; %d new entries.", n_rawnames, n_new) - report("info", 3, "db", - "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) - names.data = newfontnames + report ("info", 3, "db", + "Scanned %d font files; %d new entries.", + n_rawnames, n_newnames) + report ("info", 3, "db", + "Rebuilt in %0.f ms.", + 1000 * (osgettimeofday () - starttime)) + names.data = targetnames if dry_run ~= true then @@ -2545,11 +2551,11 @@ update_names = function (fontnames, force, dry_run) if success then logs.names_report ("info", 2, "cache", "Lookup cache emptied") - return newfontnames + return targetnames end end end - return newfontnames + return targetnames end --- unit -> bool @@ -2581,14 +2587,16 @@ end --- save_names() is usually called without the argument --- dbobj? -> bool -save_names = function (fontnames) - if not fontnames then fontnames = names.data end +save_names = function (currentnames) + if not currentnames then + currentnames = names.data + end local path = names.path.index local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, fontnames, true) + tabletofile (luaname, currentnames, true) osremove (lucname) - caches.compile (fontnames, luaname, lucname) + caches.compile (currentnames, luaname, lucname) if lfsisfile (luaname) and lfsisfile (lucname) then report ("info", 1, "db", "Font index saved") report ("info", 3, "db", "Text: " .. luaname) -- cgit v1.2.3 From 99f5ace3d56ae14baabdc494bb4b3326b8652ac7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 13:54:43 +0200 Subject: [db] adapt sanitization routines --- luaotfload-database.lua | 95 +++++++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 35 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 2a9a3e9..7be0600 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -180,14 +180,34 @@ end Auxiliary functions --doc]]-- +--- fontnames contain all kinds of garbage; as a precaution we +--- lowercase and strip them of non alphanumerical characters + --- string -> string -local sanitize_string = function (str) + +local invalidchars = "[^%a%d]" + +local sanitize_fontname = function (str) if str ~= nil then - return utf8gsub(utf8lower(str), "[^%a%d]", "") + return utf8gsub (utf8lower (str), invalidchars, "") end return nil end +local sanitize_fontnames = function (rawnames) + local result = { } + for category, namedata in next, rawnames do + local target = { } + for field, name in next, namedata do + target [field] = utf8gsub (utf8lower (name), + invalidchars, + "") + end + result [category] = target + end + return result +end + local find_files_indeed find_files_indeed = function (acc, dirs, filter) if not next (dirs) then --- done @@ -848,8 +868,8 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C if not fonts_loaded then names.data = load_names() end local data = names.data - local name = sanitize_string(specification.name) - local style = sanitize_string(specification.style) or "regular" + local name = sanitize_fontname (specification.name) + local style = sanitize_fontname (specification.style) or "regular" local askedsize @@ -909,8 +929,8 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C pfullname = facenames.pfullname metafamily = facenames.metafamily end - fontname = fontname or sanitize_string(face.fontname) - pfullname = pfullname or sanitize_string(face.fullname) + fontname = fontname or sanitize_fontname (face.fontname) + pfullname = pfullname or sanitize_fontname (face.fullname) if name == family or name == metafamily @@ -1126,7 +1146,7 @@ end --- string -> int -> bool find_closest = function (name, limit) - local name = sanitize_string(name) + local name = sanitize_fontname (name) limit = limit or fuzzy_limit if not fonts_loaded then names.data = load_names() end @@ -1198,14 +1218,6 @@ find_closest = function (name, limit) return false end --- find_closest() -local sanitize_names = function (names) - local res = { } - for idx, name in next, names do - res[idx] = sanitize_string(name) - end - return res -end - local load_font_file = function (filename, subfont) local rawfont, _msg = fontloaderopen (filename, subfont) if not rawfont then @@ -1241,7 +1253,7 @@ local get_size_info = function (metadata) return false end -local extract_namedata = function (metadata, basename) +local organize_namedata = function (metadata, basename, info) local english_names @@ -1262,16 +1274,22 @@ local extract_namedata = function (metadata, basename) local fontnames = { --- see --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html - fullname = english_names.compatfull - or english_names.fullname, - family = english_names.preffamilyname - or english_names.family, - prefmodifiers = english_names.prefmodifiers, - subfamily = english_names.subfamily, - psname = english_names.postscriptname, - pfullname = metadata.fullname, - fontname = metadata.fontname, - metafamily = metadata.familyname, + english = { + fullname = english_names.compatfull + or english_names.fullname, + family = english_names.preffamilyname + or english_names.family, + prefmodifiers = english_names.prefmodifiers, + subfamily = english_names.subfamily, + psname = english_names.postscriptname, + }, + metadata = { + pfullname = metadata.fullname, + fontname = metadata.fontname, + metafamily = metadata.familyname, + }, + info = { + }, } -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size @@ -1284,7 +1302,7 @@ local extract_namedata = function (metadata, basename) end return { - sanitized = sanitize_names (fontnames), + sanitized = sanitize_fontnames (fontnames), fontname = metadata.fontname, fullname = metadata.fullname, familyname = metadata.familyname, @@ -1301,7 +1319,12 @@ table as returned by the font file reader need to be relocated. --- string -> int -> bool -> string -> fontentry -ot_fullinfo = function (filename, subfont, location, basename, format) +ot_fullinfo = function (filename, + subfont, + location, + basename, + format, + info) local styles = { } local metadata = load_font_file (filename, subfont) @@ -1309,7 +1332,7 @@ ot_fullinfo = function (filename, subfont, location, basename, format) return nil end - local namedata = extract_namedata (metadata, basename) + local namedata = organize_namedata (metadata, basename, info) local style = { @@ -1369,7 +1392,7 @@ t1_fullinfo = function (filename, _subfont, location, basename, format) local style_synonyms_set = style_synonyms.set if weight then - weight = sanitize_string (weight) + weight = sanitize_fontname (weight) local tmp = "" if style_synonyms_set.bold[weight] then tmp = "bold" @@ -1390,7 +1413,7 @@ t1_fullinfo = function (filename, _subfont, location, basename, format) --- else italic end - namedata.sanitized = sanitize_names ({ + namedata.sanitized = sanitize_fontnames ({ fontname = fontname, psname = fullname, pfullname = fullname, @@ -1485,13 +1508,14 @@ local insert_fullinfo = function (fullname, format, location, targetmappings, - targetentrystatus) + targetentrystatus, + info) local subfont = n_font and n_font - 1 or false local fullinfo = loader (fullname, subfont, location, basename, - format) + format, info) if not fullinfo then return false @@ -1592,7 +1616,8 @@ local read_font_names = function (fullname, for n_font = 1, #info do if insert_fullinfo (fullname, basename, n_font, loader, format, location, - targetmappings, targetentrystatus) + targetmappings, targetentrystatus, + info) then success = true end @@ -2778,7 +2803,7 @@ names.update = update_names names.crude_file_lookup = crude_file_lookup names.crude_file_lookup_verbose = crude_file_lookup_verbose names.read_blacklist = read_blacklist -names.sanitize_string = sanitize_string +names.sanitize_fontname = sanitize_fontname names.getfilename = resolve_fullpath --- font cache -- cgit v1.2.3 From 87223dfb2c0768be70cdf58dce45cb877bc3757a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 17:33:27 +0200 Subject: [db] extend name extraction on fontloader.info() --- luaotfload-database.lua | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 7be0600..98b0975 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -197,13 +197,20 @@ end local sanitize_fontnames = function (rawnames) local result = { } for category, namedata in next, rawnames do - local target = { } - for field, name in next, namedata do - target [field] = utf8gsub (utf8lower (name), - invalidchars, - "") + + if type (namedata) == "string" then + result [category] = utf8gsub (utf8lower (namedata), + invalidchars, + "") + else + local target = { } + for field, name in next, namedata do + target [field] = utf8gsub (utf8lower (name), + invalidchars, + "") + end + result [category] = target end - result [category] = target end return result end @@ -1224,9 +1231,18 @@ local load_font_file = function (filename, subfont) report ("log", 1, "db", "ERROR: failed to open %s", filename) return end + local metadata = fontloaderto_table (rawfont) fontloaderclose (rawfont) + + metadata.glyphs = nil + metadata.subfonts = nil + metadata.gpos = nil + metadata.gsub = nil + metadata.lookups = nil + collectgarbage "collect" + return metadata end @@ -1279,8 +1295,8 @@ local organize_namedata = function (metadata, basename, info) or english_names.fullname, family = english_names.preffamilyname or english_names.family, - prefmodifiers = english_names.prefmodifiers, - subfamily = english_names.subfamily, + --prefmodifiers = english_names.prefmodifiers, --> style + --subfamily = english_names.subfamily, --> style psname = english_names.postscriptname, }, metadata = { @@ -1289,6 +1305,9 @@ local organize_namedata = function (metadata, basename, info) metafamily = metadata.familyname, }, info = { + fullname = info.fullname, + familyname = info.familyname, + fontname = info.fontname, }, } @@ -1628,7 +1647,8 @@ local read_font_names = function (fullname, return insert_fullinfo (fullname, basename, false, loader, format, location, - targetmappings, targetentrystatus) + targetmappings, targetentrystatus, + info) end local path_normalize -- cgit v1.2.3 From 1f8bcae1bfd697adf582d2a6c1add145eb304f9b Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 19:43:00 +0200 Subject: [db] refactor fontname extraction --- luaotfload-database.lua | 104 +++++++++++++++++++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 23 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 98b0975..b331629 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -189,7 +189,8 @@ local invalidchars = "[^%a%d]" local sanitize_fontname = function (str) if str ~= nil then - return utf8gsub (utf8lower (str), invalidchars, "") + str = utf8gsub (utf8lower (str), invalidchars, "") + return str end return nil end @@ -1269,12 +1270,12 @@ local get_size_info = function (metadata) return false end -local organize_namedata = function (metadata, basename, info) +local get_english_names = function (names, basename) local english_names - if metadata.names then - for _, raw_namedata in next, metadata.names do + if names then + for _, raw_namedata in next, names do if raw_namedata.lang == "English (US)" then english_names = raw_namedata.names end @@ -1287,23 +1288,48 @@ local organize_namedata = function (metadata, basename, info) return nil end + return english_names +end + +local organize_namedata = function (metadata, + english_names, + basename, + info) + + --print (english_names.family, "<>", english_names.preffamilyname) local fontnames = { --- see --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html + --- http://www.microsoft.com/typography/OTSPEC/name.htm#NameIDs english = { + --- where a “compatfull” field is given, the value of “fullname” is + --- either identical or differs by separating the style + --- with a hyphen and omitting spaces. + --- Of the three “fullname” fields, this one appears to be the one + --- with the entire name given in a legible, + --- non-abbreviated fashion, for most fonts at any rate. + --- However, in some fonts (e.g. CMU) all three fields are + --- identical. fullname = english_names.compatfull or english_names.fullname, - family = english_names.preffamilyname - or english_names.family, - --prefmodifiers = english_names.prefmodifiers, --> style - --subfamily = english_names.subfamily, --> style + --- we keep both the “preferred family” and the “family” + --- values around since both are valid but can turn out + --- quite differently, e.g. with Latin Modern: + --- preffamily: “Latin Modern Sans”, + --- family: “LM Sans 10” + preffamily = english_names.preffamilyname, + family = english_names.family, + prefmodifiers = english_names.prefmodifiers, + subfamily = english_names.subfamily, psname = english_names.postscriptname, }, + metadata = { - pfullname = metadata.fullname, + fullname = metadata.fullname, fontname = metadata.fontname, metafamily = metadata.familyname, }, + info = { fullname = info.fullname, familyname = info.familyname, @@ -1311,13 +1337,24 @@ local organize_namedata = function (metadata, basename, info) }, } - -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size +-- print (fontnames.english.fullname, +-- fontnames.metadata.fullname, +-- fontnames.info.fullname) + if metadata.fontstyle_name then + --- not present in all fonts, often differs from the preferred + --- subfamily as well as subfamily fields, e.g. with LMSans10-BoldOblique: + --- subfamily: “Bold Italic” + --- prefmodifiers: “10 Bold Oblique” + --- fontstyle_name: “Bold Oblique” for _, name in next, metadata.fontstyle_name do if name.lang == 1033 then --- I hate magic numbers fontnames.fontstyle_name = name.name end end + + --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", english_names.prefmodifiers) + --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", fontnames.fontstyle_name) end return { @@ -1329,6 +1366,32 @@ local organize_namedata = function (metadata, basename, info) end +local organize_styledata = function (metadata, english_names, info) + local pfminfo = metadata.pfminfo + local names = metadata.names + +-- print (">", pfminfo.avgwidth, +-- (metadata.italicangle == info.italicangle) and "T" or +-- string.format ("%f / %f", metadata.italicangle, info.italicangle), +-- pfminfo.width, pfminfo.weight, info.weight) + return { + -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size + size = get_size_info (metadata), + weight = { + pfminfo.weight, -- integer (multiple of 100?) + sanitize_fontname (info.weight), -- style name + }, + width = pfminfo.width, + italicangle = metadata.italicangle, +-- italicangle = { +-- metadata.italicangle, -- float +-- info.italicangle, -- truncated to integer point size? +-- }, + --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details + units_per_em = metadata.units_per_em, + version = metadata.version, + } +end --[[doc-- The data inside an Opentype font file can be quite heterogeneous. @@ -1344,25 +1407,20 @@ ot_fullinfo = function (filename, basename, format, info) - local styles = { } local metadata = load_font_file (filename, subfont) if not metadata then return nil end - local namedata = organize_namedata (metadata, basename, info) - - - local style = { - size = get_size_info (metadata), - weight = metadata.pfminfo.weight, - width = metadata.pfminfo.width, - slant = metadata.italicangle, - --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details - units_per_em = metadata.units_per_em, - version = metadata.version, - } + local english_names = get_english_names (metadata.names, basename) + local namedata = organize_namedata (metadata, + english_names, + basename, + info) + local style = organize_styledata (metadata, + english_names, + info) return { file = { base = basename, -- cgit v1.2.3 From 6ad39d6f1f45b088173312493f7b3b42eb71979a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 21:02:18 +0200 Subject: [db] add post-build statistics --- luaotfload-database.lua | 148 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 137 insertions(+), 11 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index b331629..475d361 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1304,7 +1304,8 @@ local organize_namedata = function (metadata, english = { --- where a “compatfull” field is given, the value of “fullname” is --- either identical or differs by separating the style - --- with a hyphen and omitting spaces. + --- with a hyphen and omitting spaces. (According to the + --- spec, “compatfull” is “Macintosh only”.) --- Of the three “fullname” fields, this one appears to be the one --- with the entire name given in a legible, --- non-abbreviated fashion, for most fonts at any rate. @@ -1547,7 +1548,7 @@ local compare_timestamps = function (fullname, if targetentrystatus ~= nil and targetentrystatus.timestamp == targettimestamp then - report ("log", 3, "db", "Font %q already read.", basename) + report ("log", 3, "db", "Font %q already read.", fullname) return false end @@ -1570,7 +1571,7 @@ local compare_timestamps = function (fullname, targetentrystatus.index [targetindex + 1] = location end - report ("log", 3, "db", "Font %q already indexed.", basename) + report ("log", 3, "db", "Font %q already indexed.", fullname) return false end @@ -2034,22 +2035,22 @@ end local scan_dir = function (dirname, currentnames, targetnames, dry_run, location) if lpegmatch (p_blacklist, dirname) then - report ("both", 3, "db", + report ("both", 4, "db", "Skipping blacklisted directory %s", dirname) --- ignore return 0, 0 end local found = find_font_files (dirname, location ~= "texmf") if not found then - report ("both", 3, "db", + report ("both", 4, "db", "No such directory: %q; skipping.", dirname) return 0, 0 end - report ("both", 3, "db", "Scanning directory %s", dirname) + report ("both", 4, "db", "Scanning directory %s", dirname) local n_new = 0 --- total of fonts collected local n_found = #found - report ("both", 4, "db", "%d font files detected", n_found) + report ("both", 5, "db", "%d font files detected", n_found) for j=1, n_found do local fullname = found[j] fullname = path_normalize(fullname) @@ -2059,7 +2060,7 @@ local scan_dir = function (dirname, currentnames, targetnames, "Would have been extracting metadata from %q", fullname) else - report ("both", 4, "db", + report ("both", 5, "db", "Extracting metadata from font %q", fullname) local new = read_font_names (fullname, currentnames, targetnames, location) @@ -2069,7 +2070,7 @@ local scan_dir = function (dirname, currentnames, targetnames, end end - report("both", 4, "db", "%d fonts found in '%s'", n_found, dirname) + report("both", 5, "db", "%d fonts found in '%s'", n_found, dirname) return n_found, n_new end @@ -2488,7 +2489,7 @@ local generate_filedata = function (mappings) if inbase then local present = inbase [basename] if present then - report ("both", 3, "db", + report ("both", 4, "db", "Conflicting basename: %q already indexed \z in category %s, ignoring.", barename, location) @@ -2518,7 +2519,7 @@ local generate_filedata = function (mappings) if inbare then local present = inbare [barename] if present then - report ("both", 3, "db", + report ("both", 4, "db", "Conflicting barename: %q already indexed \z in category %s/%s, ignoring.", barename, location, format) @@ -2559,6 +2560,7 @@ local generate_filedata = function (mappings) return filenames end + local retrieve_namedata = function (currentnames, targetnames, dry_run, @@ -2579,6 +2581,129 @@ local retrieve_namedata = function (currentnames, return n_rawnames, n_newnames end + +--- dbobj -> stats + +local collect_statistics = function (mappings) + local sum_dsnsize, n_dsnsize = 0, 0 + + local fullname, family = { }, { } + local subfamily, prefmodifiers, fontstyle_name = { }, { }, { } + + local addtohash = function (hash, item) + if item then + local times = hash [item] + if times then + hash [item] = times + 1 + else + hash [item] = 1 + end + end + end + + local setsize = function (set) + local n = 0 + for _, _ in next, set do + n = n + 1 + end + return n + end + + local hashsum = function (hash) + local n = 0 + for _, m in next, hash do + n = n + m + end + return n + end + + for _, entry in next, mappings do + local style = entry.style + local names = entry.names.sanitized + local englishnames = names.english + + addtohash (fullname, englishnames.fullname) + addtohash (family, englishnames.family) + addtohash (subfamily, englishnames.subfamily) + addtohash (prefmodifiers, englishnames.prefmodifiers) + addtohash (fontstyle_name, names.fontstyle_name) + + local sizeinfo = entry.style.size + if sizeinfo then + sum_dsnsize = sum_dsnsize + sizeinfo [1] + n_dsnsize = n_dsnsize + 1 + end + end + + local n_fullname = setsize (fullname) + local n_family = setsize (family) + + if logs.get_loglevel () > 1 then + report ("both", 0, "", "~~~~ font index statistics ~~~~") + report ("both", 0, "db", + " · Collected %d fonts (%d names) in %d families.", + #mappings, n_fullname, n_family) + + local pprint_top = function (hash, n) + local freqs = { } + local items = { } + for item, freq in next, hash do + local ifreq = items [freq] + if ifreq then + ifreq [#ifreq + 1] = item + else + items [freq] = { item } + freqs [#freqs + 1] = freq + end + end + + tablesort (freqs) + + local from = #freqs + local to = from - (n - 1) + if to < 1 then + to = 1 + end + + for i = from, to, -1 do + local freq = freqs [i] + local itemlist = items [freq] + report ("both", 0, "db", + " · %4d × %s.", + freq, tableconcat (itemlist, ", ")) + end + end + + report ("both", 0, "db", + " · %d different “subfamily” kinds", + setsize (subfamily)) + pprint_top (subfamily, 4) + + report ("both", 0, "db", + " · %d different “prefmodifiers” kinds", + setsize (prefmodifiers)) + pprint_top (prefmodifiers, 4) + + report ("both", 0, "db", + " · %d different “fontstyle_name” kinds", + setsize (fontstyle_name)) + pprint_top (fontstyle_name, 4) + end + + return { + mean_dsnsize = sum_dsnsize / n_dsnsize, + names = { + fullname = n_fullname, + families = n_family, + }, +-- style = { +-- subfamily = subfamily, +-- prefmodifiers = prefmodifiers, +-- fontstyle_name = fontstyle_name, +-- }, + } +end + --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs @@ -2625,6 +2750,7 @@ update_names = function (currentnames, force, dry_run) dry_run, n_rawnames, n_newnames) + targetnames.meta.statistics = collect_statistics (targetnames.mappings) --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping -- cgit v1.2.3 From df1267d2bea13c84056e5a30465d24499c595558 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:09:23 +0200 Subject: [db] adapt function reload_db to new db structure --- luaotfload-database.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 475d361..7026dca 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1107,17 +1107,18 @@ end --- string -> ('a -> 'a) -> 'a list -> 'a reload_db = function (why, caller, ...) local namedata = names.data - local formats = tableconcat (namedata.formats, ",") + local formats = tableconcat (namedata.meta.formats, ",") report ("both", 1, "db", "Reload initiated (formats: %s); reason: %q", formats, why) set_font_filter (formats) - names.data = update_names (names.data, false, false) + namedata = update_names (namedata, false, false) - if names.data then + if namedata then fonts_reloaded = true + names.data = namedata return caller (...) end -- cgit v1.2.3 From 6c4b470d23faf61e4f9b0aab5f871bf87b6707bb Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:10:32 +0200 Subject: [db] fix integer format string --- luaotfload-database.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 7026dca..e77d4fd 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1786,7 +1786,7 @@ local create_blacklist = function (blacklist, whitelist) local result = { } local dirs = { } - report("info", 2, "db", "Blacklisting %q files and directories", + report("info", 2, "db", "Blacklisting %d files and directories", #blacklist) for i=1, #blacklist do local entry = blacklist[i] @@ -1797,7 +1797,7 @@ local create_blacklist = function (blacklist, whitelist) end end - report("info", 2, "db", "Whitelisting %q files", #whitelist) + report("info", 2, "db", "Whitelisting %d files", #whitelist) for i=1, #whitelist do result[whitelist[i]] = nil end -- cgit v1.2.3 From 7163d50ce6e5deaefaa32f9a6d0bbcc778fcb24b Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:14:31 +0200 Subject: [db] intercept possible division by zero --- luaotfload-database.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e77d4fd..bae90c3 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2691,8 +2691,13 @@ local collect_statistics = function (mappings) pprint_top (fontstyle_name, 4) end + local mean_dsnsize = 0 + if n_dsnsize > 0 then + mean_dsnsize = sum_dsnsize / n_dsnsize + end + return { - mean_dsnsize = sum_dsnsize / n_dsnsize, + mean_dsnsize = mean_dsnsize, names = { fullname = n_fullname, families = n_family, -- cgit v1.2.3 From 8bd26bbc573d4320fce8d6535737dd25e277bd2f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:45:12 +0200 Subject: [db] print family sizes along with stats --- luaotfload-database.lua | 55 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bae90c3..8268283 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2588,7 +2588,7 @@ end local collect_statistics = function (mappings) local sum_dsnsize, n_dsnsize = 0, 0 - local fullname, family = { }, { } + local fullname, family, families = { }, { }, { } local subfamily, prefmodifiers, fontstyle_name = { }, { }, { } local addtohash = function (hash, item) @@ -2602,6 +2602,28 @@ local collect_statistics = function (mappings) end end + local appendtohash = function (hash, key, value) + if key and value then + local entry = hash [key] + if entry then + entry [#entry + 1] = value + else + hash [key] = { value } + end + end + end + + local addtoset = function (hash, key, value) + if key and value then + local set = hash [key] + if set then + set [value] = true + else + hash [key] = { [value] = true } + end + end + end + local setsize = function (set) local n = 0 for _, _ in next, set do @@ -2629,6 +2651,8 @@ local collect_statistics = function (mappings) addtohash (prefmodifiers, englishnames.prefmodifiers) addtohash (fontstyle_name, names.fontstyle_name) + addtoset (families, englishnames.family, englishnames.fullname) + local sizeinfo = entry.style.size if sizeinfo then sum_dsnsize = sum_dsnsize + sizeinfo [1] @@ -2636,19 +2660,23 @@ local collect_statistics = function (mappings) end end + --inspect (families) + local n_fullname = setsize (fullname) local n_family = setsize (family) if logs.get_loglevel () > 1 then - report ("both", 0, "", "~~~~ font index statistics ~~~~") - report ("both", 0, "db", - " · Collected %d fonts (%d names) in %d families.", - #mappings, n_fullname, n_family) + local pprint_top = function (hash, n, set) - local pprint_top = function (hash, n) local freqs = { } local items = { } - for item, freq in next, hash do + + for item, value in next, hash do + if set then + freq = setsize (value) + else + freq = value + end local ifreq = items [freq] if ifreq then ifreq [#ifreq + 1] = item @@ -2669,12 +2697,23 @@ local collect_statistics = function (mappings) for i = from, to, -1 do local freq = freqs [i] local itemlist = items [freq] + + if type (itemlist) == "table" then + itemlist = tableconcat (itemlist, ", ") + end + report ("both", 0, "db", " · %4d × %s.", - freq, tableconcat (itemlist, ", ")) + freq, itemlist) end end + report ("both", 0, "", "~~~~ font index statistics ~~~~") + report ("both", 0, "db", + " · Collected %d fonts (%d names) in %d families.", + #mappings, n_fullname, n_family) + pprint_top (families, 4, true) + report ("both", 0, "db", " · %d different “subfamily” kinds", setsize (subfamily)) -- cgit v1.2.3 From e29c39733ad90a0ca354808aef71b1b2e0acc9c9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:49:14 +0200 Subject: [tool] include stats on demand only (--stats option) --- luaotfload-database.lua | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 8268283..a148940 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2795,7 +2795,11 @@ update_names = function (currentnames, force, dry_run) dry_run, n_rawnames, n_newnames) - targetnames.meta.statistics = collect_statistics (targetnames.mappings) + + if config.luaotfload.statistics == true then + targetnames.meta.statistics = collect_statistics + (targetnames.mappings) + end --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping -- cgit v1.2.3 From 1674a5aa95a3305845364daee05c2ec8279e2703 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 2 Sep 2013 00:07:25 +0200 Subject: [db] adjust index field for non tt collections --- luaotfload-database.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index a148940..e5627ef 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1559,7 +1559,6 @@ local compare_timestamps = function (fullname, if currenttimestamp == targettimestamp and not targetentrystatus.index [1] then - --- copy old namedata into new for _, currentindex in next, currententrystatus.index do @@ -1590,7 +1589,13 @@ local insert_fullinfo = function (fullname, targetentrystatus, info) - local subfont = n_font and n_font - 1 or false + local subfont + if n_font ~= false then + subfont = n_font - 1 + else + subfont = false + n_font = 1 + end local fullinfo = loader (fullname, subfont, location, basename, @@ -2567,6 +2572,7 @@ local retrieve_namedata = function (currentnames, dry_run, n_rawnames, n_newnames) + local rawnames, new = scan_texmf_fonts (currentnames, targetnames, dry_run) -- cgit v1.2.3 From 18f6b49c8c5903926e899abaf8efdada6947fbde Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 2 Sep 2013 00:09:47 +0200 Subject: [db] fix table constructor syntax --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e5627ef..323804c 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2544,7 +2544,7 @@ local generate_filedata = function (mappings) inbare [barename] = index end else - inbare = { barename = index } + inbare = { [barename] = index } bare [location] [format] = inbare end -- cgit v1.2.3 From 0fe0554e033e1f87c4bf031b37791d0bd2f435e0 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 11 Sep 2013 13:30:30 +0200 Subject: [db] prepare families table --- luaotfload-database.lua | 222 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 212 insertions(+), 10 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 323804c..2b98d92 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -60,7 +60,6 @@ local utf8gsub = unicode.utf8.gsub local utf8lower = unicode.utf8.lower --- these come from Lualibs/Context -local getwritablepath = caches.getwritablepath local filebasename = file.basename local filecollapsepath = file.collapsepath or file.collapse_path local filedirname = file.dirname @@ -71,9 +70,11 @@ local filenameonly = file.nameonly local filereplacesuffix = file.replacesuffix local filesplitpath = file.splitpath or file.split_path local filesuffix = file.suffix +local getwritablepath = caches.getwritablepath local lfsisdir = lfs.isdir local lfsisfile = lfs.isfile local lfsmkdirs = lfs.mkdirs +local lpegsplitat = lpeg.splitat local stringis_empty = string.is_empty local stringsplit = string.split local stringstrip = string.strip @@ -375,11 +376,7 @@ mtx-fonts has in names.tma: local initialize_namedata = function (formats) --- returns dbobj return { - families = { - ["local"] = { }, - system = { }, - texmf = { }, - }, + --families = { }, status = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite names = { }, @@ -1368,7 +1365,20 @@ local organize_namedata = function (metadata, end -local organize_styledata = function (metadata, english_names, info) + +local dashsplitter = lpegsplitat "-" + +local split_fontname = function (fontname) + --- sometimes the style hides in the latter part of the + --- fontname, separated by a dash, e.g. “Iwona-Regular”, + --- “GFSSolomos-Regular” + local splitted = { lpegmatch (dashsplitter, fontname) } + if splitted then + return sanitize_fontname (splitted [#splitted]) + end +end + +local organize_styledata = function (fontname, metadata, english_names, info) local pfminfo = metadata.pfminfo local names = metadata.names @@ -1383,6 +1393,7 @@ local organize_styledata = function (metadata, english_names, info) pfminfo.weight, -- integer (multiple of 100?) sanitize_fontname (info.weight), -- style name }, + split = split_fontname (fontname), width = pfminfo.width, italicangle = metadata.italicangle, -- italicangle = { @@ -1420,7 +1431,8 @@ ot_fullinfo = function (filename, english_names, basename, info) - local style = organize_styledata (metadata, + local style = organize_styledata (namedata.fontname, + metadata, english_names, info) @@ -1432,7 +1444,6 @@ ot_fullinfo = function (filename, format = format, names = namedata, style = style, - units_per_em = metadata.units_per_em, version = metadata.version, } end @@ -2566,6 +2577,196 @@ local generate_filedata = function (mappings) return filenames end +local match_synonyms = function (pattern) + local nopattern = 1 - pattern + return nopattern^0 * pattern * Cc (true) + Cc (false) +end + +local determine_italic +local determine_bold +local pick_style +local check_regular + +do + local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") + local bold = match_synonyms (P"bold" + P"demi", P"heavy", P"black", P"ultra") + + determine_italic = function (fontstyle_name, + italicangle, + prefmodifiers, + subfamily) + if italicangle ~= 0 then + return true + elseif fontstyle_name and lpegmatch (italic, fontstyle_name) then + return true + elseif prefmodifiers and lpegmatch (italic, prefmodifiers) then + return lpegmatch (italic, prefmodifiers) + else + return lpegmatch (italic, subfamily) + end + end + + determine_bold = function (fontstyle_name, + weight, + prefmodifiers, + subfamily) + if weight [2] == "bold" then + return true + elseif fontstyle_name and lpegmatch (bold, fontstyle_name) then + return true + elseif prefmodifiers and lpegmatch (bold, prefmodifiers) then + return true + else + return lpegmatch (bold, subfamily) + end + end + + local splitfontname = lpeg.splitat "-" + + local choose_exact = function (field) + if field == "italic" or field == "oblique" then + return "i" + elseif field == "bold" then + return "b" + elseif field == "bolditalic" or field == "boldoblique" then + return "bi" + end + + return false + end + + pick_style = function (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + local style + if fontstyle_name ~= nil then + style = choose_exact (prefmodifiers) + end + if not style and prefmodifiers ~= nil then + style = choose_exact (prefmodifiers) + end + if not style then + style = choose_exact (subfamily) + end + if not style and splitstyle ~= nil then + choose_exact (splitstyle) + end + return style + end + + --- we use only exact matches here since there are constructs + --- like “regularitalic” (Cabin, Bodoni Old Fashion) + + check_regular = function (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + + if fontstyle_name == "regular" or fontstyle_name == "book" then + return "r" + end + + if prefmodifiers == "regular" or prefmodifiers == "book" then + return "r" + end + + if subfamily == "regular" or subfamily == "book" then + return "r" + end + + if splitstyle == "regular" or splitstyle == "book" then + return "r" + end + + return nil + end +end + +local collect_families = function (mappings) + + local families = { + ["local"] = { }, + system = { }, + texmf = { }, + } + + for i = 1, #mappings do + + local entry = mappings [i] + local file = entry.file + local names = entry.names + local style = entry.style + + local location = file.location + local format = entry.format + local english = names.sanitized.english + local info = names.sanitized.info + + local subtable = families [location] [format] + if not subtable then + subtable = { } + families [location] [format] = subtable + end + + local familyname = english.preffamily + or english.family + or info.familyname + local fullname = english.fullname or info.fullname + local fontname = info.fontname + + local fontstyle_name = names.sanitized.fontstyle_name + local prefmodifiers = english.prefmodifiers or "regular" + local subfamily = english.subfamily + + local weight = style.weight + local italicangle = style.italicangle + local splitstyle = style.split + + local modifier = pick_style (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + + if not modifier then --- guess + local italic = determine_italic (fontstyle_name, + italicangle, + prefmodifiers, + subfamily) + local bold = determine_bold (fontstyle_name, + weight, + prefmodifiers, + subfamily) + if bold and italic then + modifier = "bi" + elseif bold then + modifier = "b" + elseif italic then + modifier = "i" + end + end + + if not modifier then --- regular, exact only + modifier = check_regular (fontstyle_name, + subfamily, + splitstyle) + end + + if modifier then + --- stub; here we will continue building a list of optical sizes + --- no size -> hash “normal” + --- other sizes -> indexed tuples { dsnsize, idx } + local familytable = subtable [familyname] + if not familytable then + familytable = { } + subtable [familyname] = familytable + end + familytable [modifier] = entry.index + end + end + + return families +end local retrieve_namedata = function (currentnames, targetnames, @@ -2811,7 +3012,8 @@ update_names = function (currentnames, force, dry_run) --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - targetnames.files = generate_filedata (targetnames.mappings) + targetnames.files = generate_filedata (targetnames.mappings) + targetnames.families = collect_families (targetnames.mappings) --- stats: --- before rewrite | after rewrite -- cgit v1.2.3 From b3fa18a7d99d55d2daaeebd7f17db664719ef690 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 13 Sep 2013 23:27:55 +0200 Subject: [fontloader] sync with Context as of 2013-09-13 --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 2b98d92..5032955 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1373,7 +1373,7 @@ local split_fontname = function (fontname) --- fontname, separated by a dash, e.g. “Iwona-Regular”, --- “GFSSolomos-Regular” local splitted = { lpegmatch (dashsplitter, fontname) } - if splitted then + if next (splitted) then return sanitize_fontname (splitted [#splitted]) end end -- cgit v1.2.3 From cd5707a41b61198d0a53ab7cd4acbcc9240d0487 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 15 Sep 2013 22:12:42 +0200 Subject: [db] handle design sizes --- luaotfload-database.lua | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 5032955..d3f2f08 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2457,7 +2457,7 @@ end local generate_filedata = function (mappings) - report ("both", 2, "db", "Creating filename map") + report ("both", 2, "db", "Creating filename map.") local nmappings = #mappings @@ -2685,6 +2685,8 @@ end local collect_families = function (mappings) + report ("info", 2, "db", "Analyzing families, sizes, and styles.") + local families = { ["local"] = { }, system = { }, @@ -2716,7 +2718,7 @@ local collect_families = function (mappings) local fontname = info.fontname local fontstyle_name = names.sanitized.fontstyle_name - local prefmodifiers = english.prefmodifiers or "regular" + local prefmodifiers = english.prefmodifiers local subfamily = english.subfamily local weight = style.weight @@ -2755,13 +2757,35 @@ local collect_families = function (mappings) if modifier then --- stub; here we will continue building a list of optical sizes --- no size -> hash “normal” - --- other sizes -> indexed tuples { dsnsize, idx } + --- other sizes -> indexed tuples { dsnsize, high, low, idx } local familytable = subtable [familyname] if not familytable then familytable = { } subtable [familyname] = familytable end - familytable [modifier] = entry.index + + --- the style table is treated as an unordered list + local styletable = familytable [modifier] + if not styletable then + styletable = { } + familytable [modifier] = styletable + end + + if not prefmodifiers then --- default size for this style/family combo + styletable.default = entry.index + end + + local size = style.size --- dsnsize * high * low + if size then + styletable [#styletable + 1] = { + size [1], + size [2], + size [3], + entry.index, + } + else + styletable.default = entry.index + end end end -- cgit v1.2.3 From 9ab1917e9b492abaef43a9d9a45ceba23391003f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 15 Sep 2013 22:46:21 +0200 Subject: [tool] add option to skip reading font files --- luaotfload-database.lua | 229 +++++++++++++++++++++++++----------------------- 1 file changed, 118 insertions(+), 111 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index d3f2f08..bb7f17e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -20,101 +20,100 @@ local C, Cc, Cf, Cg, Cs, Ct = lpeg.C, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Cs, lpeg.Ct --- Luatex builtins -local load = load -local next = next -local pcall = pcall -local require = require -local tonumber = tonumber -local unpack = table.unpack - -local fontloaderinfo = fontloader.info -local fontloaderclose = fontloader.close -local fontloaderopen = fontloader.open -local fontloaderto_table = fontloader.to_table -local iolines = io.lines -local ioopen = io.open -local kpseexpand_path = kpse.expand_path -local kpseexpand_var = kpse.expand_var -local kpsefind_file = kpse.find_file -local kpselookup = kpse.lookup -local kpsereadable_file = kpse.readable_file -local lfsattributes = lfs.attributes -local lfschdir = lfs.chdir -local lfscurrentdir = lfs.currentdir -local lfsdir = lfs.dir -local mathabs = math.abs -local mathmin = math.min -local osgettimeofday = os.gettimeofday -local osremove = os.remove -local stringfind = string.find -local stringformat = string.format -local stringgmatch = string.gmatch -local stringgsub = string.gsub -local stringlower = string.lower -local stringsub = string.sub -local stringupper = string.upper -local tableconcat = table.concat -local tablesort = table.sort -local texiowrite_nl = texio.write_nl -local utf8gsub = unicode.utf8.gsub -local utf8lower = unicode.utf8.lower +local load = load +local next = next +local pcall = pcall +local require = require +local tonumber = tonumber +local unpack = table.unpack + +local fontloaderinfo = fontloader.info +local fontloaderclose = fontloader.close +local fontloaderopen = fontloader.open +local fontloaderto_table = fontloader.to_table +local iolines = io.lines +local ioopen = io.open +local kpseexpand_path = kpse.expand_path +local kpseexpand_var = kpse.expand_var +local kpsefind_file = kpse.find_file +local kpselookup = kpse.lookup +local kpsereadable_file = kpse.readable_file +local lfsattributes = lfs.attributes +local lfschdir = lfs.chdir +local lfscurrentdir = lfs.currentdir +local lfsdir = lfs.dir +local mathabs = math.abs +local mathmin = math.min +local osgettimeofday = os.gettimeofday +local osremove = os.remove +local stringfind = string.find +local stringformat = string.format +local stringgmatch = string.gmatch +local stringgsub = string.gsub +local stringlower = string.lower +local stringsub = string.sub +local stringupper = string.upper +local tableconcat = table.concat +local tablesort = table.sort +local texiowrite_nl = texio.write_nl +local utf8gsub = unicode.utf8.gsub +local utf8lower = unicode.utf8.lower --- these come from Lualibs/Context -local filebasename = file.basename -local filecollapsepath = file.collapsepath or file.collapse_path -local filedirname = file.dirname -local fileextname = file.extname -local fileiswritable = file.iswritable -local filejoin = file.join -local filenameonly = file.nameonly -local filereplacesuffix = file.replacesuffix -local filesplitpath = file.splitpath or file.split_path -local filesuffix = file.suffix -local getwritablepath = caches.getwritablepath -local lfsisdir = lfs.isdir -local lfsisfile = lfs.isfile -local lfsmkdirs = lfs.mkdirs -local lpegsplitat = lpeg.splitat -local stringis_empty = string.is_empty -local stringsplit = string.split -local stringstrip = string.strip -local tableappend = table.append -local tablecopy = table.copy -local tablefastcopy = table.fastcopy -local tabletofile = table.tofile -local tabletohash = table.tohash +local filebasename = file.basename +local filecollapsepath = file.collapsepath or file.collapse_path +local filedirname = file.dirname +local fileextname = file.extname +local fileiswritable = file.iswritable +local filejoin = file.join +local filenameonly = file.nameonly +local filereplacesuffix = file.replacesuffix +local filesplitpath = file.splitpath or file.split_path +local filesuffix = file.suffix +local getwritablepath = caches.getwritablepath +local lfsisdir = lfs.isdir +local lfsisfile = lfs.isfile +local lfsmkdirs = lfs.mkdirs +local lpegsplitat = lpeg.splitat +local stringis_empty = string.is_empty +local stringsplit = string.split +local stringstrip = string.strip +local tableappend = table.append +local tablecopy = table.copy +local tablefastcopy = table.fastcopy +local tabletofile = table.tofile +local tabletohash = table.tohash --- the font loader namespace is “fonts”, same as in Context --- we need to put some fallbacks into place for when running --- as a script -fonts = fonts or { } -fonts.names = fonts.names or { } -fonts.definers = fonts.definers or { } +fonts = fonts or { } +fonts.names = fonts.names or { } +fonts.definers = fonts.definers or { } -local names = fonts.names +local names = fonts.names -config = config or { } -config.luaotfload = config.luaotfload or { } -config.luaotfload.resolver = config.luaotfload.resolver or "normal" -config.luaotfload.formats = config.luaotfload.formats or "otf,ttf,ttc,dfont" +local luaotfloadconfig = config.luaotfload --- always present +luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" +luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" -if config.luaotfload.update_live ~= false then +if luaotfloadconfig.update_live ~= false then --- this option allows for disabling updates --- during a TeX run - config.luaotfload.update_live = true + luaotfloadconfig.update_live = true end -names.version = 2.4 -names.data = nil --- contains the loaded database -names.lookups = nil --- contains the lookup cache +names.version = 2.4 +names.data = nil --- contains the loaded database +names.lookups = nil --- contains the lookup cache -names.path = { index = { }, lookups = { } } -names.path.globals = { - prefix = "", --- writable_path/names_dir - names_dir = config.luaotfload.names_dir or "names", - index_file = config.luaotfload.index_file - or "luaotfload-names.lua", - lookups_file = "luaotfload-lookup-cache.lua", +names.path = { index = { }, lookups = { } } +names.path.globals = { + prefix = "", --- writable_path/names_dir + names_dir = luaotfloadconfig.names_dir or "names", + index_file = luaotfloadconfig.index_file + or "luaotfload-names.lua", + lookups_file = "luaotfload-lookup-cache.lua", } --- string -> (string * string) @@ -1937,7 +1936,7 @@ do end --- initialize - set_font_filter (config.luaotfload.formats) + set_font_filter (luaotfloadconfig.formats) end local process_dir_tree @@ -2565,7 +2564,7 @@ local generate_filedata = function (mappings) end --- TODO adapt to new mechanism! --- if config.luaotfload.prioritize == "texmf" then +-- if luaotfloadconfig.prioritize == "texmf" then -- report("both", 2, "db", "Preferring texmf fonts") -- addmap(sys) -- addmap(texmf) @@ -2986,7 +2985,9 @@ end --- dbobj? -> bool? -> bool? -> dbobj update_names = function (currentnames, force, dry_run) - if config.luaotfload.update_live == false then + local targetnames + + if luaotfloadconfig.update_live == false then report ("info", 2, "db", "Skipping database update") --- skip all db updates @@ -3004,30 +3005,43 @@ update_names = function (currentnames, force, dry_run) report ("both", 2, "db", "Updating the font names database" .. (force and " forcefully" or "")) - if force then - currentnames = initialize_namedata (get_font_filter ()) + if luaotfloadconfig.skip_read == true then + --- the difference to a “dry run” is that we don’t search + --- for font files entirely. we also ignore the “force” + --- parameter since it concerns only the font files. + report ("info", 2, "db", + "Ignoring font files, reusing old data.") + currentnames = load_names (false) + targetnames = currentnames else - if not currentnames then - currentnames = load_names (dry_run) - end - if currentnames.meta.version ~= names.version then - report ("both", 1, "db", "No font names database or old " - .. "one found; generating new one") + if force then currentnames = initialize_namedata (get_font_filter ()) + else + if not currentnames then + currentnames = load_names (dry_run) + end + if currentnames.meta.version ~= names.version then + report ("both", 1, "db", "No font names database or old " + .. "one found; generating new one") + currentnames = initialize_namedata (get_font_filter ()) + end end - end - local targetnames = initialize_namedata (get_font_filter ()) + targetnames = initialize_namedata (get_font_filter ()) - read_blacklist () + read_blacklist () - local n_rawnames, n_newnames = retrieve_namedata (currentnames, - targetnames, - dry_run, - n_rawnames, - n_newnames) + local n_raw, n_new= retrieve_namedata (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) + report ("info", 3, "db", + "Scanned %d font files; %d new entries.", + n_rawnames, n_newnames) + end - if config.luaotfload.statistics == true then + if luaotfloadconfig.statistics == true then targetnames.meta.statistics = collect_statistics (targetnames.mappings) end @@ -3039,13 +3053,6 @@ update_names = function (currentnames, force, dry_run) targetnames.files = generate_filedata (targetnames.mappings) targetnames.families = collect_families (targetnames.mappings) - --- stats: - --- before rewrite | after rewrite - --- partial: 804 ms | 701 ms - --- forced: 45384 ms | 44714 ms - report ("info", 3, "db", - "Scanned %d font files; %d new entries.", - n_rawnames, n_newnames) report ("info", 3, "db", "Rebuilt in %0.f ms.", 1000 * (osgettimeofday () - starttime)) @@ -3205,7 +3212,7 @@ end local getwritablecachepath = function ( ) --- fonts.handlers.otf doesn’t exist outside a Luatex run, --- so we have to improvise - local writable = getwritablepath (config.luaotfload.cache_dir) + local writable = getwritablepath (luaotfloadconfig.cache_dir) if writable then return writable end @@ -3213,7 +3220,7 @@ end local getreadablecachepaths = function ( ) local readables = caches.getreadablepaths - (config.luaotfload.cache_dir) + (luaotfloadconfig.cache_dir) local result = { } if readables then for i=1, #readables do @@ -3297,7 +3304,7 @@ names.erase_cache = erase_cache names.show_cache = show_cache --- replace the resolver from luatex-fonts -if config.luaotfload.resolver == "cached" then +if luaotfloadconfig.resolver == "cached" then report("both", 2, "cache", "caching of name: lookups active") names.resolve = resolve_cached names.resolvespec = resolve_cached -- cgit v1.2.3 From b8f01ab3aa15499ab7f883e9dfce47a235a3e9dd Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 15 Sep 2013 22:58:52 +0200 Subject: [db] order design sizes --- luaotfload-database.lua | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bb7f17e..6c52b5e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2791,6 +2791,27 @@ local collect_families = function (mappings) return families end +local cmp_sizes = function (a, b) + return a [1] < b [1] +end + +local order_design_sizes = function (families) + + report ("info", 2, "db", "Ordering design sizes.") + + for location, data in next, families do + for format, data in next, data do + for familyname, data in next, data do + for style, data in next, data do + tablesort (data, cmp_sizes) + end + end + end + end + + return families +end + local retrieve_namedata = function (currentnames, targetnames, dry_run, @@ -3005,6 +3026,9 @@ update_names = function (currentnames, force, dry_run) report ("both", 2, "db", "Updating the font names database" .. (force and " forcefully" or "")) + --- pass 1 get raw data: read font files (normal case) or reuse + --- information present in index + if luaotfloadconfig.skip_read == true then --- the difference to a “dry run” is that we don’t search --- for font files entirely. we also ignore the “force” @@ -3041,6 +3065,7 @@ update_names = function (currentnames, force, dry_run) n_rawnames, n_newnames) end + --- pass 2 (optional): collect some stats about the raw font info if luaotfloadconfig.statistics == true then targetnames.meta.statistics = collect_statistics (targetnames.mappings) @@ -3050,9 +3075,16 @@ update_names = function (currentnames, force, dry_run) --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent + --- pass 3: build filename table targetnames.files = generate_filedata (targetnames.mappings) + + --- pass 4: build family lookup table targetnames.families = collect_families (targetnames.mappings) + --- pass 5: order design size tables + targetnames.families = order_design_sizes (targetnames.families) + + report ("info", 3, "db", "Rebuilt in %0.f ms.", 1000 * (osgettimeofday () - starttime)) -- cgit v1.2.3 From 31abae6c9f15d75bf0b09be212eef35ae124b4a8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 18:57:31 +0100 Subject: [db] fix test for successful update --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 6c52b5e..aa1370f 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -477,7 +477,7 @@ load_names = function (dry_run) [[This can take several minutes; please be patient.]]) data = update_names (initialize_namedata (get_font_filter ()), nil, dry_run) - if not success then + if not data then report ("both", 0, "db", "Database creation unsuccessful.") end end -- cgit v1.2.3 From 5e5f8de85320ba0ed005fc1676e6df4d877845df Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 19:30:46 +0100 Subject: [db,tool] add --compress option for gzipping the index --- luaotfload-database.lua | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index aa1370f..1c694fe 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,6 +31,7 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table +local gzipsave = gzip.save local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -83,6 +84,7 @@ local tablecopy = table.copy local tablefastcopy = table.fastcopy local tabletofile = table.tofile local tabletohash = table.tohash +local tableserialize = table.serialize --- the font loader namespace is “fonts”, same as in Context --- we need to put some fallbacks into place for when running @@ -3134,6 +3136,11 @@ save_lookups = function ( ) return false end +local tabletogzip = function (filename, ...) + local serialized = tableserialize(true, ...) + gzipsave (filenmaeserialized) +end + --- save_names() is usually called without the argument --- dbobj? -> bool save_names = function (currentnames) @@ -3143,9 +3150,15 @@ save_names = function (currentnames) local path = names.path.index local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, currentnames, true) osremove (lucname) - caches.compile (currentnames, luaname, lucname) + if luaotfloadconfig.compress then + local serialized = tableserialize (currentnames, true) + gzipsave (luaname, serialized) + caches.compile (currentnames, "", lucname) + else + tabletofile (luaname, currentnames, true) + caches.compile (currentnames, luaname, lucname) + end if lfsisfile (luaname) and lfsisfile (lucname) then report ("info", 1, "db", "Font index saved") report ("info", 3, "db", "Text: " .. luaname) -- cgit v1.2.3 From 2f0fb9ef09f796e493776e4ffec164fb014e4283 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 19:53:03 +0100 Subject: [db] add gzip check to script loader --- luaotfload-database.lua | 51 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 16 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1c694fe..df2199e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,6 +31,7 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table +local gzipload = gzip.load local gzipsave = gzip.save local iolines = io.lines local ioopen = io.open @@ -396,27 +397,37 @@ end --- string -> (string * table) local load_lua_file = function (path) - local foundname = filereplacesuffix(path, "luc") + local foundname = filereplacesuffix (path, "luc") - local fh = ioopen(foundname, "rb") -- try bin first + if false then + local fh = ioopen (foundname, "rb") -- try bin first if fh then local chunk = fh:read"*all" fh:close() - code = load(chunk, "b") + code = load (chunk, "b") end +end if not code then --- fall back to text file - foundname = filereplacesuffix(path, "lua") + foundname = filereplacesuffix (path, "lua") fh = ioopen(foundname, "rb") if fh then local chunk = fh:read"*all" fh:close() - code = load(chunk, "t") + code = load (chunk, "t") + end + end + + if not code then --- probe gzipped file + foundname = filereplacesuffix (path, "lua.gz") + local chunk = gzipload (foundname) + if chunk then + code = load (chunk, "t") end end if not code then return nil, nil end - return foundname, code() + return foundname, code () end --- define locals in scope @@ -3136,11 +3147,6 @@ save_lookups = function ( ) return false end -local tabletogzip = function (filename, ...) - local serialized = tableserialize(true, ...) - gzipsave (filenmaeserialized) -end - --- save_names() is usually called without the argument --- dbobj? -> bool save_names = function (currentnames) @@ -3151,22 +3157,35 @@ save_names = function (currentnames) local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then osremove (lucname) + local gzname = luaname .. ".gz" if luaotfloadconfig.compress then local serialized = tableserialize (currentnames, true) - gzipsave (luaname, serialized) + gzipsave (gzname, serialized) caches.compile (currentnames, "", lucname) else tabletofile (luaname, currentnames, true) caches.compile (currentnames, luaname, lucname) end - if lfsisfile (luaname) and lfsisfile (lucname) then - report ("info", 1, "db", "Font index saved") + report ("info", 1, "db", "Font index saved at ...") + local success = false + if lfsisfile (luaname) then report ("info", 3, "db", "Text: " .. luaname) + success = true + end + if lfsisfile (gzname) then + report ("info", 3, "db", "Gzip: " .. gzname) + success = true + end + if lfsisfile (lucname) then report ("info", 3, "db", "Byte: " .. lucname) + success = true + end + if success then return true + else + report ("info", 0, "db", "Could not compile font index") + return false end - report ("info", 0, "db", "Could not compile font index") - return false end report ("info", 0, "db", "Index file not writable") if not fileiswritable (luaname) then -- cgit v1.2.3 From 8e65e677bb678de920ea55097b8889b4245c0e15 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 4 Nov 2013 20:45:27 +0100 Subject: [db] stop exposing the global index object --- luaotfload-database.lua | 100 ++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 50 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index df2199e..cba2f26 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -95,6 +95,7 @@ fonts.names = fonts.names or { } fonts.definers = fonts.definers or { } local names = fonts.names +local name_index = nil -- upvalue for names.data local luaotfloadconfig = config.luaotfload --- always present luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" @@ -271,7 +272,7 @@ This is a sketch of the luaotfload db: type dbobj = { families : familytable; - filenames : filemap; + files : filemap; status : filestatus; mappings : fontentry list; meta : metadata; @@ -382,7 +383,7 @@ local initialize_namedata = function (formats) --- returns dbobj status = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite names = { }, --- filenames = { }, -- created later +-- files = { }, -- created later meta = { formats = formats, statistics = { }, @@ -442,7 +443,7 @@ local load_lookups local read_blacklist local read_fonts_conf local reload_db -local resolve +local resolve_name local resolve_cached local resolve_fullpath local save_names @@ -452,7 +453,6 @@ local get_font_filter local set_font_filter --- state of the database -local fonts_loaded = false local fonts_reloaded = false --- limit output when approximate font matching (luaotfload-tool -F) @@ -494,7 +494,6 @@ load_names = function (dry_run) report ("both", 0, "db", "Database creation unsuccessful.") end end - fonts_loaded = true return data end @@ -590,18 +589,17 @@ end --- string -> (string * string * bool) crude_file_lookup_verbose = function (filename) - if not names.data then names.data = load_names() end - local data = names.data - local mappings = data.mappings - local filenames = data.filenames + if not name_index then name_index = load_names() end + local mappings = name_index.mappings + local files = name_index.files local found --- look up in db first ... - found = verbose_lookup(filenames, "bare", filename) + found = verbose_lookup(files, "bare", filename) if found then return found, nil, true end - found = verbose_lookup(filenames, "base", filename) + found = verbose_lookup(files, "base", filename) if found then return found, nil, true end @@ -618,18 +616,17 @@ end --- string -> (string * string * bool) crude_file_lookup = function (filename) - if not names.data then names.data = load_names() end - local data = names.data - local mappings = data.mappings - local filenames = data.filenames + if not name_index then name_index = load_names() end + local mappings = name_index.mappings + local files = name_index.files local found - found = filenames.base[filename] - or filenames.bare[filename] + found = files.base[filename] + or files.bare[filename] if found then - found = filenames.full[found] + found = files.full[found] if found == nil then found = dummy_findfile(filename) end @@ -672,15 +669,15 @@ the texmf or filesystem. --doc]]-- local verify_font_file = function (basename) - if not names.data then names.data = load_names() end - local filenames = names.data.filenames - local idx = filenames.base[basename] + if not name_index then name_index = load_names() end + local files = name_index.files + local idx = files.base[basename] if not idx then return false end --- firstly, check filesystem - local fullname = filenames.full[idx] + local fullname = files.full[idx] if fullname and lfsisfile(fullname) then return true end @@ -881,6 +878,7 @@ the font database created by the luaotfload-tool script. --- 'a -> 'a -> table -> (string * string | bool * bool) --- +--[===[ resolve = function (_, _, specification) -- the 1st two parameters are used by ConTeXt if not fonts_loaded then names.data = load_names() end local data = names.data @@ -1006,7 +1004,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C --- “found” is really synonymous with “registered in the db”. local entry = found[1] local success, filename, subfont - = get_font_file(data.filenames.full, entry) + = get_font_file(data.files.full, entry) if success == true then report("log", 0, "resolve", "Font family='%s', subfamily='%s' found: %s", @@ -1047,7 +1045,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C end local success, filename, subfont - = get_font_file(data.filenames.full, match) + = get_font_file(data.files.full, match) if success == true then report("log", 0, "resolve", "Font family='%s', subfamily='%s' found: %s", @@ -1058,7 +1056,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C elseif fallback then local success, filename, subfont - = get_font_file(data.filenames.full, fallback) + = get_font_file(data.files.full, fallback) if success == true then report("log", 0, "resolve", "No exact match for request %s; using fallback", @@ -1074,7 +1072,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C --- pick the first candidate encountered local entry = candidates[1] local success, filename, subfont - = get_font_file(data.filenames.full, entry) + = get_font_file(data.files.full, entry) if success == true then report("log", 0, "resolve", "Font family='%s', subfamily='%s' found: %s", @@ -1096,16 +1094,19 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C --- else, default to requested name return specification.name, false, false end --- resolve() +]===] + +resolve_name = function (specification) + if not name_index then name_index = load_names () end +end resolve_fullpath = function (fontname, ext) --- getfilename() - if not fonts_loaded then - names.data = load_names() - end - local filenames = names.data.filenames - local idx = filenames.base[fontname] - or filenames.bare[fontname] + if not name_index then name_index = load_names () end + local files = name_index.files + local idx = files.base[fontname] + or files.bare[fontname] if idx then - return filenames.full[idx] + return files.full[idx] end return "" end @@ -1115,7 +1116,7 @@ end --- string -> ('a -> 'a) -> 'a list -> 'a reload_db = function (why, caller, ...) - local namedata = names.data + local namedata = name_index local formats = tableconcat (namedata.meta.formats, ",") report ("both", 1, "db", @@ -1127,7 +1128,7 @@ reload_db = function (why, caller, ...) if namedata then fonts_reloaded = true - names.data = namedata + name_index = namedata return caller (...) end @@ -1167,16 +1168,14 @@ find_closest = function (name, limit) local name = sanitize_fontname (name) limit = limit or fuzzy_limit - if not fonts_loaded then names.data = load_names() end - - local data = names.data - - if type(data) ~= "table" then + if not name_index then name_index = load_names () end + if not name_index or type (name_index) ~= "table" then if not fonts_reloaded then return reload_db("no database", find_closest, name) end return false end + local by_distance = { } --- (int, string list) dict local distances = { } --- int list local cached = { } --- (string, int) dict @@ -2473,7 +2472,7 @@ local generate_filedata = function (mappings) local nmappings = #mappings - local filenames = { + local files = { bare = { ["local"] = { }, system = { }, --- mapped to mapping format -> index in full @@ -2487,9 +2486,9 @@ local generate_filedata = function (mappings) full = { }, --- non-texmf } - local base = filenames.base - local bare = filenames.bare - local full = filenames.full + local base = files.base + local bare = files.bare + local full = files.full local conflicts = { basenames = 0, @@ -2586,7 +2585,7 @@ local generate_filedata = function (mappings) -- addmap(sys) -- end - return filenames + return files end local match_synonyms = function (pattern) @@ -3025,7 +3024,7 @@ update_names = function (currentnames, force, dry_run) report ("info", 2, "db", "Skipping database update") --- skip all db updates - return currentnames or names.data + return currentnames or name_index end local starttime = osgettimeofday () @@ -3101,7 +3100,7 @@ update_names = function (currentnames, force, dry_run) report ("info", 3, "db", "Rebuilt in %0.f ms.", 1000 * (osgettimeofday () - starttime)) - names.data = targetnames + name_index = targetnames if dry_run ~= true then @@ -3151,7 +3150,7 @@ end --- dbobj? -> bool save_names = function (currentnames) if not currentnames then - currentnames = names.data + currentnames = name_index end local path = names.path.index local luaname, lucname = path.lua, path.luc @@ -3354,6 +3353,7 @@ names.set_font_filter = set_font_filter names.flush_lookup_cache = flush_lookup_cache names.save_lookups = save_lookups names.load = load_names +names.data = function () return name_index end names.save = save_names names.update = update_names names.crude_file_lookup = crude_file_lookup @@ -3373,8 +3373,8 @@ if luaotfloadconfig.resolver == "cached" then names.resolve = resolve_cached names.resolvespec = resolve_cached else - names.resolve = resolve - names.resolvespec = resolve + names.resolvespec = resolve_name + names.resolve_name = resolve_name end names.find_closest = find_closest -- cgit v1.2.3 From 3da77f501c148c757ca9cb570666ea409aff1ff3 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 00:27:31 +0100 Subject: [db] add first draft font family query --- luaotfload-database.lua | 218 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 206 insertions(+), 12 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index cba2f26..390c3ea 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -648,16 +648,21 @@ Existence of the resolved file name is verified differently depending on whether the index entry has a texmf flag set. --doc]]-- -local get_font_file = function (fullnames, entry) - local basename = entry.basename - if entry.location == "texmf" then +local get_font_file = function (index) + local entry = name_index.mappings [index] + if not entry then + return false + end + local filedata = entry.file + local basename = filedata.base + if filedata.location == "texmf" then if kpselookup(basename) then - return true, basename, entry.subfont + return true, basename, filedata.subfont end - else - local fullname = fullnames[entry.index] - if lfsisfile(fullname) then - return true, basename, entry.subfont + else --- system, local + local fullname = name_index.files.full [index] + if lfsisfile (fullname) then + return true, basename, filedata.subfont end end return false @@ -1096,17 +1101,206 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C end --- resolve() ]===] +local choose_closest = function (distances) + local closest = 2^51 + local match + for i = 1, #distances do + local d, index = unpack (distances [i]) + if d < closest then + closest = d + match = index + end + end + return match +end + +--[[doc-- + + choose_size -- Pick a font face of appropriate size from the list + of family members with matching style. There are three categories: + + 1. exact matches: if there is a face whose design size equals + the asked size, it is returned immediately and no further + candidates are inspected. + + 2. range matches: of all faces in whose design range the + requested size falls the one whose center the requested + size is closest to is returned. + + 3. out-of-range matches: of all other faces (i. e. whose range + is above or below the asked size) the one is chosen whose + boundary (upper or lower) is closest to the requested size. + + 4. default matches: if no design size or a design size of zero + is requested, the face with the default size is returned. + +--doc]]-- + +--- int * int * int * int list -> int -> int +local choose_size = function (sizes, askedsize) + local mappings = name_index.mappings + local match = sizes.default + local exact + local inrange = { } --- distance * index list + local norange = { } --- distance * index list + local fontname, subfont + if askedsize ~= 0 then + --- firstly, look for an exactly matching design size or + --- matching range + for i = 1, #sizes do + local dsnsize, high, low, index = unpack (sizes [i]) + if dsnsize == askedsize then + --- exact match, this is what we were looking for + exact = index + goto skip + elseif askedsize < low then + --- below range, add to the norange table + local d = low - askedsize + norange [#norange + 1] = { d, index } + elseif askedsize > high then + --- beyond range, add to the norange table + local d = askedsize - high + norange [#norange + 1] = { d, index } + else + --- range match + local d = ((low + high) / 2) - askedsize + if d < 0 then + d = -d + end + inrange [#inrange + 1] = { d, index } + end + end + end +::skip:: + if exact then + match = exact + elseif #inrange > 0 then + match = choose_closest (inrange) + elseif #norange > 0 then + match = choose_closest (norange) + end + return match +end + +local format_precedence = { + "otf", "ttc", "ttf", + "dfont", "afm", "pfb", + "pfa", +} + +local location_precedence = { + "local", "system", "texmf", +} + +--[[doc-- + + resolve_familyname -- Query the families table for an entry + matching the specification. + The parameters “name” and “style” are pre-sanitized. + +--doc]]-- +--- spec -> string -> string -> int -> string * int +local resolve_familyname = function (specification, name, style, askedsize) + local families = name_index.families + local mappings = name_index.mappings + local candidates = nil + --- arrow code alert + for i = 1, #location_precedence do + local location = location_precedence [i] + local locgroup = families [location] + for j = 1, #format_precedence do + local format = format_precedence [j] + local fmtgroup = locgroup [format] + if fmtgroup then + local familygroup = fmtgroup [name] + if familygroup then + local stylegroup = familygroup [style] + if stylegroup then --- suitable match + candidates = stylegroup + goto done + end + end + end + end + end + if true then + return nil, nil + end +::done:: + index = choose_size (candidates, askedsize) + local success, resolved, subfont = get_font_file (index) + if not success then + return nil, nil + end + report ("info", 2, "db", "Match found: %s(%d)", + resolved, subfont or 0) + return resolved, subfont +end + +local resolve_fontname = function (specification, name, style, askedsize) + local resolved, subfont + --[[ TODO ]] + return resolved, subfont +end + +--[[doc-- + + resolve_name -- Perform a name: lookup. This first queries the + font families table and, if there is no match for the spec, the + font names table. + The return value is a pair consisting of the file name and the + subfont index if appropriate.. + +--doc]]-- + +--- table -> string * (int | bool) resolve_name = function (specification) + local resolved, subfont if not name_index then name_index = load_names () end + local name = sanitize_fontname (specification.name) + local style = sanitize_fontname (specification.style) or "r" + local askedsize = specification.optsize + + if askedsize then + askedsize = tonumber (askedsize) + else + askedsize = specification.size + if askedsize and askedsize >= 0 then + askedsize = askedsize / 65536 + else + askedsize = 0 + end + end + + resolved, subfont = resolve_familyname (specification, name, style, askedsize) + if not resolved then + resolved, subfont = resolve_fontname (specification, name, style, askedsize) + end + if not resolved then + resolved = specification.name, false + end + return resolved, subfont end resolve_fullpath = function (fontname, ext) --- getfilename() if not name_index then name_index = load_names () end local files = name_index.files - local idx = files.base[fontname] - or files.bare[fontname] - if idx then - return files.full[idx] + local basedata = files.base + local baredata = files.bare + for i = 1, #location_precedence do + local location = location_precedence [i] + local basenames = basedata [location] + local barenames = baredata [location] [ext] + local idx + if basenames ~= nil then + idx = basenames [fontname] + end + if not idx and barenames ~= nil then + idx = barenames [fontname] + end + if idx then + return files.full [idx] + end end return "" end -- cgit v1.2.3 From 847b5a1a519b3945a1734f5859a80d459e865457 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 15:07:11 +0100 Subject: [db,tool] adapt --find to new db model --- luaotfload-database.lua | 72 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 22 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 390c3ea..1a79c3e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -140,6 +140,16 @@ local noncomma = 1-comma local splitcomma = Ct((C(noncomma^1) + comma)^1) patterns.splitcomma = splitcomma +local format_precedence = { + "otf", "ttc", "ttf", + "dfont", "afm", "pfb", + "pfa", +} + +local location_precedence = { + "local", "system", "texmf", +} + --[[doc-- We use the functions in the cache.* namespace that come with the fontloader (see luat-basics-gen). it’s safe to use for the most part @@ -614,23 +624,51 @@ crude_file_lookup_verbose = function (filename) return filename, nil, false end +local lookup_filename = function (filename) + if not name_index then name_index = load_names () end + local files = name_index.files + local basedata = files.base + local baredata = files.bare + for i = 1, #location_precedence do + local location = location_precedence [i] + local basenames = basedata [location] + local barenames = baredata [location] + local idx + if basenames ~= nil then + idx = basenames [filename] + if idx then + goto done + end + end + if barenames ~= nil then + for j = 1, #format_precedence do + local format = format_precedence [j] + local filemap = barenames [format] + if filemap then + idx = barenames [format] [filename] + if idx then + break + end + end + end + end +::done:: + if idx then + return files.full [idx] + end + end +end + --- string -> (string * string * bool) crude_file_lookup = function (filename) - if not name_index then name_index = load_names() end - local mappings = name_index.mappings - local files = name_index.files - - local found + local found = lookup_filename (filename) - found = files.base[filename] - or files.bare[filename] + if not found then + found = dummy_findfile(filename) + end if found then - found = files.full[found] - if found == nil then - found = dummy_findfile(filename) - end - return found or filename, nil, true + return found, nil, true end for i=1, #type1_formats do @@ -1182,16 +1220,6 @@ local choose_size = function (sizes, askedsize) return match end -local format_precedence = { - "otf", "ttc", "ttf", - "dfont", "afm", "pfb", - "pfa", -} - -local location_precedence = { - "local", "system", "texmf", -} - --[[doc-- resolve_familyname -- Query the families table for an entry -- cgit v1.2.3 From 58acd9e4271e17cd9d8073d4de69f6f68048adb4 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 17:07:38 +0100 Subject: [db] add font name resolver --- luaotfload-database.lua | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1a79c3e..5c60046 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1265,10 +1265,22 @@ local resolve_familyname = function (specification, name, style, askedsize) return resolved, subfont end -local resolve_fontname = function (specification, name, style, askedsize) - local resolved, subfont - --[[ TODO ]] - return resolved, subfont +local resolve_fontname = function (specification, name) + local mappings = name_index.mappings + for i = 1, #mappings do + local face = mappings [i] + local sanitized = face.names.sanitized + local info = sanitized.info + local english = sanitized.english + if info.fontname == name + or info.fullname == name + or english.psname == name + or english.fullname == name + then + return face.file.base, face.file.subfont + end + end + return nil, nil end --[[doc-- @@ -1302,7 +1314,7 @@ resolve_name = function (specification) resolved, subfont = resolve_familyname (specification, name, style, askedsize) if not resolved then - resolved, subfont = resolve_fontname (specification, name, style, askedsize) + resolved, subfont = resolve_fontname (specification, name) end if not resolved then resolved = specification.name, false -- cgit v1.2.3 From bbb62449eb4d64284ec6f9c70663b43cb4945939 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 17:15:26 +0100 Subject: [db] add further synonyms for regular --- luaotfload-database.lua | 128 ++++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 64 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 5c60046..aaba68d 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -521,57 +521,65 @@ load_lookups = function ( ) return data end -local style_synonyms = { set = { } } -do - local combine = function (ta, tb) - local result = { } - for i=1, #ta do - for j=1, #tb do - result[#result+1] = ta[i] .. tb[j] - end - end - return result - end - - --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html - --- tl;dr: font style synonyms are unreliable. - --- - --- Context matches font names against lists of known identifiers - --- for weight, style, width, and variant, so that including - --- the family name there are five dimensions for choosing a - --- match. The sad thing is, while this is a decent heuristic it - --- makes no sense to imitate it in luaotfload because the user - --- interface must fit into the much more limited Xetex scheme that - --- distinguishes between merely four style categories (variants): - --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, - --- some of the styles are lumped together although they can differ - --- significantly (like “medium” and “bold”). - - --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, - --- “normal”, and “roman” as synonyms for “regular”. - local list = { - regular = { "normal", "roman", - "plain", "book", - "light", "extralight", - "ultralight", }, - bold = { "demi", "demibold", - "semibold", "boldregular", - "medium", "mediumbold", - "ultrabold", "extrabold", - "heavy", "black", - "bold", }, - italic = { "regularitalic", "normalitalic", - "oblique", "slanted", - "italic", }, - } - - list.bolditalic = combine(list.bold, list.italic) - style_synonyms.list = list - - for category, synonyms in next, style_synonyms.list do - style_synonyms.set[category] = tabletohash(synonyms, true) - end -end +--local style_synonyms = { set = { } } +--do +-- local combine = function (ta, tb) +-- local result = { } +-- for i=1, #ta do +-- for j=1, #tb do +-- result[#result+1] = ta[i] .. tb[j] +-- end +-- end +-- return result +-- end +-- +-- --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html +-- --- tl;dr: font style synonyms are unreliable. +-- --- +-- --- Context matches font names against lists of known identifiers +-- --- for weight, style, width, and variant, so that including +-- --- the family name there are five dimensions for choosing a +-- --- match. The sad thing is, while this is a decent heuristic it +-- --- makes no sense to imitate it in luaotfload because the user +-- --- interface must fit into the much more limited Xetex scheme that +-- --- distinguishes between merely four style categories (variants): +-- --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, +-- --- some of the styles are lumped together although they can differ +-- --- significantly (like “medium” and “bold”). +-- +-- --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, +-- --- “normal”, and “roman” as synonyms for “regular”. +-- local list = { +-- regular = { "normal", "roman", +-- "plain", "book", +-- "light", "extralight", +-- "ultralight", }, +-- bold = { "demi", "demibold", +-- "semibold", "boldregular", +-- "medium", "mediumbold", +-- "ultrabold", "extrabold", +-- "heavy", "black", +-- "bold", }, +-- italic = { "regularitalic", "normalitalic", +-- "oblique", "slanted", +-- "italic", }, +-- } +-- +-- list.bolditalic = combine(list.bold, list.italic) +-- style_synonyms.list = list +-- +-- for category, synonyms in next, style_synonyms.list do +-- style_synonyms.set[category] = tabletohash(synonyms, true) +-- end +--end + +local regular_synonym = { + book = true, + normal = true, + plain = true, + regular = true, + roman = true, +} local type1_formats = { "tfm", "ofm", } @@ -2908,19 +2916,11 @@ do subfamily, splitstyle) - if fontstyle_name == "regular" or fontstyle_name == "book" then - return "r" - end - - if prefmodifiers == "regular" or prefmodifiers == "book" then - return "r" - end - - if subfamily == "regular" or subfamily == "book" then - return "r" - end - - if splitstyle == "regular" or splitstyle == "book" then + if regular_synonym [fontstyle_name] + or regular_synonym [prefmodifiers] + or regular_synonym [subfamily] + or regular_synonym [splitstyle] + then return "r" end -- cgit v1.2.3 From 36129aa2413bf76f7d9663e8de505e00eee5065f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 18:17:58 +0100 Subject: [db] adapt fuzzy matcher --- luaotfload-database.lua | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index aaba68d..c3256f0 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1421,12 +1421,13 @@ find_closest = function (name, limit) local by_distance = { } --- (int, string list) dict local distances = { } --- int list local cached = { } --- (string, int) dict - local mappings = data.mappings + local mappings = name_index.mappings local n_fonts = #mappings for n = 1, n_fonts do local current = mappings[n] - local cnames = current.sanitized + local names = current.names + local sanitized = names.sanitized --[[ This is simplistic but surpisingly fast. Matching is performed against the “fullname” field @@ -1436,10 +1437,13 @@ find_closest = function (name, limit) font name categories as well as whatever agrep does. --]] - if cnames then - local fullname, sfullname = current.fullname, cnames.fullname + if names then + local fullname = names.fullname + local sfullname = sanitized.info.fullname + or sanitized.english.fullname + or sanitized.metadata.fullname + local dist = cached[sfullname]--- maybe already calculated - local dist = cached[sfullname]--- maybe already calculated if not dist then dist = iterative_levenshtein(name, sfullname) cached[sfullname] = dist @@ -1589,10 +1593,6 @@ local organize_namedata = function (metadata, }, } --- print (fontnames.english.fullname, --- fontnames.metadata.fullname, --- fontnames.info.fullname) - if metadata.fontstyle_name then --- not present in all fonts, often differs from the preferred --- subfamily as well as subfamily fields, e.g. with LMSans10-BoldOblique: -- cgit v1.2.3 From e6613fb70ec97494b72970f9a53c10c69759dcd9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 18:38:04 +0100 Subject: [db] thow away obsolete name resolver Killing off one of the last remainders from the pre-TL2013 luaotfload. Good riddance ;-) --- luaotfload-database.lua | 310 ++++++------------------------------------------ 1 file changed, 38 insertions(+), 272 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index c3256f0..fec4efa 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -875,278 +875,6 @@ local add_to_match = function (found, size, face) return found, continue end ---[[doc-- - -Luatex-fonts, the font-loader package luaotfload imports, comes with -basic file location facilities (see luatex-fonts-syn.lua). -However, not only does the builtin functionality rely on Context’s font -name database, it is also too limited to be of more than basic use. -For this reason, luaotfload supplies its own resolvers that accesses -the font database created by the luaotfload-tool script. - ---doc]]-- - - ---- ---- the request specification has the fields: ---- ---- · features: table ---- · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig } ---- · ??? ---- · forced: string ---- · lookup: "name" ---- · method: string ---- · name: string ---- · resolved: string ---- · size: int ---- · specification: string (== ":" ) ---- · sub: string ---- ---- The “size” field deserves special attention: if its value is ---- negative, then it actually specifies a scalefactor of the ---- design size of the requested font. This happens e.g. if a font is ---- requested without an explicit “at size”. If the font is part of a ---- larger collection with different design sizes, this complicates ---- matters a bit: Normally, the resolver prefers fonts that have a ---- design size as close as possible to the requested size. If no ---- size specified, then the design size is implied. But which design ---- size should that be? Xetex appears to pick the “normal” (unmarked) ---- size: with Adobe fonts this would be the one that is neither ---- “caption” nor “subhead” nor “display” &c ... For fonts by Adobe this ---- seems to be the one that does not receive a “prefmodifiers” field. ---- (IOW Adobe uses the “prefmodifiers” field to encode the design size ---- in more or less human readable format.) However, this is not true ---- of LM and EB Garamond. As this matters only where there are ---- multiple design sizes to a given font/style combination, we put a ---- workaround in place that chooses that unmarked version. - ---- ---- the first return value of “resolve” is the file name of the ---- requested font (string) ---- the second is of type bool or string and indicates the subfont of a ---- ttc ---- ---- 'a -> 'a -> table -> (string * string | bool * bool) ---- - ---[===[ -resolve = function (_, _, specification) -- the 1st two parameters are used by ConTeXt - if not fonts_loaded then names.data = load_names() end - local data = names.data - - local name = sanitize_fontname (specification.name) - local style = sanitize_fontname (specification.style) or "regular" - - local askedsize - - if specification.optsize then - askedsize = tonumber(specification.optsize) - else - local specsize = specification.size - if specsize and specsize >= 0 then - askedsize = specsize / 65536 - end - end - - if type(data) ~= "table" then - --- this catches a case where load_names() doesn’t - --- return a database object, which can happen only - --- in case there is valid Lua code in the database, - --- but it’s not a table, e.g. it contains an integer. - if not fonts_reloaded then - return reload_db("invalid database; not a table", - resolve, nil, nil, specification) - end - --- unsucessfully reloaded; bail - return specification.name, false, false - end - - if not data.mappings then - if not fonts_reloaded then - return reload_db("invalid database; missing font mapping", - resolve, nil, nil, specification) - end - return specification.name, false, false - end - - local synonym_set = style_synonyms.set - local stylesynonyms = synonym_set[style] - local regularsynonyms = synonym_set.regular - - local exact = { } --> collect exact style matches - local synonymous = { } --> collect matching style synonyms - local fallback --> e.g. non-matching style (fontspec is anal about this) - local candidates = { } --> secondary results, incomplete matches - - for n, face in next, data.mappings do - local family, metafamily - local prefmodifiers, fontstyle_name, subfamily - local psname, fullname, fontname, pfullname - - local facenames = face.sanitized - if facenames then - family = facenames.family - subfamily = facenames.subfamily - fontstyle_name = facenames.fontstyle_name - prefmodifiers = facenames.prefmodifiers or fontstyle_name or subfamily - fullname = facenames.fullname - psname = facenames.psname - fontname = facenames.fontname - pfullname = facenames.pfullname - metafamily = facenames.metafamily - end - fontname = fontname or sanitize_fontname (face.fontname) - pfullname = pfullname or sanitize_fontname (face.fullname) - - if name == family - or name == metafamily - then - if style == prefmodifiers - or style == fontstyle_name - then - local continue - exact, continue = add_to_match(exact, askedsize, face) - if continue == false then break end - elseif style == subfamily then - exact = add_to_match(exact, askedsize, face) - elseif stylesynonyms and stylesynonyms[prefmodifiers] - or regularsynonyms[prefmodifiers] - then - --- treat synonyms for prefmodifiers as first-class - --- (needed to prioritize DejaVu Book over Condensed) - exact = add_to_match(exact, askedsize, face) - elseif name == fullname - or name == pfullname - or name == fontname - or name == psname - then - synonymous = add_to_match(synonymous, askedsize, face) - elseif stylesynonyms and stylesynonyms[subfamily] - or regularsynonyms[subfamily] - then - synonymous = add_to_match(synonymous, askedsize, face) - elseif prefmodifiers == "regular" - or subfamily == "regular" then - fallback = face - else --- mark as last straw but continue - candidates[#candidates+1] = face - end - else - if name == fullname - or name == pfullname - or name == fontname - or name == psname then - local continue - exact, continue = add_to_match(exact, askedsize, face) - if continue == false then break end - end - end - end - - local found - if next(exact) then - found = exact - else - found = synonymous - end - - --- this is a monster - if #found == 1 then - --- “found” is really synonymous with “registered in the db”. - local entry = found[1] - local success, filename, subfont - = get_font_file(data.files.full, entry) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - - elseif #found > 1 then - -- we found matching font(s) but not in the requested optical - -- sizes, so we loop through the matches to find the one with - -- least difference from the requested size. - local match - - if askedsize then --- choose by design size - local closest - local least = math.huge -- initial value is infinity - - for i, face in next, found do - local dsnsize = face.size and face.size [1] or 0 - local difference = mathabs (dsnsize - askedsize) - if difference < least then - closest = face - least = difference - end - end - - match = closest - else --- choose “unmarked” match, for Adobe fonts this - --- is the one without a “prefmodifiers” field. - match = found [1] --- fallback - for i, face in next, found do - if not face.sanitized.prefmodifiers then - match = face - break - end - end - end - - local success, filename, subfont - = get_font_file(data.files.full, match) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - - elseif fallback then - local success, filename, subfont - = get_font_file(data.files.full, fallback) - if success == true then - report("log", 0, "resolve", - "No exact match for request %s; using fallback", - specification.specification - ) - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - elseif next(candidates) then - --- pick the first candidate encountered - local entry = candidates[1] - local success, filename, subfont - = get_font_file(data.files.full, entry) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - end - - --- no font found so far - if not fonts_reloaded then - --- last straw: try reloading the database - return reload_db( - "unresolved font name: '" .. name .. "'", - resolve, nil, nil, specification - ) - end - - --- else, default to requested name - return specification.name, false, false -end --- resolve() -]===] - local choose_closest = function (distances) local closest = 2^51 local match @@ -1299,6 +1027,44 @@ end The return value is a pair consisting of the file name and the subfont index if appropriate.. + the request specification has the fields: + + · features: table + · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig } + · ??? + · forced: string + · lookup: "name" + · method: string + · name: string + · resolved: string + · size: int + · specification: string (== ":" ) + · sub: string + + The “size” field deserves special attention: if its value is + negative, then it actually specifies a scalefactor of the + design size of the requested font. This happens e.g. if a font is + requested without an explicit “at size”. If the font is part of a + larger collection with different design sizes, this complicates + matters a bit: Normally, the resolver prefers fonts that have a + design size as close as possible to the requested size. If no + size specified, then the design size is implied. But which design + size should that be? Xetex appears to pick the “normal” (unmarked) + size: with Adobe fonts this would be the one that is neither + “caption” nor “subhead” nor “display” &c ... For fonts by Adobe this + seems to be the one that does not receive a “prefmodifiers” field. + (IOW Adobe uses the “prefmodifiers” field to encode the design size + in more or less human readable format.) However, this is not true + of LM and EB Garamond. As this matters only where there are + multiple design sizes to a given font/style combination, we put a + workaround in place that chooses that unmarked version. + + The first return value of “resolve_name” is the file name of the + requested font (string). It can be passed to the fullname resolver + get_font_file(). + The second value is either “false” or an integer indicating the + subfont index in a TTC. + --doc]]-- --- table -> string * (int | bool) -- cgit v1.2.3 From de6ef7d6566f53b7554bf3861bb10712417b83fd Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 19:54:25 +0100 Subject: [db] strip extraneous information from mappings table --- luaotfload-database.lua | 171 ++++++++++++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 64 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index fec4efa..4632626 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -99,7 +99,8 @@ local name_index = nil -- upvalue for names.data local luaotfloadconfig = config.luaotfload --- always present luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" -luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" +luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" +luaotfloadconfig.strip = luaotfloadconfig.strip == true if luaotfloadconfig.update_live ~= false then --- this option allows for disabling updates @@ -699,16 +700,15 @@ local get_font_file = function (index) if not entry then return false end - local filedata = entry.file - local basename = filedata.base - if filedata.location == "texmf" then + local basename = entry.basename + if entry.location == "texmf" then if kpselookup(basename) then - return true, basename, filedata.subfont + return true, basename, entry.subfont end else --- system, local local fullname = name_index.files.full [index] if lfsisfile (fullname) then - return true, basename, filedata.subfont + return true, basename, entry.subfont end end return false @@ -1004,16 +1004,12 @@ end local resolve_fontname = function (specification, name) local mappings = name_index.mappings for i = 1, #mappings do - local face = mappings [i] - local sanitized = face.names.sanitized - local info = sanitized.info - local english = sanitized.english - if info.fontname == name - or info.fullname == name - or english.psname == name - or english.fullname == name + local face = mappings [i] + if face.fontname == name + or face.fullname == name + or face.psname == name then - return face.file.base, face.file.subfont + return face.basename, face.subfont end end return nil, nil @@ -1192,8 +1188,6 @@ find_closest = function (name, limit) for n = 1, n_fonts do local current = mappings[n] - local names = current.names - local sanitized = names.sanitized --[[ This is simplistic but surpisingly fast. Matching is performed against the “fullname” field @@ -1203,26 +1197,22 @@ find_closest = function (name, limit) font name categories as well as whatever agrep does. --]] - if names then - local fullname = names.fullname - local sfullname = sanitized.info.fullname - or sanitized.english.fullname - or sanitized.metadata.fullname - local dist = cached[sfullname]--- maybe already calculated - - if not dist then - dist = iterative_levenshtein(name, sfullname) - cached[sfullname] = dist - end - local namelst = by_distance[dist] - if not namelst then --- first entry - namelst = { fullname } - distances[#distances+1] = dist - else --- append - namelst[#namelst+1] = fullname - end - by_distance[dist] = namelst + local fullname = current.plainname + local sfullname = current.fullname + local dist = cached[sfullname]--- maybe already calculated + + if not dist then + dist = iterative_levenshtein(name, sfullname) + cached[sfullname] = dist end + local namelst = by_distance[dist] + if not namelst then --- first entry + namelst = { fullname } + distances[#distances+1] = dist + else --- append + namelst[#namelst+1] = fullname + end + by_distance[dist] = namelst end --- print the matches according to their distance @@ -2507,14 +2497,29 @@ local generate_filedata = function (mappings) local entry = mappings [index] local filedata = entry.file + local format + local location + local fullpath + local basename + local barename + local subfont + + if filedata then --- new entry + format = entry.format --- otf, afm, ... + location = filedata.location --- texmf, system, ... + fullpath = filedata.full + basename = filedata.base + barename = filenameonly (fullpath) + subfont = filedata.subfont + else + format = entry.format --- otf, afm, ... + location = entry.location --- texmf, system, ... + fullpath = entry.fullpath + basename = entry.basename + barename = filenameonly (fullpath) + subfont = entry.subfont + end - local format = entry.format --- otf, afm, ... - local location = filedata.location --- texmf, system, ... - - local fullname = filedata.full - local basename = filedata.base - local barename = filenameonly (fullname) - local subfont = filedata.subfont entry.index = index @@ -2578,9 +2583,9 @@ local generate_filedata = function (mappings) bare [location] [format] = inbare end - --- 3) add to fullname map + --- 3) add to fullpath map - full [index] = fullname + full [index] = fullpath end --- TODO adapt to new mechanism! @@ -2694,6 +2699,46 @@ do end end +local pull_values = function (entry) + local file = entry.file + local names = entry.names + local style = entry.style + local sanitized = names.sanitized + local english = sanitized.english + local info = sanitized.info + + --- pull file info ... + entry.basename = file.base + entry.fullpath = file.full + entry.location = file.location + entry.subfont = file.subfont + + --- pull name info ... + entry.psname = english.psname + entry.fontname = info.fontname + entry.fullname = english.fullname or info.fullname + entry.familyname = english.preffamily + or english.family + or info.familyname + entry.fontstyle_name = sanitized.fontstyle_name + entry.plainname = names.fullname + entry.prefmodifiers = english.prefmodifiers + entry.subfamily = english.subfamily + + --- pull style info ... + entry.italicangle = style.italicangle + entry.size = style.size + entry.splitstyle = style.split + entry.weight = style.weight + + if luaotfloadconfig.strip == true then + --if false then + entry.file = nil + entry.names = nil + entry.style = nil + end +end + local collect_families = function (mappings) report ("info", 2, "db", "Analyzing families, sizes, and styles.") @@ -2706,15 +2751,14 @@ local collect_families = function (mappings) for i = 1, #mappings do - local entry = mappings [i] - local file = entry.file - local names = entry.names - local style = entry.style + local entry = mappings [i] + + if entry.file then + pull_values (entry) + end - local location = file.location + local location = entry.location local format = entry.format - local english = names.sanitized.english - local info = names.sanitized.info local subtable = families [location] [format] if not subtable then @@ -2722,19 +2766,17 @@ local collect_families = function (mappings) families [location] [format] = subtable end - local familyname = english.preffamily - or english.family - or info.familyname - local fullname = english.fullname or info.fullname - local fontname = info.fontname + --local fullname = english.fullname or info.fullname + --local fontname = info.fontname - local fontstyle_name = names.sanitized.fontstyle_name - local prefmodifiers = english.prefmodifiers - local subfamily = english.subfamily + local familyname = entry.familyname + local fontstyle_name = entry.fontstyle_name + local prefmodifiers = entry.prefmodifiers + local subfamily = entry.subfamily - local weight = style.weight - local italicangle = style.italicangle - local splitstyle = style.split + local weight = entry.weight + local italicangle = entry.italicangle + local splitstyle = entry.splitstyle local modifier = pick_style (fontstyle_name, prefmodifiers, @@ -2786,7 +2828,7 @@ local collect_families = function (mappings) styletable.default = entry.index end - local size = style.size --- dsnsize * high * low + local size = entry.size --- dsnsize * high * low if size then styletable [#styletable + 1] = { size [1], @@ -2800,6 +2842,7 @@ local collect_families = function (mappings) end end + collectgarbage "collect" return families end -- cgit v1.2.3 From b697b1fa3322d1ce40cc6b9149363b15ebe6f9f7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 20:59:46 +0100 Subject: [db] fix damage suffered during last merge --- luaotfload-database.lua | 6 ------ 1 file changed, 6 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1deca07..ad954a2 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2526,12 +2526,6 @@ local generate_filedata = function (mappings) barename = filenameonly (fullpath) subfont = entry.subfont end - end - - local addmap = function (lst) - --- this will overwrite existing entries - for i=1, #lst do - local idx, base, bare, intexmf, full = unpack(lst[i]) entry.index = index -- cgit v1.2.3 From bf11a971192a6d8cb6aa4b23d0a1c48fd74a2538 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 22:15:58 +0100 Subject: [db] adapt type1 scanner --- luaotfload-database.lua | 117 +++++++++++++++++------------------------------- 1 file changed, 41 insertions(+), 76 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index ad954a2..e93a299 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1355,7 +1355,8 @@ local organize_namedata = function (metadata, -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size if metadata.fontstyle_name then --- not present in all fonts, often differs from the preferred - --- subfamily as well as subfamily fields, e.g. with LMSans10-BoldOblique: + --- subfamily as well as subfamily fields, e.g. with + --- LMSans10-BoldOblique: --- subfamily: “Bold Italic” --- prefmodifiers: “10 Bold Oblique” --- fontstyle_name: “Bold Oblique” @@ -1364,9 +1365,6 @@ local organize_namedata = function (metadata, fontnames.fontstyle_name = name.name end end - - --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", english_names.prefmodifiers) - --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", fontnames.fontstyle_name) end return { @@ -1391,14 +1389,13 @@ local split_fontname = function (fontname) end end -local organize_styledata = function (fontname, metadata, english_names, info) +local organize_styledata = function (fontname, + metadata, + english_names, + info) local pfminfo = metadata.pfminfo local names = metadata.names --- print (">", pfminfo.avgwidth, --- (metadata.italicangle == info.italicangle) and "T" or --- string.format ("%f / %f", metadata.italicangle, info.italicangle), --- pfminfo.width, pfminfo.weight, info.weight) return { -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size size = get_size_info (metadata), @@ -1474,8 +1471,8 @@ end --- string -> int -> bool -> string -> fontentry t1_fullinfo = function (filename, _subfont, location, basename, format) - local namedata = { } - local metadata = load_font_file (filename) + local sanitized + local metadata = load_font_file (filename) local fontname = metadata.fontname local fullname = metadata.fullname @@ -1483,68 +1480,35 @@ t1_fullinfo = function (filename, _subfont, location, basename, format) local italicangle = metadata.italicangle local weight = metadata.weight --- string identifier - --- we have to improvise and detect whether we deal with - --- italics since pfb fonts don’t come with a “subfamily” - --- field - local style - if italicangle == 0 then - style = false - else - style = "italic" - end - - local style_synonyms_set = style_synonyms.set - if weight then - weight = sanitize_fontname (weight) - local tmp = "" - if style_synonyms_set.bold[weight] then - tmp = "bold" - end - if style then - style = tmp .. style - else - if style_synonyms_set.regular[weight] then - style = "regular" - else - style = tmp - end - end - end - - if not style or style == "" then - style = "regular" - --- else italic - end - - namedata.sanitized = sanitize_fontnames ({ + sanitized = sanitize_fontnames ({ fontname = fontname, psname = fullname, pfullname = fullname, metafamily = family, - family = familyname, + familyname = familyname, subfamily = weight, prefmodifiers = style, }) - namedata.fontname = fontname - namedata.fullname = fullname - namedata.familyname = familyname - - namedata.slant = italicangle - namedata.units_per_em = 1000 --- ps fonts standard - namedata.version = metadata.version - namedata.weight = metadata.pfminfo.weight --- integer - namedata.width = metadata.pfminfo.width - - namedata.size = false - - namedata.filename = filename --> sys - namedata.basename = basename --> texmf - namedata.format = format - namedata.location = location or "system" - namedata.subfont = false - - return namedata + return { + basename = basename, + fullpath = filename, + subfont = false, + location = location or "system", + format = format, + fullname = sanitized.fullname, + fontname = sanitized.fontname, + familyname = sanitized.familyname, + plainname = fullname, + psname = sanitized.fontname, + version = metadata.version, + size = false, + splitstyle = split_fontname (fontname), + fontstyle_name = sanitized.subfamily, + weight = { metadata.pfminfo.weight, + sanitized.subfamily }, + italicangle = italicangle, + } end local loaders = { @@ -1698,7 +1662,6 @@ local read_font_names = function (fullname, local format = stringlower (filesuffix (basename)) local loader = loaders [format] --- ot_fullinfo, t1_fullinfo - local loader = loaders[format] --- ot_fullinfo, t1_fullinfo if not loader then report ("both", 0, "db", "Unknown format: %q, skipping.", format) @@ -2091,8 +2054,8 @@ local scan_dir = function (dirname, currentnames, targetnames, "Would have been loading %q", fullname) else report_status ("both", "db", "Loading font %q", fullname) - local new = load_font (fullname, fontnames, - newfontnames, texmf) + local new = read_font_names (fullname, currentnames, + targetnames, texmf) if new == true then n_new = n_new + 1 end @@ -2627,13 +2590,14 @@ do subfamily) if italicangle ~= 0 then return true - elseif fontstyle_name and lpegmatch (italic, fontstyle_name) then + elseif fontstyle_name ~= nil and lpegmatch (italic, fontstyle_name) then return true - elseif prefmodifiers and lpegmatch (italic, prefmodifiers) then + elseif prefmodifiers ~= nil and lpegmatch (italic, prefmodifiers) then return lpegmatch (italic, prefmodifiers) - else - return lpegmatch (italic, subfamily) + elseif subfamily ~= nil and lpegmatch (italic, subfamily) then + return true end + return false end determine_bold = function (fontstyle_name, @@ -2642,13 +2606,14 @@ do subfamily) if weight [2] == "bold" then return true - elseif fontstyle_name and lpegmatch (bold, fontstyle_name) then + elseif fontstyle_name ~= nil and lpegmatch (bold, fontstyle_name) then return true - elseif prefmodifiers and lpegmatch (bold, prefmodifiers) then + elseif prefmodifiers ~= nil and lpegmatch (bold, prefmodifiers) then + return true + elseif subfamily ~= nil and lpegmatch (bold, subfamily) then return true - else - return lpegmatch (bold, subfamily) end + return false end local splitfontname = lpeg.splitat "-" -- cgit v1.2.3 From e0a62d13f16573b7461269bb022516634b7fce3d Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 22:16:42 +0100 Subject: [db] throw away some cruft (old synonyms table) --- luaotfload-database.lua | 52 ------------------------------------------------- 1 file changed, 52 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e93a299..291abe5 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -525,58 +525,6 @@ load_lookups = function ( ) return data end ---local style_synonyms = { set = { } } ---do --- local combine = function (ta, tb) --- local result = { } --- for i=1, #ta do --- for j=1, #tb do --- result[#result+1] = ta[i] .. tb[j] --- end --- end --- return result --- end --- --- --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html --- --- tl;dr: font style synonyms are unreliable. --- --- --- --- Context matches font names against lists of known identifiers --- --- for weight, style, width, and variant, so that including --- --- the family name there are five dimensions for choosing a --- --- match. The sad thing is, while this is a decent heuristic it --- --- makes no sense to imitate it in luaotfload because the user --- --- interface must fit into the much more limited Xetex scheme that --- --- distinguishes between merely four style categories (variants): --- --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, --- --- some of the styles are lumped together although they can differ --- --- significantly (like “medium” and “bold”). --- --- --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, --- --- “normal”, and “roman” as synonyms for “regular”. --- local list = { --- regular = { "normal", "roman", --- "plain", "book", --- "light", "extralight", --- "ultralight", }, --- bold = { "demi", "demibold", --- "semibold", "boldregular", --- "medium", "mediumbold", --- "ultrabold", "extrabold", --- "heavy", "black", --- "bold", }, --- italic = { "regularitalic", "normalitalic", --- "oblique", "slanted", --- "italic", }, --- } --- --- list.bolditalic = combine(list.bold, list.italic) --- style_synonyms.list = list --- --- for category, synonyms in next, style_synonyms.list do --- style_synonyms.set[category] = tabletohash(synonyms, true) --- end ---end - local regular_synonym = { book = true, normal = true, -- cgit v1.2.3 From 947a07ebf509f7961673fdfdafb0cf1ea6faec4a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 22:22:37 +0100 Subject: [db,tool] adapt --prefer-texmf --- luaotfload-database.lua | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 291abe5..868541b 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -154,6 +154,10 @@ local location_precedence = { "local", "system", "texmf", } +local set_location_precedence = function (precedence) + location_precedence = precedence +end + --[[doc-- We use the functions in the cache.* namespace that come with the fontloader (see luat-basics-gen). it’s safe to use for the most part @@ -2505,16 +2509,6 @@ local generate_filedata = function (mappings) full [index] = fullpath end - --- TODO adapt to new mechanism! --- if luaotfloadconfig.prioritize == "texmf" then --- report("both", 2, "db", "Preferring texmf fonts") --- addmap(sys) --- addmap(texmf) --- else --- sys --- addmap(texmf) --- addmap(sys) --- end - return files end @@ -2530,7 +2524,7 @@ local check_regular do local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") - local bold = match_synonyms (P"bold" + P"demi", P"heavy", P"black", P"ultra") + local bold = match_synonyms (P"bold" + P"demi" + P"heavy" + P"black" + P"ultra") determine_italic = function (fontstyle_name, italicangle, @@ -3323,6 +3317,7 @@ names.crude_file_lookup_verbose = crude_file_lookup_verbose names.read_blacklist = read_blacklist names.sanitize_fontname = sanitize_fontname names.getfilename = resolve_fullpath +names.set_location_precedence = set_location_precedence --- font cache names.purge_cache = purge_cache -- cgit v1.2.3 From 06af053c14846af44b3306b28a8443e11a559cfe Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 23:24:50 +0100 Subject: [db] stop matching bold synonyms as bold --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 868541b..84f8474 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2524,7 +2524,7 @@ local check_regular do local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") - local bold = match_synonyms (P"bold" + P"demi" + P"heavy" + P"black" + P"ultra") + local bold = match_synonyms (P"bold") determine_italic = function (fontstyle_name, italicangle, -- cgit v1.2.3 From aa118e1f44ac27ed1d9046c45d8baa855fa2c107 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 02:04:47 +0100 Subject: [db] tweak font family matching Addresses https://github.com/lualatex/luaotfload/issues/118 but without crude hacks ;-) Now the test file processes perfectly: https://bitbucket.org/phg/lua-la-tex-tests/src/tip/ltx-fontspec-optical-size-3.tex --- luaotfload-database.lua | 208 +++++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 117 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 84f8474..e439e2a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -530,11 +530,17 @@ load_lookups = function ( ) end local regular_synonym = { - book = true, - normal = true, - plain = true, - regular = true, - roman = true, + book = "r", + normal = "r", + plain = "r", + regular = "r", + roman = "r", +} + +local italic_synonym = { + oblique = true, + slanted = true, + italic = true, } local type1_formats = { "tfm", "ofm", } @@ -1294,7 +1300,7 @@ local organize_namedata = function (metadata, metadata = { fullname = metadata.fullname, fontname = metadata.fontname, - metafamily = metadata.familyname, + familyname = metadata.familyname, }, info = { @@ -2512,56 +2518,16 @@ local generate_filedata = function (mappings) return files end -local match_synonyms = function (pattern) - local nopattern = 1 - pattern - return nopattern^0 * pattern * Cc (true) + Cc (false) -end - local determine_italic local determine_bold local pick_style local check_regular do - local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") - local bold = match_synonyms (P"bold") - - determine_italic = function (fontstyle_name, - italicangle, - prefmodifiers, - subfamily) - if italicangle ~= 0 then - return true - elseif fontstyle_name ~= nil and lpegmatch (italic, fontstyle_name) then - return true - elseif prefmodifiers ~= nil and lpegmatch (italic, prefmodifiers) then - return lpegmatch (italic, prefmodifiers) - elseif subfamily ~= nil and lpegmatch (italic, subfamily) then - return true - end - return false - end - - determine_bold = function (fontstyle_name, - weight, - prefmodifiers, - subfamily) - if weight [2] == "bold" then - return true - elseif fontstyle_name ~= nil and lpegmatch (bold, fontstyle_name) then - return true - elseif prefmodifiers ~= nil and lpegmatch (bold, prefmodifiers) then - return true - elseif subfamily ~= nil and lpegmatch (bold, subfamily) then - return true - end - return false - end - local splitfontname = lpeg.splitat "-" local choose_exact = function (field) - if field == "italic" or field == "oblique" then + if italic_synonym [field] then return "i" elseif field == "bold" then return "b" @@ -2577,17 +2543,18 @@ do subfamily, splitstyle) local style - if fontstyle_name ~= nil then - style = choose_exact (prefmodifiers) - end - if not style and prefmodifiers ~= nil then - style = choose_exact (prefmodifiers) + if fontstyle_name then + style = choose_exact (fontstyle_name) end if not style then - style = choose_exact (subfamily) + if prefmodifiers then + style = choose_exact (prefmodifiers) + elseif subfamily then + --style = choose_exact (subfamily) + end end - if not style and splitstyle ~= nil then - choose_exact (splitstyle) + if not style and splitstyle then + style = choose_exact (splitstyle) end return style end @@ -2600,12 +2567,14 @@ do subfamily, splitstyle) - if regular_synonym [fontstyle_name] - or regular_synonym [prefmodifiers] - or regular_synonym [subfamily] - or regular_synonym [splitstyle] - then - return "r" + if fontstyle_name then + return regular_synonym [fontstyle_name] + elseif prefmodifiers then + return regular_synonym [prefmodifiers] + elseif subfamily then + return regular_synonym [subfamily] + elseif splitstyle then + return regular_synonym [splitstyle] end return nil @@ -2619,6 +2588,7 @@ local pull_values = function (entry) local sanitized = names.sanitized local english = sanitized.english local info = sanitized.info + local metadata = sanitized.metadata --- pull file info ... entry.basename = file.base @@ -2630,12 +2600,15 @@ local pull_values = function (entry) entry.psname = english.psname entry.fontname = info.fontname entry.fullname = english.fullname or info.fullname - entry.familyname = english.preffamily - or english.family - or info.familyname + entry.prefmodifiers = english.prefmodifiers + local metafamily = metadata.familyname + local familyname = english.preffamily or english.family + entry.familyname = familyname + if familyname ~= metafamily then + entry.metafamily = metadata.familyname + end entry.fontstyle_name = sanitized.fontstyle_name entry.plainname = names.fullname - entry.prefmodifiers = english.prefmodifiers entry.subfamily = english.subfamily --- pull style info ... @@ -2645,13 +2618,43 @@ local pull_values = function (entry) entry.weight = style.weight if luaotfloadconfig.strip == true then - --if false then entry.file = nil entry.names = nil entry.style = nil end end +local add_family = function (name, subtable, modifier, entry) + local familytable = subtable [name] + if not familytable then + familytable = { } + subtable [name] = familytable + end + + --- the style table is treated as an unordered list + local styletable = familytable [modifier] + if not styletable then + styletable = { } + familytable [modifier] = styletable + end + + if not entry.prefmodifiers then --- default size for this style/family combo + styletable.default = entry.index + end + + local size = entry.size --- dsnsize * high * low + if size then + styletable [#styletable + 1] = { + size [1], + size [2], + size [3], + entry.index, + } + else + styletable.default = entry.index + end +end + local collect_families = function (mappings) report ("info", 2, "db", "Analyzing families, sizes, and styles.") @@ -2679,10 +2682,8 @@ local collect_families = function (mappings) families [location] [format] = subtable end - --local fullname = english.fullname or info.fullname - --local fontname = info.fontname - local familyname = entry.familyname + local metafamily = entry.metafamily local fontstyle_name = entry.fontstyle_name local prefmodifiers = entry.prefmodifiers local subfamily = entry.subfamily @@ -2696,24 +2697,24 @@ local collect_families = function (mappings) subfamily, splitstyle) - if not modifier then --- guess - local italic = determine_italic (fontstyle_name, - italicangle, - prefmodifiers, - subfamily) - local bold = determine_bold (fontstyle_name, - weight, - prefmodifiers, - subfamily) - if bold and italic then - modifier = "bi" - elseif bold then - modifier = "b" - elseif italic then - modifier = "i" - end - end - +-- if not modifier then --- guess +-- local italic = determine_italic (fontstyle_name, +-- italicangle, +-- prefmodifiers, +-- subfamily) +-- local bold = determine_bold (fontstyle_name, +-- weight, +-- prefmodifiers, +-- subfamily) +-- if bold and italic then +-- modifier = "bi" +-- elseif bold then +-- modifier = "b" +-- elseif italic then +-- modifier = "i" +-- end +-- end +-- if not modifier then --- regular, exact only modifier = check_regular (fontstyle_name, subfamily, @@ -2721,36 +2722,9 @@ local collect_families = function (mappings) end if modifier then - --- stub; here we will continue building a list of optical sizes - --- no size -> hash “normal” - --- other sizes -> indexed tuples { dsnsize, high, low, idx } - local familytable = subtable [familyname] - if not familytable then - familytable = { } - subtable [familyname] = familytable - end - - --- the style table is treated as an unordered list - local styletable = familytable [modifier] - if not styletable then - styletable = { } - familytable [modifier] = styletable - end - - if not prefmodifiers then --- default size for this style/family combo - styletable.default = entry.index - end - - local size = entry.size --- dsnsize * high * low - if size then - styletable [#styletable + 1] = { - size [1], - size [2], - size [3], - entry.index, - } - else - styletable.default = entry.index + add_family (familyname, subtable, modifier, entry) + if metafamily ~= nil then + add_family (metafamily, subtable, modifier, entry) end end end -- cgit v1.2.3 From 5c7df96b71cc281898067971ff96eb939f8d9d45 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 15:31:11 +0100 Subject: [db] adapt cached resolver --- luaotfload-database.lua | 119 +++++++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 51 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e439e2a..f103caa 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,8 +31,6 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table -local gzipload = gzip.load -local gzipsave = gzip.save local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -60,6 +58,7 @@ local tablesort = table.sort local texiowrite_nl = texio.write_nl local utf8gsub = unicode.utf8.gsub local utf8lower = unicode.utf8.lower +local zlibcompress = zlib.compress --- these come from Lualibs/Context local filebasename = file.basename @@ -94,9 +93,6 @@ fonts = fonts or { } fonts.names = fonts.names or { } fonts.definers = fonts.definers or { } -local names = fonts.names -local name_index = nil -- upvalue for names.data - local luaotfloadconfig = config.luaotfload --- always present luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" @@ -108,6 +104,9 @@ if luaotfloadconfig.update_live ~= false then luaotfloadconfig.update_live = true end +local names = fonts.names +local name_index = nil --> upvalue for names.data +local lookup_cache = nil --> for names.lookups names.version = 2.4 names.data = nil --- contains the loaded database names.lookups = nil --- contains the lookup cache @@ -186,11 +185,11 @@ if not runasscript then end globals.prefix = prefix - local lookups = names.path.lookups + local lookup_path = names.path.lookups local index = names.path.index local lookups_file = filejoin (prefix, globals.lookups_file) local index_file = filejoin (prefix, globals.index_file) - lookups.lua, lookups.luc = make_luanames (lookups_file) + lookup_path.lua, lookup_path.luc = make_luanames (lookups_file) index.lua, index.luc = make_luanames (index_file) else --- running as script, inject some dummies caches = { } @@ -410,6 +409,33 @@ local initialize_namedata = function (formats) --- returns dbobj } end +--[[doc-- + + Since Luaotfload does not depend on the lualibs anymore we + have to put our own small wrappers for the gzip library in + place. + +--doc]]-- + +local load_gzipped = function (filename) + local gh = gzip.open (filename,"rb") + if gh then + local data = gh:read "*all" + gh:close () + return data + end +end + +local save_gzipped = function (filename, data) + local gh = ioopen (filename, "wb") + if gh then + local bin = zlibcompress (data, 9, nil, 15 + 16) + gh:write (bin) + gh:close () + return #bin + end +end + --- When loading a lua file we try its binary complement first, which --- is assumed to be located at an identical path, carrying the suffix --- .luc. @@ -417,15 +443,14 @@ end --- string -> (string * table) local load_lua_file = function (path) local foundname = filereplacesuffix (path, "luc") + local code = nil - if false then local fh = ioopen (foundname, "rb") -- try bin first if fh then local chunk = fh:read"*all" fh:close() code = load (chunk, "b") end -end if not code then --- fall back to text file foundname = filereplacesuffix (path, "lua") @@ -439,7 +464,7 @@ end if not code then --- probe gzipped file foundname = filereplacesuffix (path, "lua.gz") - local chunk = gzipload (foundname) + local chunk = load_gzipped (foundname) if chunk then code = load (chunk, "t") end @@ -515,7 +540,7 @@ load_names = function (dry_run) return data end ---- unit -> dbobj +--- unit -> unit load_lookups = function ( ) local foundname, data = load_lua_file(names.path.lookups.lua) if data then @@ -526,7 +551,7 @@ load_lookups = function ( ) "No lookup cache, creating empty.") data = { } end - return data + lookup_cache = data end local regular_synonym = { @@ -677,28 +702,18 @@ end --[[doc-- We need to verify if the result of a cached lookup actually exists in -the texmf or filesystem. +the texmf or filesystem. Again, due to the schizoprenic nature of the +font managment we have to check both the system path and the texmf. --doc]]-- local verify_font_file = function (basename) - if not name_index then name_index = load_names() end - local files = name_index.files - local idx = files.base[basename] - if not idx then - return false - end - - --- firstly, check filesystem - local fullname = files.full[idx] - if fullname and lfsisfile(fullname) then + local path = resolve_fullpath (basename) + if path and lfsisfile(path) then return true end - - --- secondly, locate via kpathsea if kpsefind_file(basename) then return true end - return false end @@ -733,8 +748,8 @@ Idk what the “spec” resolver is for. optsize, size spec: name, sub resolved, sub, name, forced -* name: contains both the name resolver from luatex-fonts and resolve() - below +* name: contains both the name resolver from luatex-fonts and + resolve_name() below From my reading of font-def.lua, what a resolver does is basically rewrite the “name” field of the specification record @@ -763,20 +778,20 @@ local hash_request = function (specification) end --- 'a -> 'a -> table -> (string * int|boolean * boolean) -resolve_cached = function (_, _, specification) - if not names.lookups then names.lookups = load_lookups() end +resolve_cached = function (specification) + if not lookup_cache then load_lookups () end local request = hash_request(specification) report("both", 4, "cache", "Looking for %q in cache ...", request) - local found = names.lookups[request] + local found = lookup_cache [request] --- case 1) cache positive ---------------------------------------- if found then --- replay fields from cache hit report("info", 4, "cache", "Found!") local basename = found[1] --- check the presence of the file in case it’s been removed - local success = verify_font_file(basename) + local success = verify_font_file (basename) if success == true then return basename, found[2], true end @@ -787,22 +802,24 @@ resolve_cached = function (_, _, specification) --- case 2) cache negative ---------------------------------------- --- first we resolve normally ... - local filename, subfont, success = resolve(nil, nil, specification) - if not success then return filename, subfont, false end + local filename, subfont = resolve_name (specification) + if not filename then + return nil, nil + end --- ... then we add the fields to the cache ... ... local entry = { filename, subfont } report("both", 4, "cache", "New entry: %s", request) - names.lookups[request] = entry + lookup_cache [request] = entry --- obviously, the updated cache needs to be stored. --- TODO this should trigger a save only once the --- document is compiled (finish_pdffile callback?) report("both", 5, "cache", "Saving updated cache") - local success = save_lookups() + local success = save_lookups () if not success then --- sad, but not critical - report("both", 0, "cache", "Could not write to cache") + report("both", 0, "cache", "Error writing cache.") end - return filename, subfont, true + return filename, subfont end --- this used to be inlined; with the lookup cache we don’t @@ -1061,13 +1078,15 @@ resolve_fullpath = function (fontname, ext) --- getfilename() for i = 1, #location_precedence do local location = location_precedence [i] local basenames = basedata [location] - local barenames = baredata [location] [ext] local idx if basenames ~= nil then idx = basenames [fontname] end - if not idx and barenames ~= nil then - idx = barenames [fontname] + if ext then + local barenames = baredata [location] [ext] + if not idx and barenames ~= nil then + idx = barenames [fontname] + end end if idx then return files.full [idx] @@ -2383,10 +2402,9 @@ end --- unit -> (bool, lookup_cache) flush_lookup_cache = function () - if not names.lookups then names.lookups = load_lookups() end - names.lookups = { } + lookup_cache = { } collectgarbage "collect" - return true, names.lookups + return true, lookup_cache end @@ -3038,7 +3056,7 @@ update_names = function (currentnames, force, dry_run) local success, _lookups = flush_lookup_cache () if success then - local success = names.save_lookups () + local success = save_lookups () if success then logs.names_report ("info", 2, "cache", "Lookup cache emptied") @@ -3051,13 +3069,12 @@ end --- unit -> bool save_lookups = function ( ) - local lookups = names.lookups local path = names.path.lookups local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, lookups, true) + tabletofile (luaname, lookup_cache, true) osremove (lucname) - caches.compile (lookups, luaname, lucname) + caches.compile (lookup_cache, luaname, lucname) --- double check ... if lfsisfile (luaname) and lfsisfile (lucname) then report ("both", 3, "cache", "Lookup cache saved") @@ -3089,7 +3106,7 @@ save_names = function (currentnames) local gzname = luaname .. ".gz" if luaotfloadconfig.compress then local serialized = tableserialize (currentnames, true) - gzipsave (gzname, serialized) + save_gzipped (gzname, serialized) caches.compile (currentnames, "", lucname) else tabletofile (luaname, currentnames, true) @@ -3301,8 +3318,8 @@ names.show_cache = show_cache --- replace the resolver from luatex-fonts if luaotfloadconfig.resolver == "cached" then report("both", 2, "cache", "caching of name: lookups active") - names.resolve = resolve_cached - names.resolvespec = resolve_cached + names.resolvespec = resolve_cached + names.resolve_name = resolve_cached else names.resolvespec = resolve_name names.resolve_name = resolve_name -- cgit v1.2.3 From 4467a6f5150c49cd0992a624c294018cf02be07c Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 17:22:54 +0100 Subject: [db] remove dependency on lualibs for compression --- luaotfload-database.lua | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index f103caa..567149a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,6 +31,7 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table +local gzipopen = gzip.open local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -415,10 +416,13 @@ end have to put our own small wrappers for the gzip library in place. + load_gzipped -- Read and decompress and entire gzipped file. + Returns the uncompressed content as a string. + --doc]]-- local load_gzipped = function (filename) - local gh = gzip.open (filename,"rb") + local gh = gzipopen (filename,"rb") if gh then local data = gh:read "*all" gh:close () @@ -426,13 +430,21 @@ local load_gzipped = function (filename) end end +--[[doc-- + + save_gzipped -- Compress and write a string to file. The return + value is the number of bytes written. Zlib parameters are: best + compression and default strategy. + +--doc]]-- + local save_gzipped = function (filename, data) - local gh = ioopen (filename, "wb") + local gh = gzipopen (filename, "wb9") if gh then - local bin = zlibcompress (data, 9, nil, 15 + 16) - gh:write (bin) + gh:write (data) + local bytes = gh:seek () gh:close () - return #bin + return bytes end end -- cgit v1.2.3 From 480327fcaf3d5da8f2766e7cf9fa6e2fbb59ceb5 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 17:52:17 +0100 Subject: [db] fix test for regular style --- luaotfload-database.lua | 1 + 1 file changed, 1 insertion(+) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 567149a..9101f94 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2747,6 +2747,7 @@ local collect_families = function (mappings) -- if not modifier then --- regular, exact only modifier = check_regular (fontstyle_name, + prefmodifiers, subfamily, splitstyle) end -- cgit v1.2.3 From 049edfd2afc26472a0f41df1c02a73cc1fd6e812 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 18:03:38 +0100 Subject: [db] remove cruft --- luaotfload-database.lua | 20 -------------------- 1 file changed, 20 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e439e2a..0987508 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2518,8 +2518,6 @@ local generate_filedata = function (mappings) return files end -local determine_italic -local determine_bold local pick_style local check_regular @@ -2697,24 +2695,6 @@ local collect_families = function (mappings) subfamily, splitstyle) --- if not modifier then --- guess --- local italic = determine_italic (fontstyle_name, --- italicangle, --- prefmodifiers, --- subfamily) --- local bold = determine_bold (fontstyle_name, --- weight, --- prefmodifiers, --- subfamily) --- if bold and italic then --- modifier = "bi" --- elseif bold then --- modifier = "b" --- elseif italic then --- modifier = "i" --- end --- end --- if not modifier then --- regular, exact only modifier = check_regular (fontstyle_name, subfamily, -- cgit v1.2.3 From 19d57e135e09d9311d0e17c6f7c375a161b8c37f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 18:31:07 +0100 Subject: [db] do not add metafamilies to the family table --- luaotfload-database.lua | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'luaotfload-database.lua') diff --git a/luaotfload-database.lua b/luaotfload-database.lua index c5b156c..d8d4a6f 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2734,9 +2734,29 @@ local collect_families = function (mappings) if modifier then add_family (familyname, subtable, modifier, entry) - if metafamily ~= nil then - add_family (metafamily, subtable, modifier, entry) - end + --- registering the metafamilies is unreliable within the + --- same table as identifiers might interfere with an + --- unmarked style that lacks a metafamily, e.g. + --- + --- iwona condensed regular -> + --- family: iwonacond + --- metafamily: iwona + --- iwona regular -> + --- family: iwona + --- metafamily: ø + --- + --- Both would be registered as under the same family, + --- i.e. “iwona”, and depending on the loading order + --- the query “name:iwona” can resolve to the condensed + --- version instead of the actual unmarked one. The only + --- way around this would be to introduce a separate + --- table for metafamilies and do fallback queries on it. + --- At the moment this is not pressing enough to justify + --- further increasing the index size, maybe if need + --- arises from the user side. +-- if metafamily and metafamily ~= familyname then +-- add_family (metafamily, subtable, modifier, entry) +-- end end end -- cgit v1.2.3