From 08135c42930576980e9bdae7a82208cb9dcf42f9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 19:40:48 +0200 Subject: [db] new structure --- luaotfload-database.lua | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 755ad52..067b48c 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -318,11 +318,19 @@ mtx-fonts has in names.tma: local fontnames_init = function (formats) --- returns dbobj return { - mappings = { }, - status = { }, + families = { + texmf = { }, + system = { }, + }, + index = { }, -- was: status; map abspath -> mapping + mappings = { }, -- TODO: check if still necessary after rewrite + names = { }, -- filenames = { }, -- created later - version = names.version, - formats = formats, + meta = { + formats = formats, + statistics = { }, + version = names.version, + }, } end @@ -2191,8 +2199,14 @@ local gen_fast_lookups = function (fontnames) local nmappings = #mappings --- this is needlessly complicated due to texmf priorization local filenames = { - bare = { }, - base = { }, + bare = { + system = { }, --- mapped to mapping format -> index in full + texmf = { }, --- mapped to mapping format -> “true” + }, + base = { + system = { }, --- mapped to index in “full” + texmf = { }, --- set; all values are “true” + }, full = { }, --- non-texmf } -- cgit v1.2.3 From 4f04f33f36933aaec7626154e055687b2abe903c Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 19:59:24 +0200 Subject: [db] document new db layout for rewrite --- luaotfload-database.lua | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 067b48c..6e49556 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -238,17 +238,41 @@ end This is a sketch of the luaotfload db: type dbobj = { - formats : string list; // { "otf", "ttf", "ttc", "dfont" } mappings : fontentry list; - status : filestatus; - version : float; - // new in v2.3; these supersede the basenames / barenames - // hashes from v2.2 + index : filestatus; + families : familytable; + names : namedata; // TODO: check for relevance after db is finalized + meta : metadata; filenames : filemap; } + and familytable = { + texmf : (format, familyentry) hash; + system : (format, familyentry) hash; + } + and familyentry = { + regular : sizes; + italic : sizes; + bold : sizes; + bolditalic : sizes; + } + and sizes = { + default = int; // points into mappings or names + other = (int, int) list; // design size -> index entry + } + and metadata = { + formats : string list; // { "otf", "ttf", "ttc", "dfont" } + statistics : TODO; + version : float; + } and filemap = { - base : (string, int) hash; // basename -> idx - bare : (string, int) hash; // barename -> idx + base : { + texmf : (string, int) hash; // basename -> idx + system : (string, int) hash; + }; + bare : { + texmf : (string, (string, int) hash) hash; // format -> (barename -> idx) + system : (string, (string, int) hash) hash; + }; full : (int, string) hash; // idx -> full path } and fontentry = { -- cgit v1.2.3 From d66597559f15ce654845a958e2f905bd502a1ed8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:00:52 +0200 Subject: [db] bump db version number --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 6e49556..dedcd01 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -102,7 +102,7 @@ if config.luaotfload.update_live ~= false then config.luaotfload.update_live = true end -names.version = 2.210 +names.version = 2.4 names.data = nil --- contains the loaded database names.lookups = nil --- contains the lookup cache -- cgit v1.2.3 From 55ba70c6d8f76706f083818e7dd38006e88dcfb7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:06:40 +0200 Subject: =?UTF-8?q?[db]=20introduce=20=E2=80=9Clocal=E2=80=9D=20as=20third?= =?UTF-8?q?=20location=20category?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- luaotfload-database.lua | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index dedcd01..5d662e0 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -238,14 +238,15 @@ end This is a sketch of the luaotfload db: type dbobj = { - mappings : fontentry list; - index : filestatus; - families : familytable; - names : namedata; // TODO: check for relevance after db is finalized - meta : metadata; - filenames : filemap; + families : familytable; + filenames : filemap; + index : filestatus; + mappings : fontentry list; + meta : metadata; + names : namedata; // TODO: check for relevance after db is finalized } and familytable = { + local : (format, familyentry) hash; // specified with include dir texmf : (format, familyentry) hash; system : (format, familyentry) hash; } @@ -256,8 +257,8 @@ This is a sketch of the luaotfload db: bolditalic : sizes; } and sizes = { - default = int; // points into mappings or names - other = (int, int) list; // design size -> index entry + default : int; // points into mappings or names + optical : (int, int) list; // design size -> index entry } and metadata = { formats : string list; // { "otf", "ttf", "ttc", "dfont" } @@ -266,12 +267,14 @@ This is a sketch of the luaotfload db: } and filemap = { base : { - texmf : (string, int) hash; // basename -> idx + local : (string, int) hash; // basename -> idx system : (string, int) hash; + texmf : (string, int) hash; }; bare : { - texmf : (string, (string, int) hash) hash; // format -> (barename -> idx) + local : (string, (string, int) hash) hash; // format -> (barename -> idx) system : (string, (string, int) hash) hash; + texmf : (string, (string, int) hash) hash; }; full : (int, string) hash; // idx -> full path } @@ -295,7 +298,7 @@ This is a sketch of the luaotfload db: size : int list; slant : int; subfont : int; - texmf : bool; + location : local | system | texmf; weight : int; width : int; units_per_em : int; // mainly 1000, but also 2048 or 256 @@ -343,8 +346,9 @@ mtx-fonts has in names.tma: local fontnames_init = function (formats) --- returns dbobj return { families = { - texmf = { }, - system = { }, + ["local"] = { }, + system = { }, + texmf = { }, }, index = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite -- cgit v1.2.3 From 4ae6dab28fdf719fd10ba165d1b2414bef6794d8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:40:33 +0200 Subject: [db] refactor texmf and system font scanning --- luaotfload-database.lua | 74 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 5d662e0..0139d89 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -240,7 +240,7 @@ This is a sketch of the luaotfload db: type dbobj = { families : familytable; filenames : filemap; - index : filestatus; + status : filestatus; mappings : fontentry list; meta : metadata; names : namedata; // TODO: check for relevance after db is finalized @@ -301,10 +301,11 @@ This is a sketch of the luaotfload db: location : local | system | texmf; weight : int; width : int; - units_per_em : int; // mainly 1000, but also 2048 or 256 + units_per_em : int; // mainly 1000, but also 2048 or 256 } - and filestatus = (fullname, - { index : int list; timestamp : int }) dict + and filestatus = (string, // fullname + { index : int list; // pointer into mappings + timestamp : int; }) dict beware that this is a reconstruction and may be incomplete. @@ -350,7 +351,7 @@ local fontnames_init = function (formats) --- returns dbobj system = { }, texmf = { }, }, - index = { }, -- was: status; map abspath -> mapping + status = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite names = { }, -- filenames = { }, -- created later @@ -1404,17 +1405,22 @@ local loaders = { pfa = t1_fullinfo, } ---- we return true if the fond is new or re-indexed +--- we return true if the font is new or re-indexed --- string -> dbobj -> dbobj -> bool -local load_font = function (fullname, fontnames, newfontnames, texmf) + +local read_font_names = function (fullname, + fontnames, + newfontnames, + texmf) + local newmappings = newfontnames.mappings local newstatus = newfontnames.status --- by full path local mappings = fontnames.mappings local status = fontnames.status - local basename = filebasename(fullname) - local barename = filenameonly(fullname) + local basename = filebasename (fullname) + local barename = filenameonly (fullname) local format = stringlower (filesuffix (basename)) @@ -1438,7 +1444,7 @@ local load_font = function (fullname, fontnames, newfontnames, texmf) local newentrystatus = newstatus[fullname] --- newentrystatus: nil | false | table if newentrystatus and newentrystatus.timestamp == new_timestamp then - -- already indexed this run + -- already statused this run return false end @@ -1824,6 +1830,7 @@ end --doc]]-- --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) + local scan_dir = function (dirname, fontnames, newfontnames, dry_run, texmf) if lpegmatch (p_blacklist, dirname) then @@ -1849,12 +1856,13 @@ local scan_dir = function (dirname, fontnames, newfontnames, local new if dry_run == true then report ("both", 1, "db", - "Would have been loading %q", fullname) + "Would have been extracting metadata from %q", + fullname) else report ("both", 4, "db", - "Loading font %q", fullname) - local new = load_font (fullname, fontnames, - newfontnames, texmf) + "Extracting metadata from font %q", fullname) + local new = read_font_names (fullname, fontnames, + newfontnames, texmf) if new == true then n_new = n_new + 1 end @@ -1890,6 +1898,7 @@ local path_separator = ostype == "windows" and ";" or ":" --doc]]-- --- dbobj -> dbobj -> bool? -> (int * int) + local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) local n_scanned, n_new, fontdirs = 0, 0 @@ -2301,6 +2310,24 @@ local gen_fast_lookups = function (fontnames) return fontnames end +local retrieve_namedata = function (fontnames, newfontnames, dry_run) + local n_rawnames, n_new = 0, 0 + + local rawnames, new = scan_texmf_fonts (fontnames, + newfontnames, + dry_run) + + n_rawnames = n_rawnames + rawnames + n_new = n_new + new + + rawnames, new = scan_os_fonts (fontnames, newfontnames, dry_run) + + n_rawnames = n_rawnames + rawnames + n_new = n_new + new + + return n_rawnames, n_new +end + --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs @@ -2314,8 +2341,8 @@ update_names = function (fontnames, force, dry_run) return fontnames or names.data end - local starttime = os.gettimeofday() - local n_scanned, n_new = 0, 0 + local starttime = os.gettimeofday() + local n_rawnames, n_new = 0, 0 --[[ The main function, scans everything @@ -2340,14 +2367,11 @@ update_names = function (fontnames, force, dry_run) local newfontnames = fontnames_init (get_font_filter ()) read_blacklist () - local scanned, new - scanned, new = scan_texmf_fonts (fontnames, newfontnames, dry_run) - n_scanned = n_scanned + scanned - n_new = n_new + new - - scanned, new = scan_os_fonts (fontnames, newfontnames, dry_run) - n_scanned = n_scanned + scanned - n_new = n_new + new + local rawnames, new = retrieve_namedata (fontnames, + newfontnames, + dry_run) + n_rawnames = n_rawnames + rawnames + n_new = n_new + new --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping @@ -2359,7 +2383,7 @@ update_names = function (fontnames, force, dry_run) --- partial: 804 ms | 701 ms --- forced: 45384 ms | 44714 ms report("info", 3, "db", - "Scanned %d font files; %d new entries.", n_scanned, n_new) + "Scanned %d font files; %d new entries.", n_rawnames, n_new) report("info", 3, "db", "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) names.data = newfontnames -- cgit v1.2.3 From ef4e74ea5906f81b8b12ddab7bb8417a90635d50 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 20:46:56 +0200 Subject: [db] choose more meaningful identifiers in name extractor --- luaotfload-database.lua | 48 +++++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 0139d89..bc654fa 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1409,22 +1409,22 @@ local loaders = { --- string -> dbobj -> dbobj -> bool local read_font_names = function (fullname, - fontnames, - newfontnames, + currentnames, + targetnames, texmf) - local newmappings = newfontnames.mappings - local newstatus = newfontnames.status --- by full path + local targetmappings = targetnames.mappings + local targetstatus = targetnames.status --- by full path - local mappings = fontnames.mappings - local status = fontnames.status + local currentmappings = currentnames.mappings + local currentstatus = currentnames.status - local basename = filebasename (fullname) - local barename = filenameonly (fullname) + local basename = filebasename (fullname) + local barename = filenameonly (fullname) - local format = stringlower (filesuffix (basename)) + local format = stringlower (filesuffix (basename)) - local entryname = fullname + local entryname = fullname if texmf == true then entryname = basename end @@ -1437,30 +1437,30 @@ local read_font_names = function (fullname, end local new_timestamp, current_timestamp - current_timestamp = status[fullname] - and status[fullname].timestamp + current_timestamp = currentstatus[fullname] + and currentstatus[fullname].timestamp new_timestamp = lfsattributes(fullname, "modification") - local newentrystatus = newstatus[fullname] + local newentrystatus = targetstatus[fullname] --- newentrystatus: nil | false | table if newentrystatus and newentrystatus.timestamp == new_timestamp then -- already statused this run return false end - newstatus[fullname] = newentrystatus or { } - local newentrystatus = newstatus[fullname] + targetstatus[fullname] = newentrystatus or { } + local newentrystatus = targetstatus[fullname] newentrystatus.timestamp = new_timestamp newentrystatus.index = newentrystatus.index or { } if current_timestamp == new_timestamp and not newentrystatus.index[1] then - for _, v in next, status[fullname].index do + for _, v in next, currentstatus[fullname].index do local index = #newentrystatus.index - local fullinfo = mappings[v] - local location = #newmappings + 1 - newmappings[location] = fullinfo --- keep + local fullinfo = currentmappings[v] + local location = #targetmappings + 1 + targetmappings[location] = fullinfo --- keep newentrystatus.index[index+1] = location --- is this actually used anywhere? end report("log", 2, "db", "Font %q already indexed", basename) @@ -1482,11 +1482,11 @@ local read_font_names = function (fullname, if not fullinfo then return false end - local location = #newmappings+1 + local location = #targetmappings + 1 local index = newentrystatus.index[n_font] if not index then index = location end - newmappings[index] = fullinfo + targetmappings[index] = fullinfo newentrystatus.index[n_font] = index end else @@ -1494,11 +1494,11 @@ local read_font_names = function (fullname, if not fullinfo then return false end - local location = #newmappings+1 + local location = #targetmappings + 1 local index = newentrystatus.index[1] if not index then index = location end - newmappings[index] = fullinfo + targetmappings[index] = fullinfo newentrystatus.index[1] = index end @@ -2364,7 +2364,9 @@ update_names = function (fontnames, force, dry_run) fontnames = fontnames_init (get_font_filter ()) end end + local newfontnames = fontnames_init (get_font_filter ()) + read_blacklist () local rawnames, new = retrieve_namedata (fontnames, -- cgit v1.2.3 From 599a9652b02520c6dfcd162c2f90bb1919177517 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:10:51 +0200 Subject: [db] store location (texmf/sys) as string, not bool --- luaotfload-database.lua | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bc654fa..23058cb 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -615,7 +615,7 @@ on whether the index entry has a texmf flag set. local get_font_file = function (fullnames, entry) local basename = entry.basename - if entry.texmf == true then + if entry.location == "texmf" then if kpselookup(basename) then return true, basename, entry.subfont end @@ -1224,7 +1224,7 @@ table as returned by the font file reader need to be relocated. --doc]]-- --- string -> int -> bool -> string -> fontentry -ot_fullinfo = function (filename, subfont, texmf, basename) +ot_fullinfo = function (filename, subfont, location, basename) local namedata = { } local metadata = load_font_file (filename, subfont) @@ -1306,7 +1306,7 @@ ot_fullinfo = function (filename, subfont, texmf, basename) --- file location data (used to be filename field) namedata.filename = filename --> sys namedata.basename = basename --> texmf - namedata.texmf = texmf or false + namedata.location = location or "system" --- TODO or “local”?? namedata.subfont = subfont return namedata @@ -1323,7 +1323,7 @@ end --doc]]-- --- string -> int -> bool -> string -> fontentry -t1_fullinfo = function (filename, _subfont, texmf, basename) +t1_fullinfo = function (filename, _subfont, location, basename) local namedata = { } local metadata = load_font_file (filename) @@ -1390,8 +1390,9 @@ t1_fullinfo = function (filename, _subfont, texmf, basename) namedata.filename = filename --> sys namedata.basename = basename --> texmf - namedata.texmf = texmf or false + namedata.location = location or "system" namedata.subfont = false + return namedata end @@ -1411,7 +1412,7 @@ local loaders = { local read_font_names = function (fullname, currentnames, targetnames, - texmf) + location) local targetmappings = targetnames.mappings local targetstatus = targetnames.status --- by full path @@ -1425,7 +1426,7 @@ local read_font_names = function (fullname, local format = stringlower (filesuffix (basename)) local entryname = fullname - if texmf == true then + if location == "texmf" then entryname = basename end @@ -1475,10 +1476,11 @@ local read_font_names = function (fullname, end local info = fontloaderinfo(fullname) + if info then if type(info) == "table" and #info > 1 then --- ttc for n_font = 1, #info do - local fullinfo = loader (fullname, n_font-1, texmf, basename) + local fullinfo = loader (fullname, n_font-1, location, basename) if not fullinfo then return false end @@ -1490,7 +1492,7 @@ local read_font_names = function (fullname, newentrystatus.index[n_font] = index end else - local fullinfo = loader (fullname, false, texmf, basename) + local fullinfo = loader (fullname, false, location, basename) if not fullinfo then return false end @@ -1832,14 +1834,14 @@ end --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) local scan_dir = function (dirname, fontnames, newfontnames, - dry_run, texmf) + dry_run, location) if lpegmatch (p_blacklist, dirname) then report ("both", 3, "db", "Skipping blacklisted directory %s", dirname) --- ignore return 0, 0 end - local found = find_font_files (dirname, texmf ~= true) + local found = find_font_files (dirname, location ~= "texmf") if not found then report ("both", 3, "db", "No such directory: %q; skipping.", dirname) @@ -1862,7 +1864,7 @@ local scan_dir = function (dirname, fontnames, newfontnames, report ("both", 4, "db", "Extracting metadata from font %q", fullname) local new = read_font_names (fullname, fontnames, - newfontnames, texmf) + newfontnames, location) if new == true then n_new = n_new + 1 end @@ -1928,7 +1930,7 @@ local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) "Initiating scan of %d directories.", #tasks) for _, d in next, tasks do local found, new = scan_dir (d, fontnames, newfontnames, - dry_run, true) + dry_run, "texmf") n_scanned = n_scanned + found n_new = n_new + new end @@ -2092,6 +2094,10 @@ do --- closure for read_fonts_conf() --- We exclude paths with texmf in them, as they should be --- found anyway; also duplicates are ignored by checking --- if they are elements of dirs_done. + --- + --- FIXME does this mean we cannot access paths from + --- distributions (e.g. Context minimals) installed + --- separately? if not (stringfind(path, "texmf") or dirs_done[path]) then acc[#acc+1] = path dirs_done[path] = true @@ -2230,7 +2236,7 @@ flush_lookup_cache = function () end --- dbobj -> dbobj -local gen_fast_lookups = function (fontnames) +local gen_fast_lookups = function (fontnames) --- this will become obsolete report("both", 2, "db", "Creating filename map") local mappings = fontnames.mappings local nmappings = #mappings @@ -2262,7 +2268,7 @@ local gen_fast_lookups = function (fontnames) --- substantial duplication -- entry.filename = nil - if entry.texmf == true then + if entry.location == "texmf" then texmf[#texmf+1] = { idx, basename, bare, true, nil } else sys[#sys+1] = { idx, basename, bare, false, filename } @@ -2378,6 +2384,7 @@ update_names = function (fontnames, force, dry_run) --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent + newfontnames = gen_fast_lookups(newfontnames) --- stats: -- cgit v1.2.3 From a4c38dbe7fa31d7dbd2336a3290d366237621e46 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:34:32 +0200 Subject: [db] refactor timestamp comparison --- luaotfload-database.lua | 118 ++++++++++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 44 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 23058cb..c46956d 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1406,6 +1406,52 @@ local loaders = { pfa = t1_fullinfo, } +--- not side-effect free! + +local compare_timestamps = function (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) + + local currenttimestamp = currententrystatus + and currententrystatus.timestamp + local targettimestamp = lfsattributes (fullname, "modification") + + if targetentrystatus ~= nil + and targetentrystatus.timestamp == targettimestamp then + report ("log", 3, "db", "Font %q already read.", basename) + return false + end + + targetentrystatus.timestamp = targettimestamp + targetentrystatus.index = targetentrystatus.index or { } + + if currenttimestamp == targettimestamp + and not targetentrystatus.index [1] + then + + --- copy old namedata into new + + for _, currentindex in next, currententrystatus.index do + + local targetindex = #targetentrystatus.index + local fullinfo = currentmappings [currentindex] + local location = #targetmappings + 1 + + targetmappings [location] = fullinfo + targetentrystatus.index [targetindex + 1] = location + end + + report ("log", 3, "db", "Font %q already indexed.", basename) + + return false + end +end + + --- we return true if the font is new or re-indexed --- string -> dbobj -> dbobj -> bool @@ -1414,18 +1460,25 @@ local read_font_names = function (fullname, targetnames, location) - local targetmappings = targetnames.mappings - local targetstatus = targetnames.status --- by full path + local targetmappings = targetnames.mappings + local targetstatus = targetnames.status --- by full path + local targetentrystatus = targetstatus [fullname] - local currentmappings = currentnames.mappings - local currentstatus = currentnames.status + if targetentrystatus == nil then + targetentrystatus = { } + targetstatus [fullname] = targetentrystatus + end - local basename = filebasename (fullname) - local barename = filenameonly (fullname) + local currentmappings = currentnames.mappings + local currentstatus = currentnames.status + local currententrystatus = currentstatus [fullname] - local format = stringlower (filesuffix (basename)) + local basename = filebasename (fullname) + local barename = filenameonly (fullname) + local entryname = fullname + + local format = stringlower (filesuffix (basename)) - local entryname = fullname if location == "texmf" then entryname = basename end @@ -1437,36 +1490,13 @@ local read_font_names = function (fullname, return false end - local new_timestamp, current_timestamp - current_timestamp = currentstatus[fullname] - and currentstatus[fullname].timestamp - new_timestamp = lfsattributes(fullname, "modification") - - local newentrystatus = targetstatus[fullname] - --- newentrystatus: nil | false | table - if newentrystatus and newentrystatus.timestamp == new_timestamp then - -- already statused this run - return false - end - - targetstatus[fullname] = newentrystatus or { } - local newentrystatus = targetstatus[fullname] - newentrystatus.timestamp = new_timestamp - newentrystatus.index = newentrystatus.index or { } - - if current_timestamp == new_timestamp - and not newentrystatus.index[1] - then - for _, v in next, currentstatus[fullname].index do - local index = #newentrystatus.index - local fullinfo = currentmappings[v] - local location = #targetmappings + 1 - targetmappings[location] = fullinfo --- keep - newentrystatus.index[index+1] = location --- is this actually used anywhere? - end - report("log", 2, "db", "Font %q already indexed", basename) - return false - end + local changed = compare_timestamps (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) local loader = loaders[format] --- ot_fullinfo, t1_fullinfo if not loader then @@ -1485,11 +1515,11 @@ local read_font_names = function (fullname, return false end local location = #targetmappings + 1 - local index = newentrystatus.index[n_font] + local index = targetentrystatus.index[n_font] if not index then index = location end - targetmappings[index] = fullinfo - newentrystatus.index[n_font] = index + targetmappings[index] = fullinfo + targetentrystatus.index[n_font] = index end else local fullinfo = loader (fullname, false, location, basename) @@ -1497,11 +1527,11 @@ local read_font_names = function (fullname, return false end local location = #targetmappings + 1 - local index = newentrystatus.index[1] + local index = targetentrystatus.index[1] if not index then index = location end - targetmappings[index] = fullinfo - newentrystatus.index[1] = index + targetmappings[index] = fullinfo + targetentrystatus.index[1] = index end else --- missing info -- cgit v1.2.3 From c39f4e5ea942a88a319321b8cd546532300700dc Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:39:42 +0200 Subject: [db] correctly abort if timestamps match --- luaotfload-database.lua | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index c46956d..6202079 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -431,7 +431,7 @@ load_names = function (dry_run) report ("info", 3, "db", "Loading took %0.f ms", 1000*(os.gettimeofday()-starttime)) - local db_version, nms_version = data.version, names.version + local db_version, nms_version = data.meta.version, names.version if db_version ~= nms_version then report ("both", 0, "db", [[Version mismatch; expected %4.3f, got %4.3f]], @@ -1449,6 +1449,8 @@ local compare_timestamps = function (fullname, return false end + + return true end @@ -1490,13 +1492,16 @@ local read_font_names = function (fullname, return false end - local changed = compare_timestamps (fullname, - currentstatus, - currententrystatus, - currentmappings, - targetstatus, - targetentrystatus, - targetmappings) + if not compare_timestamps (fullname, + currentstatus, + currententrystatus, + currentmappings, + targetstatus, + targetentrystatus, + targetmappings) + then + return false + end local loader = loaders[format] --- ot_fullinfo, t1_fullinfo if not loader then -- cgit v1.2.3 From 698b5178cbb9d8d9c9cf5f429478770f1645cb43 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sat, 31 Aug 2013 21:58:40 +0200 Subject: [db] refactor font info extraction --- luaotfload-database.lua | 99 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 6202079..3ffc1c4 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1453,6 +1453,36 @@ local compare_timestamps = function (fullname, return true end +local insert_fullinfo = function (fullname, + basename, + n_font, + loader, + location, + targetmappings, + targetentrystatus) + + local subfont = n_font and n_font - 1 or false + + local fullinfo = loader (fullname, subfont, + location, basename) + + if not fullinfo then + return false + end + + local index = targetentrystatus.index [n_font] + + if not index then + index = #targetmappings + 1 + end + + targetmappings [index] = fullinfo + targetentrystatus.index [n_font] = index + + return true +end + + --- we return true if the font is new or re-indexed --- string -> dbobj -> dbobj -> bool @@ -1479,19 +1509,20 @@ local read_font_names = function (fullname, local barename = filenameonly (fullname) local entryname = fullname - local format = stringlower (filesuffix (basename)) - if location == "texmf" then entryname = basename end - if names.blacklist[fullname] or names.blacklist[basename] - then + --- 1) skip if blacklisted + + if names.blacklist[fullname] or names.blacklist[basename] then report("log", 2, "db", "Ignoring blacklisted font %q", fullname) return false end + --- 2) skip if known with same timestamp + if not compare_timestamps (fullname, currentstatus, currententrystatus, @@ -1503,47 +1534,49 @@ local read_font_names = function (fullname, return false end - local loader = loaders[format] --- ot_fullinfo, t1_fullinfo + --- 3) new font; choose a loader, abort if unknown + + local format = stringlower (filesuffix (basename)) + local loader = loaders [format] --- ot_fullinfo, t1_fullinfo + if not loader then report ("both", 0, "db", "Unknown format: %q, skipping.", format) return false end - local info = fontloaderinfo(fullname) + --- 4) get basic info, abort if fontloader can’t read it - if info then - if type(info) == "table" and #info > 1 then --- ttc - for n_font = 1, #info do - local fullinfo = loader (fullname, n_font-1, location, basename) - if not fullinfo then - return false - end - local location = #targetmappings + 1 - local index = targetentrystatus.index[n_font] - if not index then index = location end + local info = fontloaderinfo (fullname) + + if not info then + report ("log", 1, "db", + "Failed to read basic information from %q", basename) + return false + end - targetmappings[index] = fullinfo - targetentrystatus.index[n_font] = index - end - else - local fullinfo = loader (fullname, false, location, basename) - if not fullinfo then - return false - end - local location = #targetmappings + 1 - local index = targetentrystatus.index[1] - if not index then index = location end - targetmappings[index] = fullinfo - targetentrystatus.index[1] = index + --- 5) check for subfonts and process each of them + + if type (info) == "table" and #info > 1 then --- ttc + + local success = false --- true if at least one subfont got read + + for n_font = 1, #info do + if insert_fullinfo (fullname, basename, n_font, + loader, location, + targetmappings, targetentrystatus) + then + success = true + end end - else --- missing info - report("log", 1, "db", "Failed to load %q", basename) - return false + return success end - return true + + return insert_fullinfo (fullname, basename, false, + loader, location, + targetmappings, targetentrystatus) end local path_normalize -- cgit v1.2.3 From 1167ef911872ca57f9deacde1c4b1d5eb71309ad Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 12:04:41 +0200 Subject: [db] reorganize mappings record; refactor metadata extraction --- luaotfload-database.lua | 262 ++++++++++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 107 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 3ffc1c4..a85079b 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -272,7 +272,7 @@ This is a sketch of the luaotfload db: texmf : (string, int) hash; }; bare : { - local : (string, (string, int) hash) hash; // format -> (barename -> idx) + local : (string, (string, int) hash) hash; // location -> (barename -> idx) system : (string, (string, int) hash) hash; texmf : (string, (string, int) hash) hash; }; @@ -1217,21 +1217,31 @@ local load_font_file = function (filename, subfont) return metadata end ---[[doc-- -The data inside an Opentype font file can be quite heterogeneous. -Thus in order to get the relevant information, parts of the original -table as returned by the font file reader need to be relocated. ---doc]]-- +--- rawdata -> (int * int * int | bool) ---- string -> int -> bool -> string -> fontentry -ot_fullinfo = function (filename, subfont, location, basename) - local namedata = { } +local get_size_info = function (metadata) + local design_size = metadata.design_size + local design_range_top = metadata.design_range_top + local design_range_bottom = metadata.design_range_bottom - local metadata = load_font_file (filename, subfont) - if not metadata then - return nil + local fallback_size = design_size ~= 0 and design_size + or design_range_bottom ~= 0 and design_range_bottom + or design_range_top ~= 0 and design_range_top + + if fallback_size then + design_size = (design_size or fallback_size) / 10 + design_range_top = (design_range_top or fallback_size) / 10 + design_range_bottom = (design_range_bottom or fallback_size) / 10 + return { + design_size, design_range_top, design_range_bottom, + } end + return false +end + +local extract_namedata = function (metadata, basename) + local english_names if metadata.names then @@ -1241,11 +1251,11 @@ ot_fullinfo = function (filename, subfont, location, basename) end end else - -- no names table, propably a broken font + -- no names table, probably a broken font report("log", 1, "db", "Broken font %s rejected due to missing names table.", basename) - return + return nil end local fontnames = { @@ -1272,44 +1282,56 @@ ot_fullinfo = function (filename, subfont, location, basename) end end - namedata.sanitized = sanitize_names (fontnames) - namedata.fontname = metadata.fontname - namedata.fullname = metadata.fullname - namedata.familyname = metadata.familyname - namedata.weight = metadata.pfminfo.weight - namedata.width = metadata.pfminfo.width - namedata.slant = metadata.italicangle - --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details - namedata.units_per_em = metadata.units_per_em - namedata.version = metadata.version - -- don't waste the space with zero values + return { + sanitized = sanitize_names (fontnames), + fontname = metadata.fontname, + fullname = metadata.fullname, + familyname = metadata.familyname, + } - local design_size = metadata.design_size - local design_range_top = metadata.design_range_top - local design_range_bottom = metadata.design_range_bottom +end - local fallback_size = design_size ~= 0 and design_size - or design_range_bottom ~= 0 and design_range_bottom - or design_range_top ~= 0 and design_range_top - if fallback_size then - design_size = (design_size or fallback_size) / 10 - design_range_top = (design_range_top or fallback_size) / 10 - design_range_bottom = (design_range_bottom or fallback_size) / 10 - namedata.size = { - design_size, design_range_top, design_range_bottom, - } - else - namedata.size = false +--[[doc-- +The data inside an Opentype font file can be quite heterogeneous. +Thus in order to get the relevant information, parts of the original +table as returned by the font file reader need to be relocated. +--doc]]-- + +--- string -> int -> bool -> string -> fontentry + +ot_fullinfo = function (filename, subfont, location, basename, format) + local styles = { } + + local metadata = load_font_file (filename, subfont) + if not metadata then + return nil end - --- file location data (used to be filename field) - namedata.filename = filename --> sys - namedata.basename = basename --> texmf - namedata.location = location or "system" --- TODO or “local”?? - namedata.subfont = subfont + local namedata = extract_namedata (metadata, basename) - return namedata + + local style = { + size = get_size_info (metadata), + weight = metadata.pfminfo.weight, + width = metadata.pfminfo.width, + slant = metadata.italicangle, + --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details + units_per_em = metadata.units_per_em, + version = metadata.version, + } + + return { + file = { base = basename, + full = filename, + subfont = subfont, + location = location or "system" }, + format = format, + names = namedata, + style = style, + units_per_em = metadata.units_per_em, + version = metadata.version, + } end --[[doc-- @@ -1323,7 +1345,8 @@ end --doc]]-- --- string -> int -> bool -> string -> fontentry -t1_fullinfo = function (filename, _subfont, location, basename) + +t1_fullinfo = function (filename, _subfont, location, basename, format) local namedata = { } local metadata = load_font_file (filename) @@ -1390,6 +1413,7 @@ t1_fullinfo = function (filename, _subfont, location, basename) namedata.filename = filename --> sys namedata.basename = basename --> texmf + namedata.format = format namedata.location = location or "system" namedata.subfont = false @@ -1457,6 +1481,7 @@ local insert_fullinfo = function (fullname, basename, n_font, loader, + format, location, targetmappings, targetentrystatus) @@ -1464,7 +1489,8 @@ local insert_fullinfo = function (fullname, local subfont = n_font and n_font - 1 or false local fullinfo = loader (fullname, subfont, - location, basename) + location, basename, + format) if not fullinfo then return false @@ -1564,7 +1590,7 @@ local read_font_names = function (fullname, for n_font = 1, #info do if insert_fullinfo (fullname, basename, n_font, - loader, location, + loader, format, location, targetmappings, targetentrystatus) then success = true @@ -1575,7 +1601,7 @@ local read_font_names = function (fullname, end return insert_fullinfo (fullname, basename, false, - loader, location, + loader, format, location, targetmappings, targetentrystatus) end @@ -2303,85 +2329,107 @@ flush_lookup_cache = function () return true, names.lookups end ---- dbobj -> dbobj -local gen_fast_lookups = function (fontnames) --- this will become obsolete - report("both", 2, "db", "Creating filename map") - local mappings = fontnames.mappings + +--- fontentry list -> filemap + +local generate_filedata = function (mappings) + + report ("both", 2, "db", "Creating filename map") + local nmappings = #mappings - --- this is needlessly complicated due to texmf priorization + local filenames = { bare = { - system = { }, --- mapped to mapping format -> index in full - texmf = { }, --- mapped to mapping format -> “true” + ["local"] = { }, + system = { }, --- mapped to mapping format -> index in full + texmf = { }, --- mapped to mapping format -> “true” }, base = { - system = { }, --- mapped to index in “full” - texmf = { }, --- set; all values are “true” + ["local"] = { }, + system = { }, --- mapped to index in “full” + texmf = { }, --- set; all values are “true” }, full = { }, --- non-texmf } - local texmf, sys = { }, { } -- quintuple list + local base = filenames.base + local bare = filenames.bare + local full = filenames.full + + local conflicts = { + basenames = 0, + barenames = 0, + } + + for index = 1, nmappings do + local entry = mappings [index] + + local filedata = entry.file + + local format = entry.format --- otf, afm, ... + local location = filedata.location --- texmf, system, ... - for idx = 1, nmappings do - local entry = mappings[idx] - local filename = entry.filename - local basename = entry.basename - local bare = filenameonly(filename) - local subfont = entry.subfont + local filename = filedata.full + local basename = filedata.base + local barename = filenameonly (filename) + local subfont = filedata.subfont - entry.index = idx ---- unfortunately, the sys/texmf schism prevents us from ---- doing away the full name, so we cannot avoid the ---- substantial duplication --- entry.filename = nil + entry.index = index - if entry.location == "texmf" then - texmf[#texmf+1] = { idx, basename, bare, true, nil } + --- 1) add to basename table + + local inbase = base [location] --- no format since the suffix is known + + if inbase then + if inbase [basename] then + report ("both", 3, "db", + "Conflicting basename: %q already indexed \z + in category %s, ignoring.", + barename, location) + conflicts.basenames = conflicts.basenames + 1 + else + inbase [basename] = index + end else - sys[#sys+1] = { idx, basename, bare, false, filename } + inbase = { basename = index } + base [location] = inbase end - end - local addmap = function (lst) - --- this will overwrite existing entries - for i=1, #lst do - local idx, base, bare, intexmf, full = unpack(lst[i]) + --- 2) add to barename table - local known = filenames.base[base] or filenames.bare[bare] - if known then --- known - report("both", 3, "db", - "Font file %q already indexed (%d)", - base, idx) - report("both", 3, "db", "> old location: %s", - (filenames.full[known] or "texmf")) - report("both", 3, "db", "> new location: %s", - (intexmf and "texmf" or full)) - end + local inbare = bare [location] [format] - filenames.bare[bare] = idx - filenames.base[base] = idx - if intexmf == true then - filenames.full[idx] = nil + if inbare then + if inbare [barename] then + report ("both", 3, "db", + "Conflicting barename: %q already indexed \z + in category %s/%s, ignoring.", + barename, location, format) + conflicts.barenames = conflicts.barenames + 1 else - filenames.full[idx] = full + inbare [barename] = index end + else + inbare = { barename = index } + bare [location] [format] = inbare end - end - if config.luaotfload.prioritize == "texmf" then - report("both", 2, "db", "Preferring texmf fonts") - addmap(sys) - addmap(texmf) - else --- sys - addmap(texmf) - addmap(sys) + --- 3) add to fullname map + + full [index] = fullname end - fontnames.filenames = filenames - texmf, sys = nil, nil - collectgarbage "collect" - return fontnames + --- TODO adapt to new mechanism! +-- if config.luaotfload.prioritize == "texmf" then +-- report("both", 2, "db", "Preferring texmf fonts") +-- addmap(sys) +-- addmap(texmf) +-- else --- sys +-- addmap(texmf) +-- addmap(sys) +-- end + + return filenames end local retrieve_namedata = function (fontnames, newfontnames, dry_run) @@ -2453,7 +2501,7 @@ update_names = function (fontnames, force, dry_run) --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - newfontnames = gen_fast_lookups(newfontnames) + newfontnames.files = generate_filedata (newfontnames.mappings) --- stats: --- before rewrite | after rewrite -- cgit v1.2.3 From 1b5783af6e22335afa514712e738273d323bc76a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 12:18:17 +0200 Subject: [db] include information about conflicting file names in record --- luaotfload-database.lua | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index a85079b..946c773 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2369,9 +2369,9 @@ local generate_filedata = function (mappings) local format = entry.format --- otf, afm, ... local location = filedata.location --- texmf, system, ... - local filename = filedata.full + local fullname = filedata.full local basename = filedata.base - local barename = filenameonly (filename) + local barename = filenameonly (fullname) local subfont = filedata.subfont entry.index = index @@ -2381,12 +2381,23 @@ local generate_filedata = function (mappings) local inbase = base [location] --- no format since the suffix is known if inbase then - if inbase [basename] then + local present = inbase [basename] + if present then report ("both", 3, "db", "Conflicting basename: %q already indexed \z in category %s, ignoring.", barename, location) conflicts.basenames = conflicts.basenames + 1 + + --- track conflicts per font + local conflictdata = entry.conflicts + + if not conflictdata then + entry.conflicts = { basename = present } + else -- some conflicts already detected + conflictdata.basename = present + end + else inbase [basename] = index end @@ -2400,12 +2411,23 @@ local generate_filedata = function (mappings) local inbare = bare [location] [format] if inbare then - if inbare [barename] then + local present = inbare [barename] + if present then report ("both", 3, "db", "Conflicting barename: %q already indexed \z in category %s/%s, ignoring.", barename, location, format) conflicts.barenames = conflicts.barenames + 1 + + --- track conflicts per font + local conflictdata = entry.conflicts + + if not conflictdata then + entry.conflicts = { barename = present } + else -- some conflicts already detected + conflictdata.barename = present + end + else inbare [barename] = index end -- cgit v1.2.3 From 0a60f38e18e916ef44a71454032471f1b2173bd6 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 13:22:10 +0200 Subject: [db] cleanup: use more consistent identifiers --- luaotfload-database.lua | 122 ++++++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 946c773..2a9a3e9 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -44,6 +44,7 @@ local lfscurrentdir = lfs.currentdir local lfsdir = lfs.dir local mathabs = math.abs local mathmin = math.min +local osgettimeofday = os.gettimeofday local osremove = os.remove local stringfind = string.find local stringformat = string.format @@ -344,7 +345,7 @@ mtx-fonts has in names.tma: --doc]]-- -local fontnames_init = function (formats) --- returns dbobj +local initialize_namedata = function (formats) --- returns dbobj return { families = { ["local"] = { }, @@ -422,14 +423,14 @@ local fuzzy_limit = 1 --- display closest only --- bool? -> dbobj load_names = function (dry_run) - local starttime = os.gettimeofday () + local starttime = osgettimeofday () local foundname, data = load_lua_file (names.path.index.lua) if data then report ("both", 2, "db", "Font names database loaded", "%s", foundname) report ("info", 3, "db", "Loading took %0.f ms", - 1000*(os.gettimeofday()-starttime)) + 1000 * (osgettimeofday () - starttime)) local db_version, nms_version = data.meta.version, names.version if db_version ~= nms_version then @@ -450,7 +451,7 @@ load_names = function (dry_run) [[Font names database not found, generating new one.]]) report ("both", 0, "db", [[This can take several minutes; please be patient.]]) - data = update_names (fontnames_init (get_font_filter ()), + data = update_names (initialize_namedata (get_font_filter ()), nil, dry_run) if not success then report ("both", 0, "db", "Database creation unsuccessful.") @@ -1918,16 +1919,16 @@ end scan_dir() scans a directory and populates the list of fonts with all the fonts it finds. - · dirname : name of the directory to scan - · fontnames : current font db object - · newnames : font db object to fill - · dry_run : don’t touch anything + · dirname : name of the directory to scan + · currentnames : current font db object + · targetnames : font db object to fill + · dry_run : don’t touch anything --doc]]-- --- string -> dbobj -> dbobj -> bool -> bool -> (int * int) -local scan_dir = function (dirname, fontnames, newfontnames, +local scan_dir = function (dirname, currentnames, targetnames, dry_run, location) if lpegmatch (p_blacklist, dirname) then report ("both", 3, "db", @@ -1957,8 +1958,8 @@ local scan_dir = function (dirname, fontnames, newfontnames, else report ("both", 4, "db", "Extracting metadata from font %q", fullname) - local new = read_font_names (fullname, fontnames, - newfontnames, location) + local new = read_font_names (fullname, currentnames, + targetnames, location) if new == true then n_new = n_new + 1 end @@ -1995,7 +1996,7 @@ local path_separator = ostype == "windows" and ";" or ":" --- dbobj -> dbobj -> bool? -> (int * int) -local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) +local scan_texmf_fonts = function (currentnames, targetnames, dry_run) local n_scanned, n_new, fontdirs = 0, 0 local osfontdir = kpseexpand_path "$OSFONTDIR" @@ -2023,7 +2024,7 @@ local scan_texmf_fonts = function (fontnames, newfontnames, dry_run) report ("info", 3, "db", "Initiating scan of %d directories.", #tasks) for _, d in next, tasks do - local found, new = scan_dir (d, fontnames, newfontnames, + local found, new = scan_dir (d, currentnames, targetnames, dry_run, "texmf") n_scanned = n_scanned + found n_new = n_new + new @@ -2303,7 +2304,8 @@ end --doc]]-- --- dbobj -> dbobj -> bool? -> (int * int) -local scan_os_fonts = function (fontnames, newfontnames, +local scan_os_fonts = function (currentnames, + targetnames, dry_run) local n_scanned, n_new = 0, 0 @@ -2312,8 +2314,8 @@ local scan_os_fonts = function (fontnames, newfontnames, "Searching in static system directories...") for _, d in next, get_os_dirs () do - local found, new = scan_dir (d, fontnames, - newfontnames, dry_run) + local found, new = scan_dir (d, currentnames, + targetnames, dry_run) n_scanned = n_scanned + found n_new = n_new + new end @@ -2454,86 +2456,90 @@ local generate_filedata = function (mappings) return filenames end -local retrieve_namedata = function (fontnames, newfontnames, dry_run) - local n_rawnames, n_new = 0, 0 - - local rawnames, new = scan_texmf_fonts (fontnames, - newfontnames, +local retrieve_namedata = function (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) + local rawnames, new = scan_texmf_fonts (currentnames, + targetnames, dry_run) n_rawnames = n_rawnames + rawnames - n_new = n_new + new + n_newnames = n_newnames + new - rawnames, new = scan_os_fonts (fontnames, newfontnames, dry_run) + rawnames, new = scan_os_fonts (currentnames, targetnames, dry_run) n_rawnames = n_rawnames + rawnames - n_new = n_new + new + n_newnames = n_newnames + new - return n_rawnames, n_new + return n_rawnames, n_newnames end --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs --- dbobj? -> bool? -> bool? -> dbobj -update_names = function (fontnames, force, dry_run) +update_names = function (currentnames, force, dry_run) if config.luaotfload.update_live == false then - report("info", 2, "db", - "Skipping database update") + report ("info", 2, "db", + "Skipping database update") --- skip all db updates - return fontnames or names.data + return currentnames or names.data end - local starttime = os.gettimeofday() - local n_rawnames, n_new = 0, 0 + local starttime = osgettimeofday () + local n_rawnames, n_newnames = 0, 0 --[[ The main function, scans everything - - “newfontnames” is the final table to return + - “targetnames” is the final table to return - force is whether we rebuild it from scratch or not ]] - report("both", 2, "db", "Updating the font names database" - .. (force and " forcefully" or "")) + report ("both", 2, "db", "Updating the font names database" + .. (force and " forcefully" or "")) if force then - fontnames = fontnames_init (get_font_filter ()) + currentnames = initialize_namedata (get_font_filter ()) else - if not fontnames then - fontnames = load_names (dry_run) + if not currentnames then + currentnames = load_names (dry_run) end - if fontnames.version ~= names.version then + if currentnames.meta.version ~= names.version then report ("both", 1, "db", "No font names database or old " .. "one found; generating new one") - fontnames = fontnames_init (get_font_filter ()) + currentnames = initialize_namedata (get_font_filter ()) end end - local newfontnames = fontnames_init (get_font_filter ()) + local targetnames = initialize_namedata (get_font_filter ()) read_blacklist () - local rawnames, new = retrieve_namedata (fontnames, - newfontnames, - dry_run) - n_rawnames = n_rawnames + rawnames - n_new = n_new + new + local n_rawnames, n_newnames = retrieve_namedata (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - newfontnames.files = generate_filedata (newfontnames.mappings) + targetnames.files = generate_filedata (targetnames.mappings) --- stats: --- before rewrite | after rewrite --- partial: 804 ms | 701 ms --- forced: 45384 ms | 44714 ms - report("info", 3, "db", - "Scanned %d font files; %d new entries.", n_rawnames, n_new) - report("info", 3, "db", - "Rebuilt in %0.f ms", 1000*(os.gettimeofday()-starttime)) - names.data = newfontnames + report ("info", 3, "db", + "Scanned %d font files; %d new entries.", + n_rawnames, n_newnames) + report ("info", 3, "db", + "Rebuilt in %0.f ms.", + 1000 * (osgettimeofday () - starttime)) + names.data = targetnames if dry_run ~= true then @@ -2545,11 +2551,11 @@ update_names = function (fontnames, force, dry_run) if success then logs.names_report ("info", 2, "cache", "Lookup cache emptied") - return newfontnames + return targetnames end end end - return newfontnames + return targetnames end --- unit -> bool @@ -2581,14 +2587,16 @@ end --- save_names() is usually called without the argument --- dbobj? -> bool -save_names = function (fontnames) - if not fontnames then fontnames = names.data end +save_names = function (currentnames) + if not currentnames then + currentnames = names.data + end local path = names.path.index local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, fontnames, true) + tabletofile (luaname, currentnames, true) osremove (lucname) - caches.compile (fontnames, luaname, lucname) + caches.compile (currentnames, luaname, lucname) if lfsisfile (luaname) and lfsisfile (lucname) then report ("info", 1, "db", "Font index saved") report ("info", 3, "db", "Text: " .. luaname) -- cgit v1.2.3 From 99f5ace3d56ae14baabdc494bb4b3326b8652ac7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 13:54:43 +0200 Subject: [db] adapt sanitization routines --- luaotfload-database.lua | 95 +++++++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 2a9a3e9..7be0600 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -180,14 +180,34 @@ end Auxiliary functions --doc]]-- +--- fontnames contain all kinds of garbage; as a precaution we +--- lowercase and strip them of non alphanumerical characters + --- string -> string -local sanitize_string = function (str) + +local invalidchars = "[^%a%d]" + +local sanitize_fontname = function (str) if str ~= nil then - return utf8gsub(utf8lower(str), "[^%a%d]", "") + return utf8gsub (utf8lower (str), invalidchars, "") end return nil end +local sanitize_fontnames = function (rawnames) + local result = { } + for category, namedata in next, rawnames do + local target = { } + for field, name in next, namedata do + target [field] = utf8gsub (utf8lower (name), + invalidchars, + "") + end + result [category] = target + end + return result +end + local find_files_indeed find_files_indeed = function (acc, dirs, filter) if not next (dirs) then --- done @@ -848,8 +868,8 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C if not fonts_loaded then names.data = load_names() end local data = names.data - local name = sanitize_string(specification.name) - local style = sanitize_string(specification.style) or "regular" + local name = sanitize_fontname (specification.name) + local style = sanitize_fontname (specification.style) or "regular" local askedsize @@ -909,8 +929,8 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C pfullname = facenames.pfullname metafamily = facenames.metafamily end - fontname = fontname or sanitize_string(face.fontname) - pfullname = pfullname or sanitize_string(face.fullname) + fontname = fontname or sanitize_fontname (face.fontname) + pfullname = pfullname or sanitize_fontname (face.fullname) if name == family or name == metafamily @@ -1126,7 +1146,7 @@ end --- string -> int -> bool find_closest = function (name, limit) - local name = sanitize_string(name) + local name = sanitize_fontname (name) limit = limit or fuzzy_limit if not fonts_loaded then names.data = load_names() end @@ -1198,14 +1218,6 @@ find_closest = function (name, limit) return false end --- find_closest() -local sanitize_names = function (names) - local res = { } - for idx, name in next, names do - res[idx] = sanitize_string(name) - end - return res -end - local load_font_file = function (filename, subfont) local rawfont, _msg = fontloaderopen (filename, subfont) if not rawfont then @@ -1241,7 +1253,7 @@ local get_size_info = function (metadata) return false end -local extract_namedata = function (metadata, basename) +local organize_namedata = function (metadata, basename, info) local english_names @@ -1262,16 +1274,22 @@ local extract_namedata = function (metadata, basename) local fontnames = { --- see --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html - fullname = english_names.compatfull - or english_names.fullname, - family = english_names.preffamilyname - or english_names.family, - prefmodifiers = english_names.prefmodifiers, - subfamily = english_names.subfamily, - psname = english_names.postscriptname, - pfullname = metadata.fullname, - fontname = metadata.fontname, - metafamily = metadata.familyname, + english = { + fullname = english_names.compatfull + or english_names.fullname, + family = english_names.preffamilyname + or english_names.family, + prefmodifiers = english_names.prefmodifiers, + subfamily = english_names.subfamily, + psname = english_names.postscriptname, + }, + metadata = { + pfullname = metadata.fullname, + fontname = metadata.fontname, + metafamily = metadata.familyname, + }, + info = { + }, } -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size @@ -1284,7 +1302,7 @@ local extract_namedata = function (metadata, basename) end return { - sanitized = sanitize_names (fontnames), + sanitized = sanitize_fontnames (fontnames), fontname = metadata.fontname, fullname = metadata.fullname, familyname = metadata.familyname, @@ -1301,7 +1319,12 @@ table as returned by the font file reader need to be relocated. --- string -> int -> bool -> string -> fontentry -ot_fullinfo = function (filename, subfont, location, basename, format) +ot_fullinfo = function (filename, + subfont, + location, + basename, + format, + info) local styles = { } local metadata = load_font_file (filename, subfont) @@ -1309,7 +1332,7 @@ ot_fullinfo = function (filename, subfont, location, basename, format) return nil end - local namedata = extract_namedata (metadata, basename) + local namedata = organize_namedata (metadata, basename, info) local style = { @@ -1369,7 +1392,7 @@ t1_fullinfo = function (filename, _subfont, location, basename, format) local style_synonyms_set = style_synonyms.set if weight then - weight = sanitize_string (weight) + weight = sanitize_fontname (weight) local tmp = "" if style_synonyms_set.bold[weight] then tmp = "bold" @@ -1390,7 +1413,7 @@ t1_fullinfo = function (filename, _subfont, location, basename, format) --- else italic end - namedata.sanitized = sanitize_names ({ + namedata.sanitized = sanitize_fontnames ({ fontname = fontname, psname = fullname, pfullname = fullname, @@ -1485,13 +1508,14 @@ local insert_fullinfo = function (fullname, format, location, targetmappings, - targetentrystatus) + targetentrystatus, + info) local subfont = n_font and n_font - 1 or false local fullinfo = loader (fullname, subfont, location, basename, - format) + format, info) if not fullinfo then return false @@ -1592,7 +1616,8 @@ local read_font_names = function (fullname, for n_font = 1, #info do if insert_fullinfo (fullname, basename, n_font, loader, format, location, - targetmappings, targetentrystatus) + targetmappings, targetentrystatus, + info) then success = true end @@ -2778,7 +2803,7 @@ names.update = update_names names.crude_file_lookup = crude_file_lookup names.crude_file_lookup_verbose = crude_file_lookup_verbose names.read_blacklist = read_blacklist -names.sanitize_string = sanitize_string +names.sanitize_fontname = sanitize_fontname names.getfilename = resolve_fullpath --- font cache -- cgit v1.2.3 From 87223dfb2c0768be70cdf58dce45cb877bc3757a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 17:33:27 +0200 Subject: [db] extend name extraction on fontloader.info() --- luaotfload-database.lua | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 7be0600..98b0975 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -197,13 +197,20 @@ end local sanitize_fontnames = function (rawnames) local result = { } for category, namedata in next, rawnames do - local target = { } - for field, name in next, namedata do - target [field] = utf8gsub (utf8lower (name), - invalidchars, - "") + + if type (namedata) == "string" then + result [category] = utf8gsub (utf8lower (namedata), + invalidchars, + "") + else + local target = { } + for field, name in next, namedata do + target [field] = utf8gsub (utf8lower (name), + invalidchars, + "") + end + result [category] = target end - result [category] = target end return result end @@ -1224,9 +1231,18 @@ local load_font_file = function (filename, subfont) report ("log", 1, "db", "ERROR: failed to open %s", filename) return end + local metadata = fontloaderto_table (rawfont) fontloaderclose (rawfont) + + metadata.glyphs = nil + metadata.subfonts = nil + metadata.gpos = nil + metadata.gsub = nil + metadata.lookups = nil + collectgarbage "collect" + return metadata end @@ -1279,8 +1295,8 @@ local organize_namedata = function (metadata, basename, info) or english_names.fullname, family = english_names.preffamilyname or english_names.family, - prefmodifiers = english_names.prefmodifiers, - subfamily = english_names.subfamily, + --prefmodifiers = english_names.prefmodifiers, --> style + --subfamily = english_names.subfamily, --> style psname = english_names.postscriptname, }, metadata = { @@ -1289,6 +1305,9 @@ local organize_namedata = function (metadata, basename, info) metafamily = metadata.familyname, }, info = { + fullname = info.fullname, + familyname = info.familyname, + fontname = info.fontname, }, } @@ -1628,7 +1647,8 @@ local read_font_names = function (fullname, return insert_fullinfo (fullname, basename, false, loader, format, location, - targetmappings, targetentrystatus) + targetmappings, targetentrystatus, + info) end local path_normalize -- cgit v1.2.3 From 1f8bcae1bfd697adf582d2a6c1add145eb304f9b Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 19:43:00 +0200 Subject: [db] refactor fontname extraction --- luaotfload-database.lua | 104 +++++++++++++++++++++++++++++++++++++----------- luaotfload-tool.lua | 6 +-- 2 files changed, 84 insertions(+), 26 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 98b0975..b331629 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -189,7 +189,8 @@ local invalidchars = "[^%a%d]" local sanitize_fontname = function (str) if str ~= nil then - return utf8gsub (utf8lower (str), invalidchars, "") + str = utf8gsub (utf8lower (str), invalidchars, "") + return str end return nil end @@ -1269,12 +1270,12 @@ local get_size_info = function (metadata) return false end -local organize_namedata = function (metadata, basename, info) +local get_english_names = function (names, basename) local english_names - if metadata.names then - for _, raw_namedata in next, metadata.names do + if names then + for _, raw_namedata in next, names do if raw_namedata.lang == "English (US)" then english_names = raw_namedata.names end @@ -1287,23 +1288,48 @@ local organize_namedata = function (metadata, basename, info) return nil end + return english_names +end + +local organize_namedata = function (metadata, + english_names, + basename, + info) + + --print (english_names.family, "<>", english_names.preffamilyname) local fontnames = { --- see --- https://developer.apple.com/fonts/TTRefMan/RM06/Chap6name.html + --- http://www.microsoft.com/typography/OTSPEC/name.htm#NameIDs english = { + --- where a “compatfull” field is given, the value of “fullname” is + --- either identical or differs by separating the style + --- with a hyphen and omitting spaces. + --- Of the three “fullname” fields, this one appears to be the one + --- with the entire name given in a legible, + --- non-abbreviated fashion, for most fonts at any rate. + --- However, in some fonts (e.g. CMU) all three fields are + --- identical. fullname = english_names.compatfull or english_names.fullname, - family = english_names.preffamilyname - or english_names.family, - --prefmodifiers = english_names.prefmodifiers, --> style - --subfamily = english_names.subfamily, --> style + --- we keep both the “preferred family” and the “family” + --- values around since both are valid but can turn out + --- quite differently, e.g. with Latin Modern: + --- preffamily: “Latin Modern Sans”, + --- family: “LM Sans 10” + preffamily = english_names.preffamilyname, + family = english_names.family, + prefmodifiers = english_names.prefmodifiers, + subfamily = english_names.subfamily, psname = english_names.postscriptname, }, + metadata = { - pfullname = metadata.fullname, + fullname = metadata.fullname, fontname = metadata.fontname, metafamily = metadata.familyname, }, + info = { fullname = info.fullname, familyname = info.familyname, @@ -1311,13 +1337,24 @@ local organize_namedata = function (metadata, basename, info) }, } - -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size +-- print (fontnames.english.fullname, +-- fontnames.metadata.fullname, +-- fontnames.info.fullname) + if metadata.fontstyle_name then + --- not present in all fonts, often differs from the preferred + --- subfamily as well as subfamily fields, e.g. with LMSans10-BoldOblique: + --- subfamily: “Bold Italic” + --- prefmodifiers: “10 Bold Oblique” + --- fontstyle_name: “Bold Oblique” for _, name in next, metadata.fontstyle_name do if name.lang == 1033 then --- I hate magic numbers fontnames.fontstyle_name = name.name end end + + --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", english_names.prefmodifiers) + --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", fontnames.fontstyle_name) end return { @@ -1329,6 +1366,32 @@ local organize_namedata = function (metadata, basename, info) end +local organize_styledata = function (metadata, english_names, info) + local pfminfo = metadata.pfminfo + local names = metadata.names + +-- print (">", pfminfo.avgwidth, +-- (metadata.italicangle == info.italicangle) and "T" or +-- string.format ("%f / %f", metadata.italicangle, info.italicangle), +-- pfminfo.width, pfminfo.weight, info.weight) + return { + -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size + size = get_size_info (metadata), + weight = { + pfminfo.weight, -- integer (multiple of 100?) + sanitize_fontname (info.weight), -- style name + }, + width = pfminfo.width, + italicangle = metadata.italicangle, +-- italicangle = { +-- metadata.italicangle, -- float +-- info.italicangle, -- truncated to integer point size? +-- }, + --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details + units_per_em = metadata.units_per_em, + version = metadata.version, + } +end --[[doc-- The data inside an Opentype font file can be quite heterogeneous. @@ -1344,25 +1407,20 @@ ot_fullinfo = function (filename, basename, format, info) - local styles = { } local metadata = load_font_file (filename, subfont) if not metadata then return nil end - local namedata = organize_namedata (metadata, basename, info) - - - local style = { - size = get_size_info (metadata), - weight = metadata.pfminfo.weight, - width = metadata.pfminfo.width, - slant = metadata.italicangle, - --- this is for querying, see www.ntg.nl/maps/40/07.pdf for details - units_per_em = metadata.units_per_em, - version = metadata.version, - } + local english_names = get_english_names (metadata.names, basename) + local namedata = organize_namedata (metadata, + english_names, + basename, + info) + local style = organize_styledata (metadata, + english_names, + info) return { file = { base = basename, diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index c4670e8..1ef74fb 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -172,7 +172,7 @@ local status_file = "luaotfload-status" local luaotfloadstatus = require (status_file) config.luaotfload.status = luaotfloadstatus -local sanitize_string = names.sanitize_string +local sanitize_fontname = names.sanitize_fontname local pathdata = names.path local names_plain = pathdata.index.lua @@ -656,7 +656,7 @@ subfont_by_name = function (lst, askedname, n) local font = lst[n] if font then - if sanitize_string(font.fullname) == askedname then + if sanitize_fontname (font.fullname) == askedname then return font end return subfont_by_name (lst, askedname, n+1) @@ -676,7 +676,7 @@ local show_font_info = function (basename, askedname, detail, warnings) local filenames = names.data.filenames local index = filenames.base[basename] local fullname = filenames.full[index] - askedname = sanitize_string(askedname) + askedname = sanitize_fontname (askedname) if not fullname then -- texmf fullname = resolvers.findfile(basename) end -- cgit v1.2.3 From 6ad39d6f1f45b088173312493f7b3b42eb71979a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 21:02:18 +0200 Subject: [db] add post-build statistics --- luaotfload-database.lua | 148 ++++++++++++++++++++++++++++++++++++++++++++---- luaotfload-override.lua | 6 +- 2 files changed, 141 insertions(+), 13 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index b331629..475d361 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1304,7 +1304,8 @@ local organize_namedata = function (metadata, english = { --- where a “compatfull” field is given, the value of “fullname” is --- either identical or differs by separating the style - --- with a hyphen and omitting spaces. + --- with a hyphen and omitting spaces. (According to the + --- spec, “compatfull” is “Macintosh only”.) --- Of the three “fullname” fields, this one appears to be the one --- with the entire name given in a legible, --- non-abbreviated fashion, for most fonts at any rate. @@ -1547,7 +1548,7 @@ local compare_timestamps = function (fullname, if targetentrystatus ~= nil and targetentrystatus.timestamp == targettimestamp then - report ("log", 3, "db", "Font %q already read.", basename) + report ("log", 3, "db", "Font %q already read.", fullname) return false end @@ -1570,7 +1571,7 @@ local compare_timestamps = function (fullname, targetentrystatus.index [targetindex + 1] = location end - report ("log", 3, "db", "Font %q already indexed.", basename) + report ("log", 3, "db", "Font %q already indexed.", fullname) return false end @@ -2034,22 +2035,22 @@ end local scan_dir = function (dirname, currentnames, targetnames, dry_run, location) if lpegmatch (p_blacklist, dirname) then - report ("both", 3, "db", + report ("both", 4, "db", "Skipping blacklisted directory %s", dirname) --- ignore return 0, 0 end local found = find_font_files (dirname, location ~= "texmf") if not found then - report ("both", 3, "db", + report ("both", 4, "db", "No such directory: %q; skipping.", dirname) return 0, 0 end - report ("both", 3, "db", "Scanning directory %s", dirname) + report ("both", 4, "db", "Scanning directory %s", dirname) local n_new = 0 --- total of fonts collected local n_found = #found - report ("both", 4, "db", "%d font files detected", n_found) + report ("both", 5, "db", "%d font files detected", n_found) for j=1, n_found do local fullname = found[j] fullname = path_normalize(fullname) @@ -2059,7 +2060,7 @@ local scan_dir = function (dirname, currentnames, targetnames, "Would have been extracting metadata from %q", fullname) else - report ("both", 4, "db", + report ("both", 5, "db", "Extracting metadata from font %q", fullname) local new = read_font_names (fullname, currentnames, targetnames, location) @@ -2069,7 +2070,7 @@ local scan_dir = function (dirname, currentnames, targetnames, end end - report("both", 4, "db", "%d fonts found in '%s'", n_found, dirname) + report("both", 5, "db", "%d fonts found in '%s'", n_found, dirname) return n_found, n_new end @@ -2488,7 +2489,7 @@ local generate_filedata = function (mappings) if inbase then local present = inbase [basename] if present then - report ("both", 3, "db", + report ("both", 4, "db", "Conflicting basename: %q already indexed \z in category %s, ignoring.", barename, location) @@ -2518,7 +2519,7 @@ local generate_filedata = function (mappings) if inbare then local present = inbare [barename] if present then - report ("both", 3, "db", + report ("both", 4, "db", "Conflicting barename: %q already indexed \z in category %s/%s, ignoring.", barename, location, format) @@ -2559,6 +2560,7 @@ local generate_filedata = function (mappings) return filenames end + local retrieve_namedata = function (currentnames, targetnames, dry_run, @@ -2579,6 +2581,129 @@ local retrieve_namedata = function (currentnames, return n_rawnames, n_newnames end + +--- dbobj -> stats + +local collect_statistics = function (mappings) + local sum_dsnsize, n_dsnsize = 0, 0 + + local fullname, family = { }, { } + local subfamily, prefmodifiers, fontstyle_name = { }, { }, { } + + local addtohash = function (hash, item) + if item then + local times = hash [item] + if times then + hash [item] = times + 1 + else + hash [item] = 1 + end + end + end + + local setsize = function (set) + local n = 0 + for _, _ in next, set do + n = n + 1 + end + return n + end + + local hashsum = function (hash) + local n = 0 + for _, m in next, hash do + n = n + m + end + return n + end + + for _, entry in next, mappings do + local style = entry.style + local names = entry.names.sanitized + local englishnames = names.english + + addtohash (fullname, englishnames.fullname) + addtohash (family, englishnames.family) + addtohash (subfamily, englishnames.subfamily) + addtohash (prefmodifiers, englishnames.prefmodifiers) + addtohash (fontstyle_name, names.fontstyle_name) + + local sizeinfo = entry.style.size + if sizeinfo then + sum_dsnsize = sum_dsnsize + sizeinfo [1] + n_dsnsize = n_dsnsize + 1 + end + end + + local n_fullname = setsize (fullname) + local n_family = setsize (family) + + if logs.get_loglevel () > 1 then + report ("both", 0, "", "~~~~ font index statistics ~~~~") + report ("both", 0, "db", + " · Collected %d fonts (%d names) in %d families.", + #mappings, n_fullname, n_family) + + local pprint_top = function (hash, n) + local freqs = { } + local items = { } + for item, freq in next, hash do + local ifreq = items [freq] + if ifreq then + ifreq [#ifreq + 1] = item + else + items [freq] = { item } + freqs [#freqs + 1] = freq + end + end + + tablesort (freqs) + + local from = #freqs + local to = from - (n - 1) + if to < 1 then + to = 1 + end + + for i = from, to, -1 do + local freq = freqs [i] + local itemlist = items [freq] + report ("both", 0, "db", + " · %4d × %s.", + freq, tableconcat (itemlist, ", ")) + end + end + + report ("both", 0, "db", + " · %d different “subfamily” kinds", + setsize (subfamily)) + pprint_top (subfamily, 4) + + report ("both", 0, "db", + " · %d different “prefmodifiers” kinds", + setsize (prefmodifiers)) + pprint_top (prefmodifiers, 4) + + report ("both", 0, "db", + " · %d different “fontstyle_name” kinds", + setsize (fontstyle_name)) + pprint_top (fontstyle_name, 4) + end + + return { + mean_dsnsize = sum_dsnsize / n_dsnsize, + names = { + fullname = n_fullname, + families = n_family, + }, +-- style = { +-- subfamily = subfamily, +-- prefmodifiers = prefmodifiers, +-- fontstyle_name = fontstyle_name, +-- }, + } +end + --- force: dictate rebuild from scratch --- dry_dun: don’t write to the db, just scan dirs @@ -2625,6 +2750,7 @@ update_names = function (currentnames, force, dry_run) dry_run, n_rawnames, n_newnames) + targetnames.meta.statistics = collect_statistics (targetnames.mappings) --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping diff --git a/luaotfload-override.lua b/luaotfload-override.lua index 8e467d5..0609541 100644 --- a/luaotfload-override.lua +++ b/luaotfload-override.lua @@ -144,8 +144,10 @@ logs.set_logout = set_logout local log = function (category, fmt, ...) local res = { module_name, "|", category, ":" } - if fmt then res[#res+1] = stringformat(fmt, ...) end - texiowrite_nl(logout, tableconcat(res, " ")) + if fmt then + res [#res + 1] = stringformat (fmt, ...) + end + texiowrite_nl (logout, tableconcat(res, " ")) end --- with faux db update with maximum verbosity: -- cgit v1.2.3 From df1267d2bea13c84056e5a30465d24499c595558 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:09:23 +0200 Subject: [db] adapt function reload_db to new db structure --- luaotfload-database.lua | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 475d361..7026dca 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1107,17 +1107,18 @@ end --- string -> ('a -> 'a) -> 'a list -> 'a reload_db = function (why, caller, ...) local namedata = names.data - local formats = tableconcat (namedata.formats, ",") + local formats = tableconcat (namedata.meta.formats, ",") report ("both", 1, "db", "Reload initiated (formats: %s); reason: %q", formats, why) set_font_filter (formats) - names.data = update_names (names.data, false, false) + namedata = update_names (namedata, false, false) - if names.data then + if namedata then fonts_reloaded = true + names.data = namedata return caller (...) end -- cgit v1.2.3 From 6c4b470d23faf61e4f9b0aab5f871bf87b6707bb Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:10:32 +0200 Subject: [db] fix integer format string --- luaotfload-database.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 7026dca..e77d4fd 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1786,7 +1786,7 @@ local create_blacklist = function (blacklist, whitelist) local result = { } local dirs = { } - report("info", 2, "db", "Blacklisting %q files and directories", + report("info", 2, "db", "Blacklisting %d files and directories", #blacklist) for i=1, #blacklist do local entry = blacklist[i] @@ -1797,7 +1797,7 @@ local create_blacklist = function (blacklist, whitelist) end end - report("info", 2, "db", "Whitelisting %q files", #whitelist) + report("info", 2, "db", "Whitelisting %d files", #whitelist) for i=1, #whitelist do result[whitelist[i]] = nil end -- cgit v1.2.3 From 7163d50ce6e5deaefaa32f9a6d0bbcc778fcb24b Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:14:31 +0200 Subject: [db] intercept possible division by zero --- luaotfload-database.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e77d4fd..bae90c3 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2691,8 +2691,13 @@ local collect_statistics = function (mappings) pprint_top (fontstyle_name, 4) end + local mean_dsnsize = 0 + if n_dsnsize > 0 then + mean_dsnsize = sum_dsnsize / n_dsnsize + end + return { - mean_dsnsize = sum_dsnsize / n_dsnsize, + mean_dsnsize = mean_dsnsize, names = { fullname = n_fullname, families = n_family, -- cgit v1.2.3 From 8bd26bbc573d4320fce8d6535737dd25e277bd2f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:45:12 +0200 Subject: [db] print family sizes along with stats --- luaotfload-database.lua | 55 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bae90c3..8268283 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2588,7 +2588,7 @@ end local collect_statistics = function (mappings) local sum_dsnsize, n_dsnsize = 0, 0 - local fullname, family = { }, { } + local fullname, family, families = { }, { }, { } local subfamily, prefmodifiers, fontstyle_name = { }, { }, { } local addtohash = function (hash, item) @@ -2602,6 +2602,28 @@ local collect_statistics = function (mappings) end end + local appendtohash = function (hash, key, value) + if key and value then + local entry = hash [key] + if entry then + entry [#entry + 1] = value + else + hash [key] = { value } + end + end + end + + local addtoset = function (hash, key, value) + if key and value then + local set = hash [key] + if set then + set [value] = true + else + hash [key] = { [value] = true } + end + end + end + local setsize = function (set) local n = 0 for _, _ in next, set do @@ -2629,6 +2651,8 @@ local collect_statistics = function (mappings) addtohash (prefmodifiers, englishnames.prefmodifiers) addtohash (fontstyle_name, names.fontstyle_name) + addtoset (families, englishnames.family, englishnames.fullname) + local sizeinfo = entry.style.size if sizeinfo then sum_dsnsize = sum_dsnsize + sizeinfo [1] @@ -2636,19 +2660,23 @@ local collect_statistics = function (mappings) end end + --inspect (families) + local n_fullname = setsize (fullname) local n_family = setsize (family) if logs.get_loglevel () > 1 then - report ("both", 0, "", "~~~~ font index statistics ~~~~") - report ("both", 0, "db", - " · Collected %d fonts (%d names) in %d families.", - #mappings, n_fullname, n_family) + local pprint_top = function (hash, n, set) - local pprint_top = function (hash, n) local freqs = { } local items = { } - for item, freq in next, hash do + + for item, value in next, hash do + if set then + freq = setsize (value) + else + freq = value + end local ifreq = items [freq] if ifreq then ifreq [#ifreq + 1] = item @@ -2669,12 +2697,23 @@ local collect_statistics = function (mappings) for i = from, to, -1 do local freq = freqs [i] local itemlist = items [freq] + + if type (itemlist) == "table" then + itemlist = tableconcat (itemlist, ", ") + end + report ("both", 0, "db", " · %4d × %s.", - freq, tableconcat (itemlist, ", ")) + freq, itemlist) end end + report ("both", 0, "", "~~~~ font index statistics ~~~~") + report ("both", 0, "db", + " · Collected %d fonts (%d names) in %d families.", + #mappings, n_fullname, n_family) + pprint_top (families, 4, true) + report ("both", 0, "db", " · %d different “subfamily” kinds", setsize (subfamily)) -- cgit v1.2.3 From e29c39733ad90a0ca354808aef71b1b2e0acc9c9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 1 Sep 2013 23:49:14 +0200 Subject: [tool] include stats on demand only (--stats option) --- luaotfload-database.lua | 6 +++++- luaotfload-tool.lua | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 8268283..a148940 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2795,7 +2795,11 @@ update_names = function (currentnames, force, dry_run) dry_run, n_rawnames, n_newnames) - targetnames.meta.statistics = collect_statistics (targetnames.mappings) + + if config.luaotfload.statistics == true then + targetnames.meta.statistics = collect_statistics + (targetnames.mappings) + end --- we always generate the file lookup tables because --- non-texmf entries are redirected there and the mapping diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 1ef74fb..9739e17 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -1068,6 +1068,7 @@ local process_cmdline = function ( ) -- unit -> jobspec fields = 1, find = 1, force = "f", + formats = 1, fuzzy = "F", help = "h", info = "i", @@ -1079,14 +1080,14 @@ local process_cmdline = function ( ) -- unit -> jobspec ["prefer-texmf"] = "p", quiet = "q", ["show-blacklist"] = "b", - formats = 1, + stats = "S", update = "u", verbose = 1, version = "V", warnings = "w", } - local short_options = "bDfFiIlnpquvVhw" + local short_options = "bDfFiIlnpqSuvVhw" local options, _, optarg = alt_getopt.get_ordered_opts (arg, short_options, long_options) @@ -1156,7 +1157,7 @@ local process_cmdline = function ( ) -- unit -> jobspec result.cache = optarg[n] elseif v == "D" then result.dry_run = true - elseif v == "p" then + elseif v == "p" then --- TODO adapt to new db structure config.luaotfload.prioritize = "texmf" elseif v == "b" then action_pending["blacklist"] = true @@ -1167,6 +1168,8 @@ local process_cmdline = function ( ) -- unit -> jobspec names.set_font_filter (optarg[n]) elseif v == "n" then config.luaotfload.update_live = false + elseif v == "S" then + config.luaotfload.statistics = true end end -- cgit v1.2.3 From 1674a5aa95a3305845364daee05c2ec8279e2703 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 2 Sep 2013 00:07:25 +0200 Subject: [db] adjust index field for non tt collections --- luaotfload-database.lua | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index a148940..e5627ef 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1559,7 +1559,6 @@ local compare_timestamps = function (fullname, if currenttimestamp == targettimestamp and not targetentrystatus.index [1] then - --- copy old namedata into new for _, currentindex in next, currententrystatus.index do @@ -1590,7 +1589,13 @@ local insert_fullinfo = function (fullname, targetentrystatus, info) - local subfont = n_font and n_font - 1 or false + local subfont + if n_font ~= false then + subfont = n_font - 1 + else + subfont = false + n_font = 1 + end local fullinfo = loader (fullname, subfont, location, basename, @@ -2567,6 +2572,7 @@ local retrieve_namedata = function (currentnames, dry_run, n_rawnames, n_newnames) + local rawnames, new = scan_texmf_fonts (currentnames, targetnames, dry_run) -- cgit v1.2.3 From 18f6b49c8c5903926e899abaf8efdada6947fbde Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 2 Sep 2013 00:09:47 +0200 Subject: [db] fix table constructor syntax --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e5627ef..323804c 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2544,7 +2544,7 @@ local generate_filedata = function (mappings) inbare [barename] = index end else - inbare = { barename = index } + inbare = { [barename] = index } bare [location] [format] = inbare end -- cgit v1.2.3 From 0fe0554e033e1f87c4bf031b37791d0bd2f435e0 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 11 Sep 2013 13:30:30 +0200 Subject: [db] prepare families table --- luaotfload-database.lua | 222 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 212 insertions(+), 10 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 323804c..2b98d92 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -60,7 +60,6 @@ local utf8gsub = unicode.utf8.gsub local utf8lower = unicode.utf8.lower --- these come from Lualibs/Context -local getwritablepath = caches.getwritablepath local filebasename = file.basename local filecollapsepath = file.collapsepath or file.collapse_path local filedirname = file.dirname @@ -71,9 +70,11 @@ local filenameonly = file.nameonly local filereplacesuffix = file.replacesuffix local filesplitpath = file.splitpath or file.split_path local filesuffix = file.suffix +local getwritablepath = caches.getwritablepath local lfsisdir = lfs.isdir local lfsisfile = lfs.isfile local lfsmkdirs = lfs.mkdirs +local lpegsplitat = lpeg.splitat local stringis_empty = string.is_empty local stringsplit = string.split local stringstrip = string.strip @@ -375,11 +376,7 @@ mtx-fonts has in names.tma: local initialize_namedata = function (formats) --- returns dbobj return { - families = { - ["local"] = { }, - system = { }, - texmf = { }, - }, + --families = { }, status = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite names = { }, @@ -1368,7 +1365,20 @@ local organize_namedata = function (metadata, end -local organize_styledata = function (metadata, english_names, info) + +local dashsplitter = lpegsplitat "-" + +local split_fontname = function (fontname) + --- sometimes the style hides in the latter part of the + --- fontname, separated by a dash, e.g. “Iwona-Regular”, + --- “GFSSolomos-Regular” + local splitted = { lpegmatch (dashsplitter, fontname) } + if splitted then + return sanitize_fontname (splitted [#splitted]) + end +end + +local organize_styledata = function (fontname, metadata, english_names, info) local pfminfo = metadata.pfminfo local names = metadata.names @@ -1383,6 +1393,7 @@ local organize_styledata = function (metadata, english_names, info) pfminfo.weight, -- integer (multiple of 100?) sanitize_fontname (info.weight), -- style name }, + split = split_fontname (fontname), width = pfminfo.width, italicangle = metadata.italicangle, -- italicangle = { @@ -1420,7 +1431,8 @@ ot_fullinfo = function (filename, english_names, basename, info) - local style = organize_styledata (metadata, + local style = organize_styledata (namedata.fontname, + metadata, english_names, info) @@ -1432,7 +1444,6 @@ ot_fullinfo = function (filename, format = format, names = namedata, style = style, - units_per_em = metadata.units_per_em, version = metadata.version, } end @@ -2566,6 +2577,196 @@ local generate_filedata = function (mappings) return filenames end +local match_synonyms = function (pattern) + local nopattern = 1 - pattern + return nopattern^0 * pattern * Cc (true) + Cc (false) +end + +local determine_italic +local determine_bold +local pick_style +local check_regular + +do + local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") + local bold = match_synonyms (P"bold" + P"demi", P"heavy", P"black", P"ultra") + + determine_italic = function (fontstyle_name, + italicangle, + prefmodifiers, + subfamily) + if italicangle ~= 0 then + return true + elseif fontstyle_name and lpegmatch (italic, fontstyle_name) then + return true + elseif prefmodifiers and lpegmatch (italic, prefmodifiers) then + return lpegmatch (italic, prefmodifiers) + else + return lpegmatch (italic, subfamily) + end + end + + determine_bold = function (fontstyle_name, + weight, + prefmodifiers, + subfamily) + if weight [2] == "bold" then + return true + elseif fontstyle_name and lpegmatch (bold, fontstyle_name) then + return true + elseif prefmodifiers and lpegmatch (bold, prefmodifiers) then + return true + else + return lpegmatch (bold, subfamily) + end + end + + local splitfontname = lpeg.splitat "-" + + local choose_exact = function (field) + if field == "italic" or field == "oblique" then + return "i" + elseif field == "bold" then + return "b" + elseif field == "bolditalic" or field == "boldoblique" then + return "bi" + end + + return false + end + + pick_style = function (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + local style + if fontstyle_name ~= nil then + style = choose_exact (prefmodifiers) + end + if not style and prefmodifiers ~= nil then + style = choose_exact (prefmodifiers) + end + if not style then + style = choose_exact (subfamily) + end + if not style and splitstyle ~= nil then + choose_exact (splitstyle) + end + return style + end + + --- we use only exact matches here since there are constructs + --- like “regularitalic” (Cabin, Bodoni Old Fashion) + + check_regular = function (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + + if fontstyle_name == "regular" or fontstyle_name == "book" then + return "r" + end + + if prefmodifiers == "regular" or prefmodifiers == "book" then + return "r" + end + + if subfamily == "regular" or subfamily == "book" then + return "r" + end + + if splitstyle == "regular" or splitstyle == "book" then + return "r" + end + + return nil + end +end + +local collect_families = function (mappings) + + local families = { + ["local"] = { }, + system = { }, + texmf = { }, + } + + for i = 1, #mappings do + + local entry = mappings [i] + local file = entry.file + local names = entry.names + local style = entry.style + + local location = file.location + local format = entry.format + local english = names.sanitized.english + local info = names.sanitized.info + + local subtable = families [location] [format] + if not subtable then + subtable = { } + families [location] [format] = subtable + end + + local familyname = english.preffamily + or english.family + or info.familyname + local fullname = english.fullname or info.fullname + local fontname = info.fontname + + local fontstyle_name = names.sanitized.fontstyle_name + local prefmodifiers = english.prefmodifiers or "regular" + local subfamily = english.subfamily + + local weight = style.weight + local italicangle = style.italicangle + local splitstyle = style.split + + local modifier = pick_style (fontstyle_name, + prefmodifiers, + subfamily, + splitstyle) + + if not modifier then --- guess + local italic = determine_italic (fontstyle_name, + italicangle, + prefmodifiers, + subfamily) + local bold = determine_bold (fontstyle_name, + weight, + prefmodifiers, + subfamily) + if bold and italic then + modifier = "bi" + elseif bold then + modifier = "b" + elseif italic then + modifier = "i" + end + end + + if not modifier then --- regular, exact only + modifier = check_regular (fontstyle_name, + subfamily, + splitstyle) + end + + if modifier then + --- stub; here we will continue building a list of optical sizes + --- no size -> hash “normal” + --- other sizes -> indexed tuples { dsnsize, idx } + local familytable = subtable [familyname] + if not familytable then + familytable = { } + subtable [familyname] = familytable + end + familytable [modifier] = entry.index + end + end + + return families +end local retrieve_namedata = function (currentnames, targetnames, @@ -2811,7 +3012,8 @@ update_names = function (currentnames, force, dry_run) --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent - targetnames.files = generate_filedata (targetnames.mappings) + targetnames.files = generate_filedata (targetnames.mappings) + targetnames.families = collect_families (targetnames.mappings) --- stats: --- before rewrite | after rewrite -- cgit v1.2.3 From b3fa18a7d99d55d2daaeebd7f17db664719ef690 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 13 Sep 2013 23:27:55 +0200 Subject: [fontloader] sync with Context as of 2013-09-13 --- luaotfload-database.lua | 2 +- luaotfload-merged.lua | 57 +++++++++++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 22 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 2b98d92..5032955 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1373,7 +1373,7 @@ local split_fontname = function (fontname) --- fontname, separated by a dash, e.g. “Iwona-Regular”, --- “GFSSolomos-Regular” local splitted = { lpegmatch (dashsplitter, fontname) } - if splitted then + if next (splitted) then return sanitize_fontname (splitted [#splitted]) end end diff --git a/luaotfload-merged.lua b/luaotfload-merged.lua index 649c537..3c63ba0 100644 --- a/luaotfload-merged.lua +++ b/luaotfload-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 08/26/13 16:41:50 +-- merge date : 09/13/13 10:59:07 do -- begin closure to overcome local limits and interference @@ -127,15 +127,22 @@ local dquote=P('"') local space=P(" ") local period=P(".") local comma=P(",") -local utfbom_32_be=P('\000\000\254\255') -local utfbom_32_le=P('\255\254\000\000') -local utfbom_16_be=P('\255\254') -local utfbom_16_le=P('\254\255') -local utfbom_8=P('\239\187\191') +local utfbom_32_be=P('\000\000\254\255') +local utfbom_32_le=P('\255\254\000\000') +local utfbom_16_be=P('\254\255') +local utfbom_16_le=P('\255\254') +local utfbom_8=P('\239\187\191') local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) local utf8next=R("\128\191") +patterns.utfbom_32_be=utfbom_32_be +patterns.utfbom_32_le=utfbom_32_le +patterns.utfbom_16_be=utfbom_16_be +patterns.utfbom_16_le=utfbom_16_le +patterns.utfbom_8=utfbom_8 +patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") +patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") patterns.utf8one=R("\000\127") patterns.utf8two=R("\194\223")*utf8next patterns.utf8three=R("\224\239")*utf8next*utf8next @@ -3770,7 +3777,7 @@ function constructors.scale(tfmdata,specification) targetproperties.script=properties.script or "dflt" targetproperties.mode=properties.mode or "base" local askedscaledpoints=scaledpoints - local scaledpoints,delta=constructors.calculatescale(tfmdata,scaledpoints) + local scaledpoints,delta=constructors.calculatescale(tfmdata,scaledpoints,nil,specification) local hdelta=delta local vdelta=delta target.designsize=parameters.designsize @@ -4286,11 +4293,11 @@ function constructors.hashinstance(specification,force) size=math.round(constructors.scaled(size,designsizes[hash])) specification.size=size end - if fallbacks then - return hash..' @ '..tostring(size)..' @ '..fallbacks - else - return hash..' @ '..tostring(size) - end + if fallbacks then + return hash..' @ '..tostring(size)..' @ '..fallbacks + else + return hash..' @ '..tostring(size) + end end function constructors.setname(tfmdata,specification) if constructors.namemode=="specification" then @@ -5705,7 +5712,7 @@ local function copytotfm(data) parameters.space_shrink=333 parameters.x_height=400 parameters.quad=1000 - if italicangle then + if italicangle and italicangle~=0 then parameters.italicangle=italicangle parameters.italicfactor=math.cos(math.rad(90+italicangle)) parameters.slant=- math.tan(italicangle*math.pi/180) @@ -7996,7 +8003,7 @@ local function copytotfm(data,cache_id) parameters.quad=units if spaceunits<2*units/5 then end - if italicangle then + if italicangle and italicangle~=0 then parameters.italicangle=italicangle parameters.italicfactor=math.cos(math.rad(90+italicangle)) parameters.slant=- math.tan(italicangle*math.pi/180) @@ -8829,7 +8836,7 @@ function injections.setkern(current,factor,rlmode,x,tfmchr) return 0,0 end end -function injections.setmark(start,base,factor,rlmode,ba,ma,index) +function injections.setmark(start,base,factor,rlmode,ba,ma,index,baseismark) local dx,dy=factor*(ba[1]-ma[1]),factor*(ba[2]-ma[2]) local bound=getattr(base,a_markbase) local index=1 @@ -8850,7 +8857,7 @@ function injections.setmark(start,base,factor,rlmode,ba,ma,index) setattr(base,a_markbase,bound) setattr(start,a_markmark,bound) setattr(start,a_markdone,index) - marks[bound]={ [index]={ dx,dy,rlmode } } + marks[bound]={ [index]={ dx,dy,rlmode,baseismark } } return dx,dy,bound end local function dir(n) @@ -8971,7 +8978,7 @@ function injections.handler(head,where,keep) local f=getfont(n) if f~=nf then nf=f - tm=fontdata[nf].resources.marks + tm=fontdata[nf].resources.marks end if tm then mk[n]=tm[getchar(n)] @@ -9084,11 +9091,17 @@ function injections.handler(head,where,keep) end end else + local wp=getfield(p,"width") + local wn=getfield(n,"width") if rlmode and rlmode>=0 then - ox=px-getfield(p,"width")+d[1] + ox=px-wp+d[1] else ox=px-d[1] end + if wn~=0 then + insert_node_before(head,n,newkern(-wn/2)) + insert_node_after(head,n,newkern(-wn/2)) + end end setfield(n,"xoffset",ox) local py=getfield(p,"yoffset") @@ -9105,6 +9118,8 @@ function injections.handler(head,where,keep) nofmarks=nofmarks-1 end end + elseif not n_markmark then + break else end end @@ -10176,7 +10191,7 @@ function handlers.gpos_mark2mark(head,start,kind,lookupname,markanchors,sequence if al[anchor] then local ma=markanchors[anchor] if ma then - local dx,dy,bound=setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) + local dx,dy,bound=setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma,true) if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to basemark %s => (%p,%p)", pref(kind,lookupname),anchor,bound,gref(markchar),gref(basechar),dx,dy) @@ -10717,7 +10732,7 @@ function chainprocs.gpos_mark2mark(head,start,stop,kind,chainname,currentcontext if al[anchor] then local ma=markanchors[anchor] if ma then - local dx,dy,bound=setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma) + local dx,dy,bound=setmark(start,base,tfmdata.parameters.factor,rlmode,ba,ma,true) if trace_marks then logprocess("%s, anchor %s, bound %s: anchoring mark %s to basemark %s => (%p,%p)", cref(kind,chainname,chainlookupname,lookupname),anchor,bound,gref(markchar),gref(basechar),dx,dy) @@ -12667,7 +12682,7 @@ addlookup("file") addlookup("name") addlookup("spec") local function getspecification(str) - return lpegmatch(splitter,str) + return lpegmatch(splitter,str or "") end definers.getspecification=getspecification function definers.registersplit(symbol,action,verbosename) -- cgit v1.2.3 From cd5707a41b61198d0a53ab7cd4acbcc9240d0487 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 15 Sep 2013 22:12:42 +0200 Subject: [db] handle design sizes --- luaotfload-database.lua | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 5032955..d3f2f08 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2457,7 +2457,7 @@ end local generate_filedata = function (mappings) - report ("both", 2, "db", "Creating filename map") + report ("both", 2, "db", "Creating filename map.") local nmappings = #mappings @@ -2685,6 +2685,8 @@ end local collect_families = function (mappings) + report ("info", 2, "db", "Analyzing families, sizes, and styles.") + local families = { ["local"] = { }, system = { }, @@ -2716,7 +2718,7 @@ local collect_families = function (mappings) local fontname = info.fontname local fontstyle_name = names.sanitized.fontstyle_name - local prefmodifiers = english.prefmodifiers or "regular" + local prefmodifiers = english.prefmodifiers local subfamily = english.subfamily local weight = style.weight @@ -2755,13 +2757,35 @@ local collect_families = function (mappings) if modifier then --- stub; here we will continue building a list of optical sizes --- no size -> hash “normal” - --- other sizes -> indexed tuples { dsnsize, idx } + --- other sizes -> indexed tuples { dsnsize, high, low, idx } local familytable = subtable [familyname] if not familytable then familytable = { } subtable [familyname] = familytable end - familytable [modifier] = entry.index + + --- the style table is treated as an unordered list + local styletable = familytable [modifier] + if not styletable then + styletable = { } + familytable [modifier] = styletable + end + + if not prefmodifiers then --- default size for this style/family combo + styletable.default = entry.index + end + + local size = style.size --- dsnsize * high * low + if size then + styletable [#styletable + 1] = { + size [1], + size [2], + size [3], + entry.index, + } + else + styletable.default = entry.index + end end end -- cgit v1.2.3 From 9ab1917e9b492abaef43a9d9a45ceba23391003f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 15 Sep 2013 22:46:21 +0200 Subject: [tool] add option to skip reading font files --- luaotfload-database.lua | 229 +++++++++++++++++++++++++----------------------- luaotfload-tool.lua | 43 +++++---- 2 files changed, 143 insertions(+), 129 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index d3f2f08..bb7f17e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -20,101 +20,100 @@ local C, Cc, Cf, Cg, Cs, Ct = lpeg.C, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Cs, lpeg.Ct --- Luatex builtins -local load = load -local next = next -local pcall = pcall -local require = require -local tonumber = tonumber -local unpack = table.unpack - -local fontloaderinfo = fontloader.info -local fontloaderclose = fontloader.close -local fontloaderopen = fontloader.open -local fontloaderto_table = fontloader.to_table -local iolines = io.lines -local ioopen = io.open -local kpseexpand_path = kpse.expand_path -local kpseexpand_var = kpse.expand_var -local kpsefind_file = kpse.find_file -local kpselookup = kpse.lookup -local kpsereadable_file = kpse.readable_file -local lfsattributes = lfs.attributes -local lfschdir = lfs.chdir -local lfscurrentdir = lfs.currentdir -local lfsdir = lfs.dir -local mathabs = math.abs -local mathmin = math.min -local osgettimeofday = os.gettimeofday -local osremove = os.remove -local stringfind = string.find -local stringformat = string.format -local stringgmatch = string.gmatch -local stringgsub = string.gsub -local stringlower = string.lower -local stringsub = string.sub -local stringupper = string.upper -local tableconcat = table.concat -local tablesort = table.sort -local texiowrite_nl = texio.write_nl -local utf8gsub = unicode.utf8.gsub -local utf8lower = unicode.utf8.lower +local load = load +local next = next +local pcall = pcall +local require = require +local tonumber = tonumber +local unpack = table.unpack + +local fontloaderinfo = fontloader.info +local fontloaderclose = fontloader.close +local fontloaderopen = fontloader.open +local fontloaderto_table = fontloader.to_table +local iolines = io.lines +local ioopen = io.open +local kpseexpand_path = kpse.expand_path +local kpseexpand_var = kpse.expand_var +local kpsefind_file = kpse.find_file +local kpselookup = kpse.lookup +local kpsereadable_file = kpse.readable_file +local lfsattributes = lfs.attributes +local lfschdir = lfs.chdir +local lfscurrentdir = lfs.currentdir +local lfsdir = lfs.dir +local mathabs = math.abs +local mathmin = math.min +local osgettimeofday = os.gettimeofday +local osremove = os.remove +local stringfind = string.find +local stringformat = string.format +local stringgmatch = string.gmatch +local stringgsub = string.gsub +local stringlower = string.lower +local stringsub = string.sub +local stringupper = string.upper +local tableconcat = table.concat +local tablesort = table.sort +local texiowrite_nl = texio.write_nl +local utf8gsub = unicode.utf8.gsub +local utf8lower = unicode.utf8.lower --- these come from Lualibs/Context -local filebasename = file.basename -local filecollapsepath = file.collapsepath or file.collapse_path -local filedirname = file.dirname -local fileextname = file.extname -local fileiswritable = file.iswritable -local filejoin = file.join -local filenameonly = file.nameonly -local filereplacesuffix = file.replacesuffix -local filesplitpath = file.splitpath or file.split_path -local filesuffix = file.suffix -local getwritablepath = caches.getwritablepath -local lfsisdir = lfs.isdir -local lfsisfile = lfs.isfile -local lfsmkdirs = lfs.mkdirs -local lpegsplitat = lpeg.splitat -local stringis_empty = string.is_empty -local stringsplit = string.split -local stringstrip = string.strip -local tableappend = table.append -local tablecopy = table.copy -local tablefastcopy = table.fastcopy -local tabletofile = table.tofile -local tabletohash = table.tohash +local filebasename = file.basename +local filecollapsepath = file.collapsepath or file.collapse_path +local filedirname = file.dirname +local fileextname = file.extname +local fileiswritable = file.iswritable +local filejoin = file.join +local filenameonly = file.nameonly +local filereplacesuffix = file.replacesuffix +local filesplitpath = file.splitpath or file.split_path +local filesuffix = file.suffix +local getwritablepath = caches.getwritablepath +local lfsisdir = lfs.isdir +local lfsisfile = lfs.isfile +local lfsmkdirs = lfs.mkdirs +local lpegsplitat = lpeg.splitat +local stringis_empty = string.is_empty +local stringsplit = string.split +local stringstrip = string.strip +local tableappend = table.append +local tablecopy = table.copy +local tablefastcopy = table.fastcopy +local tabletofile = table.tofile +local tabletohash = table.tohash --- the font loader namespace is “fonts”, same as in Context --- we need to put some fallbacks into place for when running --- as a script -fonts = fonts or { } -fonts.names = fonts.names or { } -fonts.definers = fonts.definers or { } +fonts = fonts or { } +fonts.names = fonts.names or { } +fonts.definers = fonts.definers or { } -local names = fonts.names +local names = fonts.names -config = config or { } -config.luaotfload = config.luaotfload or { } -config.luaotfload.resolver = config.luaotfload.resolver or "normal" -config.luaotfload.formats = config.luaotfload.formats or "otf,ttf,ttc,dfont" +local luaotfloadconfig = config.luaotfload --- always present +luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" +luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" -if config.luaotfload.update_live ~= false then +if luaotfloadconfig.update_live ~= false then --- this option allows for disabling updates --- during a TeX run - config.luaotfload.update_live = true + luaotfloadconfig.update_live = true end -names.version = 2.4 -names.data = nil --- contains the loaded database -names.lookups = nil --- contains the lookup cache +names.version = 2.4 +names.data = nil --- contains the loaded database +names.lookups = nil --- contains the lookup cache -names.path = { index = { }, lookups = { } } -names.path.globals = { - prefix = "", --- writable_path/names_dir - names_dir = config.luaotfload.names_dir or "names", - index_file = config.luaotfload.index_file - or "luaotfload-names.lua", - lookups_file = "luaotfload-lookup-cache.lua", +names.path = { index = { }, lookups = { } } +names.path.globals = { + prefix = "", --- writable_path/names_dir + names_dir = luaotfloadconfig.names_dir or "names", + index_file = luaotfloadconfig.index_file + or "luaotfload-names.lua", + lookups_file = "luaotfload-lookup-cache.lua", } --- string -> (string * string) @@ -1937,7 +1936,7 @@ do end --- initialize - set_font_filter (config.luaotfload.formats) + set_font_filter (luaotfloadconfig.formats) end local process_dir_tree @@ -2565,7 +2564,7 @@ local generate_filedata = function (mappings) end --- TODO adapt to new mechanism! --- if config.luaotfload.prioritize == "texmf" then +-- if luaotfloadconfig.prioritize == "texmf" then -- report("both", 2, "db", "Preferring texmf fonts") -- addmap(sys) -- addmap(texmf) @@ -2986,7 +2985,9 @@ end --- dbobj? -> bool? -> bool? -> dbobj update_names = function (currentnames, force, dry_run) - if config.luaotfload.update_live == false then + local targetnames + + if luaotfloadconfig.update_live == false then report ("info", 2, "db", "Skipping database update") --- skip all db updates @@ -3004,30 +3005,43 @@ update_names = function (currentnames, force, dry_run) report ("both", 2, "db", "Updating the font names database" .. (force and " forcefully" or "")) - if force then - currentnames = initialize_namedata (get_font_filter ()) + if luaotfloadconfig.skip_read == true then + --- the difference to a “dry run” is that we don’t search + --- for font files entirely. we also ignore the “force” + --- parameter since it concerns only the font files. + report ("info", 2, "db", + "Ignoring font files, reusing old data.") + currentnames = load_names (false) + targetnames = currentnames else - if not currentnames then - currentnames = load_names (dry_run) - end - if currentnames.meta.version ~= names.version then - report ("both", 1, "db", "No font names database or old " - .. "one found; generating new one") + if force then currentnames = initialize_namedata (get_font_filter ()) + else + if not currentnames then + currentnames = load_names (dry_run) + end + if currentnames.meta.version ~= names.version then + report ("both", 1, "db", "No font names database or old " + .. "one found; generating new one") + currentnames = initialize_namedata (get_font_filter ()) + end end - end - local targetnames = initialize_namedata (get_font_filter ()) + targetnames = initialize_namedata (get_font_filter ()) - read_blacklist () + read_blacklist () - local n_rawnames, n_newnames = retrieve_namedata (currentnames, - targetnames, - dry_run, - n_rawnames, - n_newnames) + local n_raw, n_new= retrieve_namedata (currentnames, + targetnames, + dry_run, + n_rawnames, + n_newnames) + report ("info", 3, "db", + "Scanned %d font files; %d new entries.", + n_rawnames, n_newnames) + end - if config.luaotfload.statistics == true then + if luaotfloadconfig.statistics == true then targetnames.meta.statistics = collect_statistics (targetnames.mappings) end @@ -3039,13 +3053,6 @@ update_names = function (currentnames, force, dry_run) targetnames.files = generate_filedata (targetnames.mappings) targetnames.families = collect_families (targetnames.mappings) - --- stats: - --- before rewrite | after rewrite - --- partial: 804 ms | 701 ms - --- forced: 45384 ms | 44714 ms - report ("info", 3, "db", - "Scanned %d font files; %d new entries.", - n_rawnames, n_newnames) report ("info", 3, "db", "Rebuilt in %0.f ms.", 1000 * (osgettimeofday () - starttime)) @@ -3205,7 +3212,7 @@ end local getwritablecachepath = function ( ) --- fonts.handlers.otf doesn’t exist outside a Luatex run, --- so we have to improvise - local writable = getwritablepath (config.luaotfload.cache_dir) + local writable = getwritablepath (luaotfloadconfig.cache_dir) if writable then return writable end @@ -3213,7 +3220,7 @@ end local getreadablecachepaths = function ( ) local readables = caches.getreadablepaths - (config.luaotfload.cache_dir) + (luaotfloadconfig.cache_dir) local result = { } if readables then for i=1, #readables do @@ -3297,7 +3304,7 @@ names.erase_cache = erase_cache names.show_cache = show_cache --- replace the resolver from luatex-fonts -if config.luaotfload.resolver == "cached" then +if luaotfloadconfig.resolver == "cached" then report("both", 2, "cache", "caching of name: lookups active") names.resolve = resolve_cached names.resolvespec = resolve_cached diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 9739e17..e410d44 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -102,13 +102,16 @@ to be the more appropriate. config = config or { } local config = config -config.luaotfload = config.luaotfload or { } -config.luaotfload.version = config.luaotfload.version or version -config.luaotfload.names_dir = config.luaotfload.names_dir or "names" -config.luaotfload.cache_dir = config.luaotfload.cache_dir or "fonts" -config.luaotfload.index_file = config.luaotfload.index_file +local luaotfloadconfig = config.luaotfload or { } +config.luaotfload = luaotfloadconfig +luaotfloadconfig.version = luaotfloadconfig.version or version +luaotfloadconfig.names_dir = luaotfloadconfig.names_dir or "names" +luaotfloadconfig.cache_dir = luaotfloadconfig.cache_dir or "fonts" +luaotfloadconfig.index_file = luaotfloadconfig.index_file or "luaotfload-names.lua" -config.luaotfload.formats = config.luaotfload.formats or "otf,ttf,ttc,dfont" +luaotfloadconfig.formats = luaotfloadconfig.formats + or "otf,ttf,ttc,dfont" +luaotfloadconfig.reload = false do -- we don’t have file.basename and the likes yet, so inline parser ftw local slash = P"/" @@ -122,9 +125,9 @@ do -- we don’t have file.basename and the likes yet, so inline parser ftw local self = lpegmatch(p_basename, stringlower(arg[0])) if self == "luaotfload-tool" then - config.luaotfload.self = "luaotfload-tool" + luaotfloadconfig.self = "luaotfload-tool" else - config.luaotfload.self = "mkluatexfontdb" + luaotfloadconfig.self = "mkluatexfontdb" end end @@ -170,7 +173,7 @@ local names = fonts.names local status_file = "luaotfload-status" local luaotfloadstatus = require (status_file) -config.luaotfload.status = luaotfloadstatus +luaotfloadconfig.status = luaotfloadstatus local sanitize_fontname = names.sanitize_fontname @@ -283,16 +286,16 @@ Enter 'luaotfload-tool --help' for a larger list of options. local help_msg = function (version) local template = help_messages[version] iowrite(stringformat(template, - config.luaotfload.self, + luaotfloadconfig.self, names_plain, names_bin, caches.getwritablepath ( - config.luaotfload.cache_dir))) + luaotfloadconfig.cache_dir))) end local version_msg = function ( ) local out = function (...) texiowrite_nl (stringformat (...)) end - out ("%s version %q", config.luaotfload.self, version) + out ("%s version %q", luaotfloadconfig.self, version) out ("revision %q", luaotfloadstatus.notes.revision) out ("database version %q", names.version) out ("Lua interpreter: %s; version %q", runtime[1], runtime[2]) @@ -1077,6 +1080,7 @@ local process_cmdline = function ( ) -- unit -> jobspec list = 1, log = 1, ["no-reload"] = "n", + ["skip-read"] = "R", ["prefer-texmf"] = "p", quiet = "q", ["show-blacklist"] = "b", @@ -1087,7 +1091,7 @@ local process_cmdline = function ( ) -- unit -> jobspec warnings = "w", } - local short_options = "bDfFiIlnpqSuvVhw" + local short_options = "bDfFiIlnpqRSuvVhw" local options, _, optarg = alt_getopt.get_ordered_opts (arg, short_options, long_options) @@ -1144,7 +1148,7 @@ local process_cmdline = function ( ) -- unit -> jobspec result.show_info = true result.full_info = true elseif v == "alias" then - config.luaotfload.self = optarg[n] + luaotfloadconfig.self = optarg[n] elseif v == "l" then action_pending["flush"] = true elseif v == "list" then @@ -1158,7 +1162,7 @@ local process_cmdline = function ( ) -- unit -> jobspec elseif v == "D" then result.dry_run = true elseif v == "p" then --- TODO adapt to new db structure - config.luaotfload.prioritize = "texmf" + luaotfloadconfig.prioritize = "texmf" elseif v == "b" then action_pending["blacklist"] = true elseif v == "diagnose" then @@ -1167,13 +1171,16 @@ local process_cmdline = function ( ) -- unit -> jobspec elseif v == "formats" then names.set_font_filter (optarg[n]) elseif v == "n" then - config.luaotfload.update_live = false + luaotfloadconfig.update_live = false elseif v == "S" then - config.luaotfload.statistics = true + luaotfloadconfig.statistics = true + elseif v == "R" then + --- dev only, undocumented + luaotfloadconfig.skip_read = true end end - if config.luaotfload.self == "mkluatexfontdb" then + if luaotfloadconfig.self == "mkluatexfontdb" then --- TODO drop legacy ballast after 2.4 result.help_version = "mkluatexfontdb" action_pending["generate"] = true result.log_level = math.max(1, result.log_level) -- cgit v1.2.3 From b8f01ab3aa15499ab7f883e9dfce47a235a3e9dd Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 15 Sep 2013 22:58:52 +0200 Subject: [db] order design sizes --- luaotfload-database.lua | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index bb7f17e..6c52b5e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2791,6 +2791,27 @@ local collect_families = function (mappings) return families end +local cmp_sizes = function (a, b) + return a [1] < b [1] +end + +local order_design_sizes = function (families) + + report ("info", 2, "db", "Ordering design sizes.") + + for location, data in next, families do + for format, data in next, data do + for familyname, data in next, data do + for style, data in next, data do + tablesort (data, cmp_sizes) + end + end + end + end + + return families +end + local retrieve_namedata = function (currentnames, targetnames, dry_run, @@ -3005,6 +3026,9 @@ update_names = function (currentnames, force, dry_run) report ("both", 2, "db", "Updating the font names database" .. (force and " forcefully" or "")) + --- pass 1 get raw data: read font files (normal case) or reuse + --- information present in index + if luaotfloadconfig.skip_read == true then --- the difference to a “dry run” is that we don’t search --- for font files entirely. we also ignore the “force” @@ -3041,6 +3065,7 @@ update_names = function (currentnames, force, dry_run) n_rawnames, n_newnames) end + --- pass 2 (optional): collect some stats about the raw font info if luaotfloadconfig.statistics == true then targetnames.meta.statistics = collect_statistics (targetnames.mappings) @@ -3050,9 +3075,16 @@ update_names = function (currentnames, force, dry_run) --- non-texmf entries are redirected there and the mapping --- needs to be 100% consistent + --- pass 3: build filename table targetnames.files = generate_filedata (targetnames.mappings) + + --- pass 4: build family lookup table targetnames.families = collect_families (targetnames.mappings) + --- pass 5: order design size tables + targetnames.families = order_design_sizes (targetnames.families) + + report ("info", 3, "db", "Rebuilt in %0.f ms.", 1000 * (osgettimeofday () - starttime)) -- cgit v1.2.3 From 31abae6c9f15d75bf0b09be212eef35ae124b4a8 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 18:57:31 +0100 Subject: [db] fix test for successful update --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 6c52b5e..aa1370f 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -477,7 +477,7 @@ load_names = function (dry_run) [[This can take several minutes; please be patient.]]) data = update_names (initialize_namedata (get_font_filter ()), nil, dry_run) - if not success then + if not data then report ("both", 0, "db", "Database creation unsuccessful.") end end -- cgit v1.2.3 From 5e5f8de85320ba0ed005fc1676e6df4d877845df Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 19:30:46 +0100 Subject: [db,tool] add --compress option for gzipping the index --- luaotfload-database.lua | 17 +++++++++++++++-- luaotfload-tool.lua | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index aa1370f..1c694fe 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,6 +31,7 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table +local gzipsave = gzip.save local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -83,6 +84,7 @@ local tablecopy = table.copy local tablefastcopy = table.fastcopy local tabletofile = table.tofile local tabletohash = table.tohash +local tableserialize = table.serialize --- the font loader namespace is “fonts”, same as in Context --- we need to put some fallbacks into place for when running @@ -3134,6 +3136,11 @@ save_lookups = function ( ) return false end +local tabletogzip = function (filename, ...) + local serialized = tableserialize(true, ...) + gzipsave (filenmaeserialized) +end + --- save_names() is usually called without the argument --- dbobj? -> bool save_names = function (currentnames) @@ -3143,9 +3150,15 @@ save_names = function (currentnames) local path = names.path.index local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, currentnames, true) osremove (lucname) - caches.compile (currentnames, luaname, lucname) + if luaotfloadconfig.compress then + local serialized = tableserialize (currentnames, true) + gzipsave (luaname, serialized) + caches.compile (currentnames, "", lucname) + else + tabletofile (luaname, currentnames, true) + caches.compile (currentnames, luaname, lucname) + end if lfsisfile (luaname) and lfsisfile (lucname) then report ("info", 1, "db", "Font index saved") report ("info", 3, "db", "Text: " .. luaname) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index e410d44..f2dd98a 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -1065,6 +1065,7 @@ local process_cmdline = function ( ) -- unit -> jobspec local long_options = { alias = 1, cache = 1, + compress = "c", diagnose = 1, ["dry-run"] = "D", ["flush-lookups"] = "l", @@ -1091,7 +1092,7 @@ local process_cmdline = function ( ) -- unit -> jobspec warnings = "w", } - local short_options = "bDfFiIlnpqRSuvVhw" + local short_options = "bcDfFiIlnpqRSuvVhw" local options, _, optarg = alt_getopt.get_ordered_opts (arg, short_options, long_options) @@ -1177,6 +1178,8 @@ local process_cmdline = function ( ) -- unit -> jobspec elseif v == "R" then --- dev only, undocumented luaotfloadconfig.skip_read = true + elseif v == "c" then + luaotfloadconfig.compress = true end end -- cgit v1.2.3 From 352272f48743f785c3a0cea0dcf4e76efc3270e5 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 19:35:13 +0100 Subject: [news,man] mentions --compress in the appropriate places --- NEWS | 2 ++ luaotfload-tool.lua | 1 + luaotfload-tool.rst | 3 +++ 3 files changed, 6 insertions(+) diff --git a/NEWS b/NEWS index 399ed97..937040a 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,8 @@ Change History * Test runner (script mktests) * New luaotfload-tool option: ``--no-reload`` * ``luaotfload-tool --find`` now understands request syntax + * option ``--compress`` filters text (Lua script) version of the font + index through gzip 2013/07/10, luaotfload v2.3a * Detect LuaJIT interpreter (LuaJITTeX) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index f2dd98a..ab1d4d9 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -214,6 +214,7 @@ This tool is part of the luaotfload package. Valid options are: -u --update update the database -n --no-reload suppress db update -f --force force re-indexing all fonts + -c --compress gzip index file (text version only) -l --flush-lookups empty lookup cache of font requests -D --dry-run skip loading of fonts, just scan --formats=[+|-]EXTENSIONS set, add, or subtract formats to index diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst index 03ff407..4ccc1dd 100644 --- a/luaotfload-tool.rst +++ b/luaotfload-tool.rst @@ -20,6 +20,7 @@ SYNOPSIS **luaotfload-tool** --update [ --force ] [ --quiet ] [ --verbose ] [ --prefer-texmf ] [ --dry-run ] [ --formats=[+|-]EXTENSIONS ] + [ --compress ] **luaotfload-tool** --find=FONTNAME [ --fuzzy ] [ --info ] [ --inspect ] [ --no-reload ] @@ -62,6 +63,8 @@ update mode all fonts. --no-reload, -n Suppress auto-updates to the database (e.g. when ``--find`` is passed an unknown name). +--compress Filter plain text version of font index through + gzip. --prefer-texmf, -p Organize the file name database in a way so that it prefer fonts in the *TEXMF* tree over -- cgit v1.2.3 From 2f0fb9ef09f796e493776e4ffec164fb014e4283 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 3 Nov 2013 19:53:03 +0100 Subject: [db] add gzip check to script loader --- luaotfload-database.lua | 51 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1c694fe..df2199e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,6 +31,7 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table +local gzipload = gzip.load local gzipsave = gzip.save local iolines = io.lines local ioopen = io.open @@ -396,27 +397,37 @@ end --- string -> (string * table) local load_lua_file = function (path) - local foundname = filereplacesuffix(path, "luc") + local foundname = filereplacesuffix (path, "luc") - local fh = ioopen(foundname, "rb") -- try bin first + if false then + local fh = ioopen (foundname, "rb") -- try bin first if fh then local chunk = fh:read"*all" fh:close() - code = load(chunk, "b") + code = load (chunk, "b") end +end if not code then --- fall back to text file - foundname = filereplacesuffix(path, "lua") + foundname = filereplacesuffix (path, "lua") fh = ioopen(foundname, "rb") if fh then local chunk = fh:read"*all" fh:close() - code = load(chunk, "t") + code = load (chunk, "t") + end + end + + if not code then --- probe gzipped file + foundname = filereplacesuffix (path, "lua.gz") + local chunk = gzipload (foundname) + if chunk then + code = load (chunk, "t") end end if not code then return nil, nil end - return foundname, code() + return foundname, code () end --- define locals in scope @@ -3136,11 +3147,6 @@ save_lookups = function ( ) return false end -local tabletogzip = function (filename, ...) - local serialized = tableserialize(true, ...) - gzipsave (filenmaeserialized) -end - --- save_names() is usually called without the argument --- dbobj? -> bool save_names = function (currentnames) @@ -3151,22 +3157,35 @@ save_names = function (currentnames) local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then osremove (lucname) + local gzname = luaname .. ".gz" if luaotfloadconfig.compress then local serialized = tableserialize (currentnames, true) - gzipsave (luaname, serialized) + gzipsave (gzname, serialized) caches.compile (currentnames, "", lucname) else tabletofile (luaname, currentnames, true) caches.compile (currentnames, luaname, lucname) end - if lfsisfile (luaname) and lfsisfile (lucname) then - report ("info", 1, "db", "Font index saved") + report ("info", 1, "db", "Font index saved at ...") + local success = false + if lfsisfile (luaname) then report ("info", 3, "db", "Text: " .. luaname) + success = true + end + if lfsisfile (gzname) then + report ("info", 3, "db", "Gzip: " .. gzname) + success = true + end + if lfsisfile (lucname) then report ("info", 3, "db", "Byte: " .. lucname) + success = true + end + if success then return true + else + report ("info", 0, "db", "Could not compile font index") + return false end - report ("info", 0, "db", "Could not compile font index") - return false end report ("info", 0, "db", "Index file not writable") if not fileiswritable (luaname) then -- cgit v1.2.3 From 8e65e677bb678de920ea55097b8889b4245c0e15 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 4 Nov 2013 20:45:27 +0100 Subject: [db] stop exposing the global index object --- luaotfload-auxiliary.lua | 10 ++++- luaotfload-database.lua | 100 ++++++++++++++++++++++----------------------- luaotfload-diagnostics.lua | 6 +-- luaotfload-tool.lua | 13 +++--- 4 files changed, 69 insertions(+), 60 deletions(-) diff --git a/luaotfload-auxiliary.lua b/luaotfload-auxiliary.lua index da3f5f2..92e1fd9 100644 --- a/luaotfload-auxiliary.lua +++ b/luaotfload-auxiliary.lua @@ -588,6 +588,8 @@ aux.sprint_math_dimension = sprint_math_dimension local namesresolve = fonts.names.resolve local namesscan_dir = fonts.names.scan_dir +--[====[-- TODO -> port this to new db model + --- local directories ------------------------------------------------- --- migrated from luaotfload-database.lua @@ -595,7 +597,7 @@ local namesscan_dir = fonts.names.scan_dir --- string -> (int * int) local scan_external_dir = function (dir) - local old_names, new_names = names.data + local old_names, new_names = names.data() if not old_names then old_names = load_names() end @@ -612,6 +614,12 @@ end aux.scan_external_dir = scan_external_dir +--]====]-- + +aux.scan_external_dir = function () + print "ERROR: scan_external_dir() is not implemented" +end + --- db queries -------------------------------------------------------- --- https://github.com/lualatex/luaotfload/issues/74 diff --git a/luaotfload-database.lua b/luaotfload-database.lua index df2199e..cba2f26 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -95,6 +95,7 @@ fonts.names = fonts.names or { } fonts.definers = fonts.definers or { } local names = fonts.names +local name_index = nil -- upvalue for names.data local luaotfloadconfig = config.luaotfload --- always present luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" @@ -271,7 +272,7 @@ This is a sketch of the luaotfload db: type dbobj = { families : familytable; - filenames : filemap; + files : filemap; status : filestatus; mappings : fontentry list; meta : metadata; @@ -382,7 +383,7 @@ local initialize_namedata = function (formats) --- returns dbobj status = { }, -- was: status; map abspath -> mapping mappings = { }, -- TODO: check if still necessary after rewrite names = { }, --- filenames = { }, -- created later +-- files = { }, -- created later meta = { formats = formats, statistics = { }, @@ -442,7 +443,7 @@ local load_lookups local read_blacklist local read_fonts_conf local reload_db -local resolve +local resolve_name local resolve_cached local resolve_fullpath local save_names @@ -452,7 +453,6 @@ local get_font_filter local set_font_filter --- state of the database -local fonts_loaded = false local fonts_reloaded = false --- limit output when approximate font matching (luaotfload-tool -F) @@ -494,7 +494,6 @@ load_names = function (dry_run) report ("both", 0, "db", "Database creation unsuccessful.") end end - fonts_loaded = true return data end @@ -590,18 +589,17 @@ end --- string -> (string * string * bool) crude_file_lookup_verbose = function (filename) - if not names.data then names.data = load_names() end - local data = names.data - local mappings = data.mappings - local filenames = data.filenames + if not name_index then name_index = load_names() end + local mappings = name_index.mappings + local files = name_index.files local found --- look up in db first ... - found = verbose_lookup(filenames, "bare", filename) + found = verbose_lookup(files, "bare", filename) if found then return found, nil, true end - found = verbose_lookup(filenames, "base", filename) + found = verbose_lookup(files, "base", filename) if found then return found, nil, true end @@ -618,18 +616,17 @@ end --- string -> (string * string * bool) crude_file_lookup = function (filename) - if not names.data then names.data = load_names() end - local data = names.data - local mappings = data.mappings - local filenames = data.filenames + if not name_index then name_index = load_names() end + local mappings = name_index.mappings + local files = name_index.files local found - found = filenames.base[filename] - or filenames.bare[filename] + found = files.base[filename] + or files.bare[filename] if found then - found = filenames.full[found] + found = files.full[found] if found == nil then found = dummy_findfile(filename) end @@ -672,15 +669,15 @@ the texmf or filesystem. --doc]]-- local verify_font_file = function (basename) - if not names.data then names.data = load_names() end - local filenames = names.data.filenames - local idx = filenames.base[basename] + if not name_index then name_index = load_names() end + local files = name_index.files + local idx = files.base[basename] if not idx then return false end --- firstly, check filesystem - local fullname = filenames.full[idx] + local fullname = files.full[idx] if fullname and lfsisfile(fullname) then return true end @@ -881,6 +878,7 @@ the font database created by the luaotfload-tool script. --- 'a -> 'a -> table -> (string * string | bool * bool) --- +--[===[ resolve = function (_, _, specification) -- the 1st two parameters are used by ConTeXt if not fonts_loaded then names.data = load_names() end local data = names.data @@ -1006,7 +1004,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C --- “found” is really synonymous with “registered in the db”. local entry = found[1] local success, filename, subfont - = get_font_file(data.filenames.full, entry) + = get_font_file(data.files.full, entry) if success == true then report("log", 0, "resolve", "Font family='%s', subfamily='%s' found: %s", @@ -1047,7 +1045,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C end local success, filename, subfont - = get_font_file(data.filenames.full, match) + = get_font_file(data.files.full, match) if success == true then report("log", 0, "resolve", "Font family='%s', subfamily='%s' found: %s", @@ -1058,7 +1056,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C elseif fallback then local success, filename, subfont - = get_font_file(data.filenames.full, fallback) + = get_font_file(data.files.full, fallback) if success == true then report("log", 0, "resolve", "No exact match for request %s; using fallback", @@ -1074,7 +1072,7 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C --- pick the first candidate encountered local entry = candidates[1] local success, filename, subfont - = get_font_file(data.filenames.full, entry) + = get_font_file(data.files.full, entry) if success == true then report("log", 0, "resolve", "Font family='%s', subfamily='%s' found: %s", @@ -1096,16 +1094,19 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C --- else, default to requested name return specification.name, false, false end --- resolve() +]===] + +resolve_name = function (specification) + if not name_index then name_index = load_names () end +end resolve_fullpath = function (fontname, ext) --- getfilename() - if not fonts_loaded then - names.data = load_names() - end - local filenames = names.data.filenames - local idx = filenames.base[fontname] - or filenames.bare[fontname] + if not name_index then name_index = load_names () end + local files = name_index.files + local idx = files.base[fontname] + or files.bare[fontname] if idx then - return filenames.full[idx] + return files.full[idx] end return "" end @@ -1115,7 +1116,7 @@ end --- string -> ('a -> 'a) -> 'a list -> 'a reload_db = function (why, caller, ...) - local namedata = names.data + local namedata = name_index local formats = tableconcat (namedata.meta.formats, ",") report ("both", 1, "db", @@ -1127,7 +1128,7 @@ reload_db = function (why, caller, ...) if namedata then fonts_reloaded = true - names.data = namedata + name_index = namedata return caller (...) end @@ -1167,16 +1168,14 @@ find_closest = function (name, limit) local name = sanitize_fontname (name) limit = limit or fuzzy_limit - if not fonts_loaded then names.data = load_names() end - - local data = names.data - - if type(data) ~= "table" then + if not name_index then name_index = load_names () end + if not name_index or type (name_index) ~= "table" then if not fonts_reloaded then return reload_db("no database", find_closest, name) end return false end + local by_distance = { } --- (int, string list) dict local distances = { } --- int list local cached = { } --- (string, int) dict @@ -2473,7 +2472,7 @@ local generate_filedata = function (mappings) local nmappings = #mappings - local filenames = { + local files = { bare = { ["local"] = { }, system = { }, --- mapped to mapping format -> index in full @@ -2487,9 +2486,9 @@ local generate_filedata = function (mappings) full = { }, --- non-texmf } - local base = filenames.base - local bare = filenames.bare - local full = filenames.full + local base = files.base + local bare = files.bare + local full = files.full local conflicts = { basenames = 0, @@ -2586,7 +2585,7 @@ local generate_filedata = function (mappings) -- addmap(sys) -- end - return filenames + return files end local match_synonyms = function (pattern) @@ -3025,7 +3024,7 @@ update_names = function (currentnames, force, dry_run) report ("info", 2, "db", "Skipping database update") --- skip all db updates - return currentnames or names.data + return currentnames or name_index end local starttime = osgettimeofday () @@ -3101,7 +3100,7 @@ update_names = function (currentnames, force, dry_run) report ("info", 3, "db", "Rebuilt in %0.f ms.", 1000 * (osgettimeofday () - starttime)) - names.data = targetnames + name_index = targetnames if dry_run ~= true then @@ -3151,7 +3150,7 @@ end --- dbobj? -> bool save_names = function (currentnames) if not currentnames then - currentnames = names.data + currentnames = name_index end local path = names.path.index local luaname, lucname = path.lua, path.luc @@ -3354,6 +3353,7 @@ names.set_font_filter = set_font_filter names.flush_lookup_cache = flush_lookup_cache names.save_lookups = save_lookups names.load = load_names +names.data = function () return name_index end names.save = save_names names.update = update_names names.crude_file_lookup = crude_file_lookup @@ -3373,8 +3373,8 @@ if luaotfloadconfig.resolver == "cached" then names.resolve = resolve_cached names.resolvespec = resolve_cached else - names.resolve = resolve - names.resolvespec = resolve + names.resolvespec = resolve_name + names.resolve_name = resolve_name end names.find_closest = find_closest diff --git a/luaotfload-diagnostics.lua b/luaotfload-diagnostics.lua index 2aa09fe..a11f9ea 100644 --- a/luaotfload-diagnostics.lua +++ b/luaotfload-diagnostics.lua @@ -54,12 +54,12 @@ end local check_index = function (errcnt) out "================= font names ==================" + local name_index = names.data() - if not names.data then - names.data = names.load () + if not name_index then + name_index = names.load () end - local namedata = names.data local mappings = namedata.mappings if not namedata and namedata.formats and namedata.version then diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index ab1d4d9..7c2beda 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -677,7 +677,7 @@ The font info knows two levels of detail: --doc]]-- local show_font_info = function (basename, askedname, detail, warnings) - local filenames = names.data.filenames + local filenames = names.data().filenames local index = filenames.base[basename] local fullname = filenames.full[index] askedname = sanitize_fontname (askedname) @@ -775,7 +775,7 @@ actions.generate = function (job) fontnames = names.update(fontnames, job.force_reload, job.dry_run) logs.names_report("info", 2, "db", "Fonts in the database: %i", #fontnames.mappings) - if names.data then + if names.data() then return true, true end return false, false @@ -933,8 +933,9 @@ local splitcomma = names.patterns.splitcomma actions.list = function (job) local criterion = job.criterion - local asked_fields = job.asked_fields + local name_index = names.data () + if asked_fields then asked_fields = lpegmatch(splitcomma, asked_fields) else @@ -942,11 +943,11 @@ actions.list = function (job) asked_fields = { "fullname", "version", } end - if not names.data then - names.data = names.load() + if not name_index then + name_index = names.load() end - local mappings = names.data.mappings + local mappings = name_index.mappings local nmappings = #mappings if criterion == "*" then -- cgit v1.2.3 From 3da77f501c148c757ca9cb570666ea409aff1ff3 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 00:27:31 +0100 Subject: [db] add first draft font family query --- luaotfload-database.lua | 218 ++++++++++++++++++++++++++-- luaotfload-features.lua | 6 +- luaotfload-merged.lua | 374 ++++++++++++++++++++++++++++++++++-------------- 3 files changed, 475 insertions(+), 123 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index cba2f26..390c3ea 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -648,16 +648,21 @@ Existence of the resolved file name is verified differently depending on whether the index entry has a texmf flag set. --doc]]-- -local get_font_file = function (fullnames, entry) - local basename = entry.basename - if entry.location == "texmf" then +local get_font_file = function (index) + local entry = name_index.mappings [index] + if not entry then + return false + end + local filedata = entry.file + local basename = filedata.base + if filedata.location == "texmf" then if kpselookup(basename) then - return true, basename, entry.subfont + return true, basename, filedata.subfont end - else - local fullname = fullnames[entry.index] - if lfsisfile(fullname) then - return true, basename, entry.subfont + else --- system, local + local fullname = name_index.files.full [index] + if lfsisfile (fullname) then + return true, basename, filedata.subfont end end return false @@ -1096,17 +1101,206 @@ resolve = function (_, _, specification) -- the 1st two parameters are used by C end --- resolve() ]===] +local choose_closest = function (distances) + local closest = 2^51 + local match + for i = 1, #distances do + local d, index = unpack (distances [i]) + if d < closest then + closest = d + match = index + end + end + return match +end + +--[[doc-- + + choose_size -- Pick a font face of appropriate size from the list + of family members with matching style. There are three categories: + + 1. exact matches: if there is a face whose design size equals + the asked size, it is returned immediately and no further + candidates are inspected. + + 2. range matches: of all faces in whose design range the + requested size falls the one whose center the requested + size is closest to is returned. + + 3. out-of-range matches: of all other faces (i. e. whose range + is above or below the asked size) the one is chosen whose + boundary (upper or lower) is closest to the requested size. + + 4. default matches: if no design size or a design size of zero + is requested, the face with the default size is returned. + +--doc]]-- + +--- int * int * int * int list -> int -> int +local choose_size = function (sizes, askedsize) + local mappings = name_index.mappings + local match = sizes.default + local exact + local inrange = { } --- distance * index list + local norange = { } --- distance * index list + local fontname, subfont + if askedsize ~= 0 then + --- firstly, look for an exactly matching design size or + --- matching range + for i = 1, #sizes do + local dsnsize, high, low, index = unpack (sizes [i]) + if dsnsize == askedsize then + --- exact match, this is what we were looking for + exact = index + goto skip + elseif askedsize < low then + --- below range, add to the norange table + local d = low - askedsize + norange [#norange + 1] = { d, index } + elseif askedsize > high then + --- beyond range, add to the norange table + local d = askedsize - high + norange [#norange + 1] = { d, index } + else + --- range match + local d = ((low + high) / 2) - askedsize + if d < 0 then + d = -d + end + inrange [#inrange + 1] = { d, index } + end + end + end +::skip:: + if exact then + match = exact + elseif #inrange > 0 then + match = choose_closest (inrange) + elseif #norange > 0 then + match = choose_closest (norange) + end + return match +end + +local format_precedence = { + "otf", "ttc", "ttf", + "dfont", "afm", "pfb", + "pfa", +} + +local location_precedence = { + "local", "system", "texmf", +} + +--[[doc-- + + resolve_familyname -- Query the families table for an entry + matching the specification. + The parameters “name” and “style” are pre-sanitized. + +--doc]]-- +--- spec -> string -> string -> int -> string * int +local resolve_familyname = function (specification, name, style, askedsize) + local families = name_index.families + local mappings = name_index.mappings + local candidates = nil + --- arrow code alert + for i = 1, #location_precedence do + local location = location_precedence [i] + local locgroup = families [location] + for j = 1, #format_precedence do + local format = format_precedence [j] + local fmtgroup = locgroup [format] + if fmtgroup then + local familygroup = fmtgroup [name] + if familygroup then + local stylegroup = familygroup [style] + if stylegroup then --- suitable match + candidates = stylegroup + goto done + end + end + end + end + end + if true then + return nil, nil + end +::done:: + index = choose_size (candidates, askedsize) + local success, resolved, subfont = get_font_file (index) + if not success then + return nil, nil + end + report ("info", 2, "db", "Match found: %s(%d)", + resolved, subfont or 0) + return resolved, subfont +end + +local resolve_fontname = function (specification, name, style, askedsize) + local resolved, subfont + --[[ TODO ]] + return resolved, subfont +end + +--[[doc-- + + resolve_name -- Perform a name: lookup. This first queries the + font families table and, if there is no match for the spec, the + font names table. + The return value is a pair consisting of the file name and the + subfont index if appropriate.. + +--doc]]-- + +--- table -> string * (int | bool) resolve_name = function (specification) + local resolved, subfont if not name_index then name_index = load_names () end + local name = sanitize_fontname (specification.name) + local style = sanitize_fontname (specification.style) or "r" + local askedsize = specification.optsize + + if askedsize then + askedsize = tonumber (askedsize) + else + askedsize = specification.size + if askedsize and askedsize >= 0 then + askedsize = askedsize / 65536 + else + askedsize = 0 + end + end + + resolved, subfont = resolve_familyname (specification, name, style, askedsize) + if not resolved then + resolved, subfont = resolve_fontname (specification, name, style, askedsize) + end + if not resolved then + resolved = specification.name, false + end + return resolved, subfont end resolve_fullpath = function (fontname, ext) --- getfilename() if not name_index then name_index = load_names () end local files = name_index.files - local idx = files.base[fontname] - or files.bare[fontname] - if idx then - return files.full[idx] + local basedata = files.base + local baredata = files.bare + for i = 1, #location_precedence do + local location = location_precedence [i] + local basenames = basedata [location] + local barenames = baredata [location] [ext] + local idx + if basenames ~= nil then + idx = basenames [fontname] + end + if not idx and barenames ~= nil then + idx = barenames [fontname] + end + if idx then + return files.full [idx] + end end return "" end diff --git a/luaotfload-features.lua b/luaotfload-features.lua index db0b1d0..f324be6 100644 --- a/luaotfload-features.lua +++ b/luaotfload-features.lua @@ -1097,9 +1097,9 @@ local select_lookup = function (request) end local supported = { - b = "bold", - i = "italic", - bi = "bolditalic", + b = "b", + i = "i", + bi = "bi", aat = false, icu = false, gr = false, diff --git a/luaotfload-merged.lua b/luaotfload-merged.lua index 3c63ba0..3e89856 100644 --- a/luaotfload-merged.lua +++ b/luaotfload-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 09/13/13 10:59:07 +-- merge date : 11/04/13 11:52:02 do -- begin closure to overcome local limits and interference @@ -134,6 +134,7 @@ local utfbom_16_le=P('\255\254') local utfbom_8=P('\239\187\191') local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") +local utfstricttype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8") local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) local utf8next=R("\128\191") patterns.utfbom_32_be=utfbom_32_be @@ -149,6 +150,7 @@ patterns.utf8three=R("\224\239")*utf8next*utf8next patterns.utf8four=R("\240\244")*utf8next*utf8next*utf8next patterns.utfbom=utfbom patterns.utftype=utftype +patterns.utfstricttype=utfstricttype patterns.utfoffset=utfoffset local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false) @@ -2611,6 +2613,31 @@ function number.signed(i) return "-",-i end end +local zero=P("0")^1/"" +local plus=P("+")/"" +local minus=P("-") +local separator=S(".") +local digit=R("09") +local trailing=zero^1*#S("eE") +local exponent=(S("eE")*(plus+Cs((minus*zero^0*P(-1))/"")+minus)*zero^0*(P(-1)*Cc("0")+P(1)^1)) +local pattern_a=Cs(minus^0*digit^1*(separator/""*trailing+separator*(trailing+digit)^0)*exponent) +local pattern_b=Cs((exponent+P(1))^0) +function number.sparseexponent(f,n) + if not n then + n=f + f="%e" + end + local tn=type(n) + if tn=="string" then + local m=tonumber(n) + if m then + return lpegmatch((f=="%e" or f=="%E") and pattern_a or pattern_b,format(f,m)) + end + elseif tn=="number" then + return lpegmatch((f=="%e" or f=="%E") and pattern_a or pattern_b,format(f,n)) + end + return tostring(n) +end local preamble=[[ local type = type local tostring = tostring @@ -2629,6 +2656,7 @@ local autosingle = string.autosingle local autodouble = string.autodouble local sequenced = table.sequenced local formattednumber = number.formatted +local sparseexponent = number.sparseexponent ]] local template=[[ %s @@ -2701,6 +2729,14 @@ local format_E=function(f) n=n+1 return format("format('%%%sE',a%s)",f,n) end +local format_j=function(f) + n=n+1 + return format("sparseexponent('%%%se',a%s)",f,n) +end +local format_J=function(f) + n=n+1 + return format("sparseexponent('%%%sE',a%s)",f,n) +end local format_x=function(f) n=n+1 return format("format('%%%sx',a%s)",f,n) @@ -2858,6 +2894,10 @@ local format_M=function(f) end return format([[formattednumber(a%s,%q,",")]],n,f) end +local format_z=function(f) + n=n+(tonumber(f) or 1) + return "''" +end local format_rest=function(s) return format("%q",s) end @@ -2891,12 +2931,13 @@ local builder=Cs { "start", +V("c")+V("C")+V("S") +V("Q") +V("N") -+V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("h") -+V("w") ++V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") +V("W") +V("a") +V("A") -+V("m")+V("M") ++V("j")+V("J") ++V("m")+V("M") ++V("z") +V("*") )+V("*") )*(P(-1)+Carg(1)) @@ -2932,8 +2973,11 @@ local builder=Cs { "start", ["I"]=(prefix_any*P("I"))/format_I, ["w"]=(prefix_any*P("w"))/format_w, ["W"]=(prefix_any*P("W"))/format_W, + ["j"]=(prefix_any*P("j"))/format_j, + ["J"]=(prefix_any*P("J"))/format_J, ["m"]=(prefix_tab*P("m"))/format_m, ["M"]=(prefix_tab*P("M"))/format_M, + ["z"]=(prefix_any*P("z"))/format_z, ["a"]=(prefix_any*P("a"))/format_a, ["A"]=(prefix_any*P("A"))/format_A, ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, @@ -3398,10 +3442,12 @@ nodes.handlers={} local nodecodes={} for k,v in next,node.types () do nodecodes[string.gsub(v,"_","")]=k end local whatcodes={} for k,v in next,node.whatsits() do whatcodes[string.gsub(v,"_","")]=k end local glyphcodes={ [0]="character","glyph","ligature","ghost","left","right" } +local disccodes={ [0]="discretionary","explicit","automatic","regular","first","second" } nodes.nodecodes=nodecodes nodes.whatcodes=whatcodes nodes.whatsitcodes=whatcodes nodes.glyphcodes=glyphcodes +nodes.disccodes=disccodes local free_node=node.free local remove_node=node.remove local new_node=node.new @@ -5072,6 +5118,9 @@ fonts.names.resolvespec=fonts.names.resolve function fonts.names.getfilename(askedname,suffix) return "" end +function fonts.names.ignoredfile(filename) + return false +end end -- closure @@ -5236,6 +5285,7 @@ afm.syncspace=true afm.addligatures=true afm.addtexligatures=true afm.addkerns=true +local applyruntimefixes=fonts.treatments and fonts.treatments.applyfixes local function setmode(tfmdata,value) if value then tfmdata.properties.mode=lower(value) @@ -5425,7 +5475,7 @@ end local addkerns,addligatures,addtexligatures,unify,normalize function afm.load(filename) filename=resolvers.findfile(filename,'afm') or "" - if filename~="" then + if filename~="" and not fonts.names.ignoredfile(filename) then local name=file.removesuffix(file.basename(filename)) local data=containers.read(afm.cache,name) local attr=lfs.attributes(filename) @@ -5475,6 +5525,9 @@ function afm.load(filename) data=containers.write(afm.cache,name,data) data=containers.read(afm.cache,name) end + if applyruntimefixes and data then + applyruntimefixes(filename,data) + end end return data else @@ -6325,7 +6378,7 @@ local report_otf=logs.reporter("fonts","otf loading") local fonts=fonts local otf=fonts.handlers.otf otf.glists={ "gsub","gpos" } -otf.version=2.745 +otf.version=2.747 otf.cache=containers.define("fonts","otf",otf.version,true) local fontdata=fonts.hashes.identifiers local chardata=characters and characters.data @@ -6346,11 +6399,17 @@ local syncspace=true local forcenotdef=false local includesubfonts=false local overloadkerns=false +local applyruntimefixes=fonts.treatments and fonts.treatments.applyfixes local wildcard="*" local default="dflt" local fontloaderfields=fontloader.fields local mainfields=nil local glyphfields=nil +local formats=fonts.formats +formats.otf="opentype" +formats.ttf="truetype" +formats.ttc="truetype" +formats.dfont="truetype" registerdirective("fonts.otf.loader.cleanup",function(v) cleanup=tonumber(v) or (v and 1) or 0 end) registerdirective("fonts.otf.loader.force",function(v) forceload=v end) registerdirective("fonts.otf.loader.usemetatables",function(v) usemetatables=v end) @@ -6358,6 +6417,9 @@ registerdirective("fonts.otf.loader.pack",function(v) packdata=v end) registerdirective("fonts.otf.loader.syncspace",function(v) syncspace=v end) registerdirective("fonts.otf.loader.forcenotdef",function(v) forcenotdef=v end) registerdirective("fonts.otf.loader.overloadkerns",function(v) overloadkerns=v end) +local function otf_format(filename) + return formats[lower(file.suffix(filename))] +end local function load_featurefile(raw,featurefile) if featurefile and featurefile~="" then if trace_loading then @@ -6544,7 +6606,7 @@ end function enhancers.register(what,action) actions[what]=action end -function otf.load(filename,format,sub,featurefile) +function otf.load(filename,sub,featurefile) local base=file.basename(file.removesuffix(filename)) local name=file.removesuffix(base) local attr=lfs.attributes(filename) @@ -6642,7 +6704,7 @@ function otf.load(filename,format,sub,featurefile) data={ size=size, time=time, - format=format, + format=otf_format(filename), featuredata=featurefiles, resources={ filename=resolvers.unresolve(filename), @@ -6708,6 +6770,9 @@ function otf.load(filename,format,sub,featurefile) report_otf("loading from cache using hash %a",hash) end enhance("unpack",data,filename,nil,false) + if applyruntimefixes then + applyruntimefixes(filename,data) + end enhance("add dimensions",data,filename,nil,false) if trace_sequences then showfeatureorder(data,filename) @@ -7183,14 +7248,6 @@ local g_directions={ gsub_reversecontextchain=-1, gpos_reversecontextchain=-1, } -local function supported(features) - for i=1,#features do - if features[i].ismac then - return false - end - end - return true -end actions["reorganize subtables"]=function(data,filename,raw) local resources=data.resources local sequences={} @@ -7204,7 +7261,6 @@ actions["reorganize subtables"]=function(data,filename,raw) for k=1,#dw do local gk=dw[k] local features=gk.features - if not features or supported(features) then local typ=gk.type local chain=g_directions[typ] or 0 local subtables=gk.subtables @@ -7267,7 +7323,6 @@ actions["reorganize subtables"]=function(data,filename,raw) markclass=markclass, } end - end end end end @@ -7954,10 +8009,19 @@ local function copytotfm(data,cache_id) end end end + local filename=constructors.checkedfilename(resources) + local fontname=metadata.fontname + local fullname=metadata.fullname or fontname + local units=metadata.units_per_em or 1000 + if units==0 then + units=1000 + metadata.units_per_em=1000 + report_otf("changing %a units to %a",0,units) + end local monospaced=metadata.isfixedpitch or (pfminfo.panose and pfminfo.panose.proportion=="Monospaced") local charwidth=pfminfo.avgwidth - local italicangle=metadata.italicangle local charxheight=pfminfo.os2_xheight and pfminfo.os2_xheight>0 and pfminfo.os2_xheight + local italicangle=metadata.italicangle properties.monospaced=monospaced parameters.italicangle=italicangle parameters.charwidth=charwidth @@ -7986,15 +8050,6 @@ local function copytotfm(data,cache_id) end end spaceunits=tonumber(spaceunits) or 500 - local filename=constructors.checkedfilename(resources) - local fontname=metadata.fontname - local fullname=metadata.fullname or fontname - local units=metadata.units_per_em or 1000 - if units==0 then - units=1000 - metadata.units_per_em=1000 - report_otf("changing %a units to %a",0,units) - end parameters.slant=0 parameters.space=spaceunits parameters.space_stretch=units/2 @@ -8033,7 +8088,7 @@ local function copytotfm(data,cache_id) parameters.units=units properties.space=spacer properties.encodingbytes=2 - properties.format=data.format or fonts.formats[filename] or "opentype" + properties.format=data.format or otf_format(filename) or formats.otf properties.noglyphnames=true properties.filename=filename properties.fontname=fontname @@ -8058,9 +8113,8 @@ local function otftotfm(specification) local name=specification.name local sub=specification.sub local filename=specification.filename - local format=specification.format local features=specification.features.normal - local rawdata=otf.load(filename,format,sub,features and features.featurefile) + local rawdata=otf.load(filename,sub,features and features.featurefile) if rawdata and next(rawdata) then rawdata.lookuphash={} tfmdata=copytotfm(rawdata,cache_id) @@ -8142,41 +8196,33 @@ function otf.collectlookups(rawdata,kind,script,language) end return nil,nil end -local function check_otf(forced,specification,suffix,what) +local function check_otf(forced,specification,suffix) local name=specification.name if forced then - name=file.addsuffix(name,suffix,true) + name=specification.forcedname end local fullname=findbinfile(name,suffix) or "" if fullname=="" then fullname=fonts.names.getfilename(name,suffix) or "" end - if fullname~="" then + if fullname~="" and not fonts.names.ignoredfile(fullname) then specification.filename=fullname - specification.format=what return read_from_otf(specification) end end -local function opentypereader(specification,suffix,what) +local function opentypereader(specification,suffix) local forced=specification.forced or "" - if forced=="otf" then - return check_otf(true,specification,forced,"opentype") - elseif forced=="ttf" or forced=="ttc" or forced=="dfont" then - return check_otf(true,specification,forced,"truetype") + if formats[forced] then + return check_otf(true,specification,forced) else - return check_otf(false,specification,suffix,what) + return check_otf(false,specification,suffix) end end -readers.opentype=opentypereader -local formats=fonts.formats -formats.otf="opentype" -formats.ttf="truetype" -formats.ttc="truetype" -formats.dfont="truetype" -function readers.otf (specification) return opentypereader(specification,"otf",formats.otf ) end -function readers.ttf (specification) return opentypereader(specification,"ttf",formats.ttf ) end -function readers.ttc (specification) return opentypereader(specification,"ttf",formats.ttc ) end -function readers.dfont(specification) return opentypereader(specification,"ttf",formats.dfont) end +readers.opentype=opentypereader +function readers.otf (specification) return opentypereader(specification,"otf") end +function readers.ttf (specification) return opentypereader(specification,"ttf") end +function readers.ttc (specification) return opentypereader(specification,"ttf") end +function readers.dfont(specification) return opentypereader(specification,"ttf") end function otf.scriptandlanguage(tfmdata,attr) local properties=tfmdata.properties return properties.script or "dflt",properties.language or "dflt" @@ -9283,6 +9329,11 @@ local features={ medi=s_medi, fina=s_fina, isol=s_isol, + rphf=s_rphf, + half=s_half, + pref=s_pref, + blwf=s_blwf, + pstf=s_pstf, } analyzers.states=states analyzers.features=features @@ -9659,6 +9710,7 @@ local default="dflt" local nodecodes=nodes.nodecodes local whatcodes=nodes.whatcodes local glyphcodes=nodes.glyphcodes +local disccodes=nodes.disccodes local glyph_code=nodecodes.glyph local glue_code=nodecodes.glue local disc_code=nodecodes.disc @@ -9666,6 +9718,7 @@ local whatsit_code=nodecodes.whatsit local math_code=nodecodes.math local dir_code=whatcodes.dir local localpar_code=whatcodes.localpar +local discretionary_code=disccodes.discretionary local ligature_code=glyphcodes.ligature local privateattribute=attributes.private local a_state=privateattribute('state') @@ -9913,13 +9966,13 @@ local function get_alternative_glyph(start,alternatives,value,trace_alternatives end end end -local function multiple_glyphs(head,start,multiple) +local function multiple_glyphs(head,start,multiple,ignoremarks) local nofmultiples=#multiple if nofmultiples>0 then setfield(start,"char",multiple[1]) if nofmultiples>1 then local sn=getnext(start) - for k=2,nofmultiples do + for k=2,nofmultiples do local n=copy_node(start) setfield(n,"char",multiple[k]) setfield(n,"next",sn) @@ -9954,11 +10007,11 @@ function handlers.gsub_alternate(head,start,kind,lookupname,alternative,sequence end return head,start,true end -function handlers.gsub_multiple(head,start,kind,lookupname,multiple) +function handlers.gsub_multiple(head,start,kind,lookupname,multiple,sequence) if trace_multiples then logprocess("%s: replacing %s by multiple %s",pref(kind,lookupname),gref(getchar(start)),gref(multiple)) end - return multiple_glyphs(head,start,multiple) + return multiple_glyphs(head,start,multiple,sequence.flags[1]) end function handlers.gsub_ligature(head,start,kind,lookupname,ligature,sequence) local s,stop,discfound=getnext(start),nil,false @@ -10292,7 +10345,6 @@ function handlers.gpos_pair(head,start,kind,lookupname,kerns,sequence) prev=snext snext=getnext(snext) else - local krn=kerns[nextchar] if not krn then elseif type(krn)=="table" then if lookuptype=="pair" then @@ -10365,34 +10417,6 @@ function chainprocs.reversesub(head,start,stop,kind,chainname,currentcontext,loo return head,start,false end end -local function delete_till_stop(start,stop,ignoremarks) - local n=1 - if start==stop then - elseif ignoremarks then - repeat - local next=getnext(start) - if not marks[getchar(next)] then - local components=getfield(next,"components") - if components then - flush_node_list(components) - end - delete_node(start,next) - end - n=n+1 - until next==stop - else - repeat - local next=getnext(start) - local components=getfield(next,"components") - if components then - flush_node_list(components) - end - delete_node(start,next) - n=n+1 - until next==stop - end - return n -end function chainprocs.gsub_single(head,start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,chainindex) local current=start local subtables=currentlookup.subtables @@ -10432,7 +10456,6 @@ function chainprocs.gsub_single(head,start,stop,kind,chainname,currentcontext,lo end chainmores.gsub_single=chainprocs.gsub_single function chainprocs.gsub_multiple(head,start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname) - delete_till_stop(start,stop) local startchar=getchar(start) local subtables=currentlookup.subtables local lookupname=subtables[1] @@ -10451,7 +10474,7 @@ function chainprocs.gsub_multiple(head,start,stop,kind,chainname,currentcontext, if trace_multiples then logprocess("%s: replacing %s by multiple characters %s",cref(kind,chainname,chainlookupname,lookupname),gref(startchar),gref(replacements)) end - return multiple_glyphs(head,start,replacements) + return multiple_glyphs(head,start,replacements,currentlookup.flags[1]) end end return head,start,false @@ -10835,6 +10858,7 @@ function chainprocs.gpos_single(head,start,stop,kind,chainname,currentcontext,lo end return head,start,false end +chainmores.gpos_single=chainprocs.gpos_single function chainprocs.gpos_pair(head,start,stop,kind,chainname,currentcontext,lookuphash,currentlookup,chainlookupname,chainindex,sequence) local snext=getnext(start) if snext then @@ -10903,6 +10927,7 @@ function chainprocs.gpos_pair(head,start,stop,kind,chainname,currentcontext,look end return head,start,false end +chainmores.gpos_pair=chainprocs.gpos_pair local function show_skip(kind,chainname,char,ck,class) if ck[9] then logwarning("%s: skipping char %s, class %a, rule %a, lookuptype %a, %a => %a",cref(kind,chainname),gref(char),class,ck[1],ck[2],ck[9],ck[10]) @@ -10910,6 +10935,10 @@ local function show_skip(kind,chainname,char,ck,class) logwarning("%s: skipping char %s, class %a, rule %a, lookuptype %a",cref(kind,chainname),gref(char),class,ck[1],ck[2]) end end +local quit_on_no_replacement=true +directives.register("otf.chain.quitonnoreplacement",function(value) + quit_on_no_replacement=value +end) local function normal_handle_contextchain(head,start,kind,chainname,contexts,sequence,lookuphash) local flags=sequence.flags local done=false @@ -11119,7 +11148,11 @@ local function normal_handle_contextchain(head,start,kind,chainname,contexts,seq if chainlookup then local cp=chainprocs[chainlookup.type] if cp then - head,start,done=cp(head,start,last,kind,chainname,ck,lookuphash,chainlookup,chainlookupname,nil,sequence) + local ok + head,start,ok=cp(head,start,last,kind,chainname,ck,lookuphash,chainlookup,chainlookupname,nil,sequence) + if ok then + done=true + end else logprocess("%s: %s is not yet supported",cref(kind,chainname,chainlookupname),chainlookup.type) end @@ -11146,19 +11179,24 @@ local function normal_handle_contextchain(head,start,kind,chainname,contexts,seq end end local chainlookupname=chainlookups[i] - local chainlookup=lookuptable[chainlookupname] - local cp=chainlookup and chainmores[chainlookup.type] - if cp then - local ok,n - head,start,ok,n=cp(head,start,last,kind,chainname,ck,lookuphash,chainlookup,chainlookupname,i,sequence) - if ok then - done=true - i=i+(n or 1) - else + local chainlookup=lookuptable[chainlookupname] + if not chainlookup then + i=i+1 + else + local cp=chainmores[chainlookup.type] + if not cp then + logprocess("%s: %s is not yet supported",cref(kind,chainname,chainlookupname),chainlookup.type) i=i+1 + else + local ok,n + head,start,ok,n=cp(head,start,last,kind,chainname,ck,lookuphash,chainlookup,chainlookupname,i,sequence) + if ok then + done=true + i=i+(n or 1) + else + i=i+1 + end end - else - i=i+1 end if start then start=getnext(start) @@ -11171,7 +11209,7 @@ local function normal_handle_contextchain(head,start,kind,chainname,contexts,seq if replacements then head,start,done=chainprocs.reversesub(head,start,last,kind,chainname,ck,lookuphash,replacements) else - done=true + done=quit_on_no_replacement if trace_contexts then logprocess("%s: skipping match",cref(kind,chainname)) end @@ -11368,6 +11406,40 @@ local function featuresprocessor(head,font,attr) if not lookupcache then report_missing_cache(typ,lookupname) else + local function subrun(start) + local head=start + local done=false + while start do + local id=getid(start) + if id==glyph_code and getfont(start)==font and getsubtype(start)<256 then + local a=getattr(start,0) + if a then + a=(a==attr) and (not attribute or getattr(start,a_state)==attribute) + else + a=not attribute or getattr(start,a_state)==attribute + end + if a then + local lookupmatch=lookupcache[getchar(start)] + if lookupmatch then + local ok + head,start,ok=handler(head,start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,1) + if ok then + done=true + end + end + if start then start=getnext(start) end + else + start=getnext(start) + end + else + start=getnext(start) + end + end + if done then + success=true + return head + end + end while start do local id=getid(start) if id==glyph_code then @@ -11394,6 +11466,25 @@ local function featuresprocessor(head,font,attr) else start=getnext(start) end + elseif id==disc_code then + if getsubtype(start)==discretionary_code then + local pre=getfield(start,"pre") + if pre then + local new=subrun(pre) + if new then setfield(start,"pre",new) end + end + local post=getfield(start,"post") + if post then + local new=subrun(post) + if new then setfield(start,"post",new) end + end + local replace=getfield(start,"replace") + if replace then + local new=subrun(replace) + if new then setfield(start,"replace",new) end + end + end + start=getnext(start) elseif id==whatsit_code then local subtype=getsubtype(start) if subtype==dir_code then @@ -11438,6 +11529,51 @@ local function featuresprocessor(head,font,attr) end end else + local function subrun(start) + local head=start + local done=false + while start do + local id=getid(start) + if id==glyph_code and getfont(start)==font and getsubtype(start)<256 then + local a=getattr(start,0) + if a then + a=(a==attr) and (not attribute or getattr(start,a_state)==attribute) + else + a=not attribute or getattr(start,a_state)==attribute + end + if a then + for i=1,ns do + local lookupname=subtables[i] + local lookupcache=lookuphash[lookupname] + if lookupcache then + local lookupmatch=lookupcache[getchar(start)] + if lookupmatch then + local ok + head,start,ok=handler(head,start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i) + if ok then + done=true + break + elseif not start then + break + end + end + else + report_missing_cache(typ,lookupname) + end + end + if start then start=getnext(start) end + else + start=getnext(start) + end + else + start=getnext(start) + end + end + if done then + success=true + return head + end + end while start do local id=getid(start) if id==glyph_code then @@ -11475,6 +11611,25 @@ local function featuresprocessor(head,font,attr) else start=getnext(start) end + elseif id==disc_code then + if getsubtype(start)==discretionary_code then + local pre=getfield(start,"pre") + if pre then + local new=subrun(pre) + if new then setfield(start,"pre",new) end + end + local post=getfield(start,"post") + if post then + local new=subrun(post) + if new then setfield(start,"post",new) end + end + local replace=getfield(start,"replace") + if replace then + local new=subrun(replace) + if new then setfield(start,"replace",new) end + end + end + start=getnext(start) elseif id==whatsit_code then local subtype=getsubtype(start) if subtype==dir_code then @@ -12635,6 +12790,7 @@ if not modules then modules={} end modules ['font-def']={ local format,gmatch,match,find,lower,gsub=string.format,string.gmatch,string.match,string.find,string.lower,string.gsub local tostring,next=tostring,next local lpegmatch=lpeg.match +local suffixonly,removesuffix=file.suffix,file.removesuffix local allocate=utilities.storage.allocate local trace_defining=false trackers .register("fonts.defining",function(v) trace_defining=v end) local directive_embedall=false directives.register("fonts.embedall",function(v) directive_embedall=v end) @@ -12724,10 +12880,11 @@ definers.resolvers=definers.resolvers or {} local resolvers=definers.resolvers function resolvers.file(specification) local name=resolvefile(specification.name) - local suffix=file.suffix(name) + local suffix=lower(suffixonly(name)) if fonts.formats[suffix] then specification.forced=suffix - specification.name=file.removesuffix(name) + specification.forcedname=name + specification.name=removesuffix(name) else specification.name=name end @@ -12739,10 +12896,11 @@ function resolvers.name(specification) if resolved then specification.resolved=resolved specification.sub=sub - local suffix=file.suffix(resolved) + local suffix=lower(suffixonly(resolved)) if fonts.formats[suffix] then specification.forced=suffix - specification.name=file.removesuffix(resolved) + specification.forcedname=resolved + specification.name=removesuffix(resolved) else specification.name=resolved end @@ -12758,8 +12916,9 @@ function resolvers.spec(specification) if resolved then specification.resolved=resolved specification.sub=sub - specification.forced=file.suffix(resolved) - specification.name=file.removesuffix(resolved) + specification.forced=lower(suffixonly(resolved)) + specification.forcedname=resolved + specification.name=removesuffix(resolved) end else resolvers.name(specification) @@ -12774,8 +12933,7 @@ function definers.resolve(specification) end if specification.forced=="" then specification.forced=nil - else - specification.forced=specification.forced + specification.forcedname=nil end specification.hash=lower(specification.name..' @ '..constructors.hashfeatures(specification)) if specification.sub and specification.sub~="" then @@ -12820,7 +12978,7 @@ function definers.loadfont(specification) if not tfmdata then local forced=specification.forced or "" if forced~="" then - local reader=readers[lower(forced)] + local reader=readers[lower(forced)] tfmdata=reader and reader(specification) if not tfmdata then report_defining("forced type %a of %a not found",forced,specification.name) -- cgit v1.2.3 From 74e0f21a51d805f9599dee98a6fe3804e4edee06 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 00:38:36 +0100 Subject: [main] import changes to luaotfload.lua into dtx --- luaotfload.dtx | 60 +++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/luaotfload.dtx b/luaotfload.dtx index d1ed313..51c4e7e 100644 --- a/luaotfload.dtx +++ b/luaotfload.dtx @@ -1904,8 +1904,11 @@ loadmodule"colors.lua" --- “font-clr” % % \begin{macrocode} +local filesuffix = file.suffix +local fileremovesuffix = file.removesuffix local request_resolvers = fonts.definers.resolvers -local formats = fonts.formats -- nice table; does lowercasing ... +local formats = fonts.formats +local names = fonts.names formats.ofm = "type1" % \end{macrocode} @@ -1922,24 +1925,28 @@ formats.ofm = "type1" % With the release version 2.2 the file names are indexed in the database % as well and we are ready to resolve \verb|file:| lookups this way. % Thus we no longer need to call the \identifier{kpathsea} library in -% most cases when looking up font files, only when generating the database. +% most cases when looking up font files, only when generating the database, +% and when verifying the existence of a file in the \fileent{texmf} tree. % % \begin{macrocode} -local resolvefile = fonts.names.crude_file_lookup ---local resolvefile = fonts.names.crude_file_lookup_verbose +local resolve_file = names.crude_file_lookup +--local resolve_file = names.crude_file_lookup_verbose +local resolve_name = names.resolve_name -request_resolvers.file = function (specification) - local name = resolvefile(specification.name) - local suffix = file.suffix(name) +local file_resolver = function (specification) + local name = resolve_file (specification.name) + local suffix = filesuffix(name) if formats[suffix] then - specification.forced = suffix - specification.name = file.removesuffix(name) + specification.forced = suffix + specification.forcedname = file.removesuffix(name) else specification.name = name end end +request_resolvers.file = file_resolver + % \end{macrocode} % We classify as \verb|anon:| those requests that have neither a % prefix nor brackets. According to Khaled\footnote{% @@ -1974,7 +1981,8 @@ request_resolvers.anon = function (specification) for i=1, #type1_formats do local format = type1_formats[i] if resolvers.findfile(name, format) then - specification.name = file.addsuffix(name, format) + specification.forcedname = file.addsuffix(name, format) + specification.forced = format return end end @@ -2011,9 +2019,9 @@ request_resolvers.path = function (specification) logs.names_report("log", 1, "load", "path lookup of %q unsuccessful, falling back to file:", name) - request_resolvers.file(specification) + file_resolver (specification) else - local suffix = file.suffix(name) + local suffix = filesuffix (name) if formats[suffix] then specification.forced = suffix specification.name = file.removesuffix(name) @@ -2032,12 +2040,12 @@ end request_resolvers.kpse = function (specification) local name = specification.name - local suffix = file.suffix(name) + local suffix = filesuffix(name) if suffix and formats[suffix] then name = file.removesuffix(name) if resolvers.findfile(name, suffix) then - specification.forced = suffix - specification.name = name + specification.forced = suffix + specification.forcedname = name return end end @@ -2050,6 +2058,28 @@ request_resolvers.kpse = function (specification) end end +% \end{macrocode} +% The \verb|name:| resolver wraps the database function +% \luafunction{resolve_name}. +% +% \begin{macrocode} + +--- fonts.names.resolvers.name -- Customized version of the +--- generic name resolver. + +request_resolvers.name = function (specification) + local resolved, subfont = resolve_name (specification) + if resolved then + specification.resolved = resolved + specification.sub = subfont + specification.forced = filesuffix (resolved) + specification.forcedname = resolved + specification.name = fileremovesuffix (resolved) + else + file_resolver (specification) + end +end + % \end{macrocode} % Also {\bfseries EXPERIMENTAL}: % custom file resolvers via callback. -- cgit v1.2.3 From 847b5a1a519b3945a1734f5859a80d459e865457 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 15:07:11 +0100 Subject: [db,tool] adapt --find to new db model --- luaotfload-database.lua | 72 ++++++++++++++++++++++++++++++++++--------------- luaotfload-tool.lua | 2 +- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 390c3ea..1a79c3e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -140,6 +140,16 @@ local noncomma = 1-comma local splitcomma = Ct((C(noncomma^1) + comma)^1) patterns.splitcomma = splitcomma +local format_precedence = { + "otf", "ttc", "ttf", + "dfont", "afm", "pfb", + "pfa", +} + +local location_precedence = { + "local", "system", "texmf", +} + --[[doc-- We use the functions in the cache.* namespace that come with the fontloader (see luat-basics-gen). it’s safe to use for the most part @@ -614,23 +624,51 @@ crude_file_lookup_verbose = function (filename) return filename, nil, false end +local lookup_filename = function (filename) + if not name_index then name_index = load_names () end + local files = name_index.files + local basedata = files.base + local baredata = files.bare + for i = 1, #location_precedence do + local location = location_precedence [i] + local basenames = basedata [location] + local barenames = baredata [location] + local idx + if basenames ~= nil then + idx = basenames [filename] + if idx then + goto done + end + end + if barenames ~= nil then + for j = 1, #format_precedence do + local format = format_precedence [j] + local filemap = barenames [format] + if filemap then + idx = barenames [format] [filename] + if idx then + break + end + end + end + end +::done:: + if idx then + return files.full [idx] + end + end +end + --- string -> (string * string * bool) crude_file_lookup = function (filename) - if not name_index then name_index = load_names() end - local mappings = name_index.mappings - local files = name_index.files - - local found + local found = lookup_filename (filename) - found = files.base[filename] - or files.bare[filename] + if not found then + found = dummy_findfile(filename) + end if found then - found = files.full[found] - if found == nil then - found = dummy_findfile(filename) - end - return found or filename, nil, true + return found, nil, true end for i=1, #type1_formats do @@ -1182,16 +1220,6 @@ local choose_size = function (sizes, askedsize) return match end -local format_precedence = { - "otf", "ttc", "ttf", - "dfont", "afm", "pfb", - "pfa", -} - -local location_precedence = { - "local", "system", "texmf", -} - --[[doc-- resolve_familyname -- Query the families table for an entry diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 7c2beda..f9071d9 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -837,7 +837,7 @@ actions.query = function (job) if tmpspec.lookup == "name" or tmpspec.lookup == "anon" --- not *exactly* as resolvers.anon then - foundname, subfont = names.resolve (nil, nil, tmpspec) + foundname, subfont = names.resolve_name (tmpspec) if foundname then foundname, _, success = names.crude_file_lookup (foundname) end -- cgit v1.2.3 From f07090bcb38819badfbef38e0f929aab57ee8501 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 15:49:19 +0100 Subject: [tool] adapt ``--list`` --- luaotfload-tool.lua | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index f9071d9..29d3a43 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -138,6 +138,7 @@ config.lualibs.load_extended = true require "lualibs" local tabletohash = table.tohash +local stringsplit = string.split --[[doc-- \fileent{luatex-basics-gen.lua} calls functions from the @@ -882,14 +883,19 @@ end local get_fields get_fields = function (entry, fields, acc, n) if not acc then - return get_fields(entry, fields, { }, 1) + return get_fields (entry, fields, { }, 1) end - local field = fields[n] + local field = fields [n] if field then - local value = entry[field] + local chain = stringsplit (field, "->") + local tmp = entry + for i = 1, #chain - 1 do + tmp = tmp [chain [i]] + end + local value = tmp [chain [#chain]] acc[#acc+1] = value or false - return get_fields(entry, fields, acc, n+1) + return get_fields (entry, fields, acc, n+1) end return acc end @@ -940,7 +946,7 @@ actions.list = function (job) asked_fields = lpegmatch(splitcomma, asked_fields) else --- some defaults - asked_fields = { "fullname", "version", } + asked_fields = { "names->fullname", "version", } end if not name_index then @@ -1019,6 +1025,8 @@ actions.list = function (job) end end + texiowrite_nl "" + return true, true end -- cgit v1.2.3 From 80747ab46772d786b2adf066cd2db0ede11457bb Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 15:51:36 +0100 Subject: [tool] fall back to defaults with empty --fields query --- luaotfload-tool.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 29d3a43..3d149d0 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -944,7 +944,9 @@ actions.list = function (job) if asked_fields then asked_fields = lpegmatch(splitcomma, asked_fields) - else + end + + if not asked_fields then --- some defaults asked_fields = { "names->fullname", "version", } end -- cgit v1.2.3 From 711249d64487c3a0bde0ba510e11dbf46ebfb3cb Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 15:58:15 +0100 Subject: [man] update info regarding --fields --- luaotfload-tool.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst index 4ccc1dd..d123e04 100644 --- a/luaotfload-tool.rst +++ b/luaotfload-tool.rst @@ -127,7 +127,12 @@ query mode matches ``value``. --fields=FIELDS Comma-separated list of fields that should be - printed. The default is *fullname,version*. + printed. + Subfields of a record can be addressed using + the ``->`` separator, e. g. + ``file->location``, ``style->units_per_em``, + or ``names->sanitized->english->subfamily``. + The default is *names->fullname,version*. (Only meaningful with ``--list``.) font and lookup caches -- cgit v1.2.3 From bf255ef48e5b7710008141e4935183d8ca9a42aa Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 16:05:54 +0100 Subject: [tool] adapt --list queries with criterion --- luaotfload-tool.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 3d149d0..17d9399 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -993,7 +993,12 @@ actions.list = function (job) local categories, by_category = { }, { } for i=1, nmappings do local entry = mappings[i] - local value = entry[criterion] + local tmp = entry + local chain = stringsplit (criterion, "->") + for i = 1, #chain - 1 do + tmp = tmp [chain [i]] + end + local value = tmp [chain [#chain]] if value then --value = tostring(value) local entries = by_category[value] -- cgit v1.2.3 From 0228ec5a8a6d969c94cc2f5b7befd0cdb17a3c63 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 16:12:43 +0100 Subject: [man] expand paragraph about --list --- luaotfload-tool.rst | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst index d123e04..6971163 100644 --- a/luaotfload-tool.rst +++ b/luaotfload-tool.rst @@ -120,12 +120,37 @@ query mode 1) the character ``*``, selecting all entries; 2) a field of a database entry, for instance - *fullname* or *units_per_em*, according to - which the output will be sorted; or + *version* or *format**, according to which + the output will be sorted; or + Subfields of a record can be addressed using + the ``->`` separator, e. g. + ``file->location``, ``style->units_per_em``, + or + ``names->sanitized->english->prefmodifiers``. + NB: shell syntax requires that arguments + containing ``->`` be properly quoted! 3) an expression of the form ``field:value`` to limit the output to entries whose ``field`` matches ``value``. + For example, in order to output file names and + corresponding versions, sorted by the font + format:: + + ./luaotfload-tool.lua --list="format" --fields="file->base,version" + + This prints:: + + otf latinmodern-math.otf Version 1.958 + otf lmromancaps10-oblique.otf 2.004 + otf lmmono8-regular.otf 2.004 + otf lmmonoproplt10-bold.otf 2.004 + otf lmsans10-oblique.otf 2.004 + otf lmromanslant8-regular.otf 2.004 + otf lmroman12-italic.otf 2.004 + otf lmsansdemicond10-oblique.otf 2.004 + ... + --fields=FIELDS Comma-separated list of fields that should be printed. Subfields of a record can be addressed using -- cgit v1.2.3 From 8fbfd38b52e4e4c6381d10610211a5e66140c0c1 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 16:17:45 +0100 Subject: [tool] protect --list and --field against invalid queries --- luaotfload-tool.lua | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 17d9399..3bae175 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -892,9 +892,17 @@ local get_fields get_fields = function (entry, fields, acc, n) local tmp = entry for i = 1, #chain - 1 do tmp = tmp [chain [i]] + if not tmp then + --- invalid field + break + end + end + if tmp then + local value = tmp [chain [#chain]] + acc[#acc+1] = value or false + else + acc[#acc+1] = false end - local value = tmp [chain [#chain]] - acc[#acc+1] = value or false return get_fields (entry, fields, acc, n+1) end return acc @@ -997,8 +1005,11 @@ actions.list = function (job) local chain = stringsplit (criterion, "->") for i = 1, #chain - 1 do tmp = tmp [chain [i]] + if not tmp then + break + end end - local value = tmp [chain [#chain]] + local value = tmp and tmp [chain [#chain]] or "" if value then --value = tostring(value) local entries = by_category[value] -- cgit v1.2.3 From 653e9f10b170aff798cc9348d41bae472d7414f0 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 16:36:53 +0100 Subject: [main] fix afm lookup --- luaotfload.dtx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/luaotfload.dtx b/luaotfload.dtx index 51c4e7e..e19cdc3 100644 --- a/luaotfload.dtx +++ b/luaotfload.dtx @@ -1911,6 +1911,8 @@ local formats = fonts.formats local names = fonts.names formats.ofm = "type1" +fonts.encodings.known = fonts.encodings.known or { } + % \end{macrocode} % \identifier{luaotfload} promises easy access to system fonts. % Without additional precautions, this cannot be achieved by -- cgit v1.2.3 From 58acd9e4271e17cd9d8073d4de69f6f68048adb4 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 17:07:38 +0100 Subject: [db] add font name resolver --- luaotfload-database.lua | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1a79c3e..5c60046 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1265,10 +1265,22 @@ local resolve_familyname = function (specification, name, style, askedsize) return resolved, subfont end -local resolve_fontname = function (specification, name, style, askedsize) - local resolved, subfont - --[[ TODO ]] - return resolved, subfont +local resolve_fontname = function (specification, name) + local mappings = name_index.mappings + for i = 1, #mappings do + local face = mappings [i] + local sanitized = face.names.sanitized + local info = sanitized.info + local english = sanitized.english + if info.fontname == name + or info.fullname == name + or english.psname == name + or english.fullname == name + then + return face.file.base, face.file.subfont + end + end + return nil, nil end --[[doc-- @@ -1302,7 +1314,7 @@ resolve_name = function (specification) resolved, subfont = resolve_familyname (specification, name, style, askedsize) if not resolved then - resolved, subfont = resolve_fontname (specification, name, style, askedsize) + resolved, subfont = resolve_fontname (specification, name) end if not resolved then resolved = specification.name, false -- cgit v1.2.3 From bbb62449eb4d64284ec6f9c70663b43cb4945939 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 17:15:26 +0100 Subject: [db] add further synonyms for regular --- luaotfload-database.lua | 128 ++++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 5c60046..aaba68d 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -521,57 +521,65 @@ load_lookups = function ( ) return data end -local style_synonyms = { set = { } } -do - local combine = function (ta, tb) - local result = { } - for i=1, #ta do - for j=1, #tb do - result[#result+1] = ta[i] .. tb[j] - end - end - return result - end - - --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html - --- tl;dr: font style synonyms are unreliable. - --- - --- Context matches font names against lists of known identifiers - --- for weight, style, width, and variant, so that including - --- the family name there are five dimensions for choosing a - --- match. The sad thing is, while this is a decent heuristic it - --- makes no sense to imitate it in luaotfload because the user - --- interface must fit into the much more limited Xetex scheme that - --- distinguishes between merely four style categories (variants): - --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, - --- some of the styles are lumped together although they can differ - --- significantly (like “medium” and “bold”). - - --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, - --- “normal”, and “roman” as synonyms for “regular”. - local list = { - regular = { "normal", "roman", - "plain", "book", - "light", "extralight", - "ultralight", }, - bold = { "demi", "demibold", - "semibold", "boldregular", - "medium", "mediumbold", - "ultrabold", "extrabold", - "heavy", "black", - "bold", }, - italic = { "regularitalic", "normalitalic", - "oblique", "slanted", - "italic", }, - } - - list.bolditalic = combine(list.bold, list.italic) - style_synonyms.list = list - - for category, synonyms in next, style_synonyms.list do - style_synonyms.set[category] = tabletohash(synonyms, true) - end -end +--local style_synonyms = { set = { } } +--do +-- local combine = function (ta, tb) +-- local result = { } +-- for i=1, #ta do +-- for j=1, #tb do +-- result[#result+1] = ta[i] .. tb[j] +-- end +-- end +-- return result +-- end +-- +-- --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html +-- --- tl;dr: font style synonyms are unreliable. +-- --- +-- --- Context matches font names against lists of known identifiers +-- --- for weight, style, width, and variant, so that including +-- --- the family name there are five dimensions for choosing a +-- --- match. The sad thing is, while this is a decent heuristic it +-- --- makes no sense to imitate it in luaotfload because the user +-- --- interface must fit into the much more limited Xetex scheme that +-- --- distinguishes between merely four style categories (variants): +-- --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, +-- --- some of the styles are lumped together although they can differ +-- --- significantly (like “medium” and “bold”). +-- +-- --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, +-- --- “normal”, and “roman” as synonyms for “regular”. +-- local list = { +-- regular = { "normal", "roman", +-- "plain", "book", +-- "light", "extralight", +-- "ultralight", }, +-- bold = { "demi", "demibold", +-- "semibold", "boldregular", +-- "medium", "mediumbold", +-- "ultrabold", "extrabold", +-- "heavy", "black", +-- "bold", }, +-- italic = { "regularitalic", "normalitalic", +-- "oblique", "slanted", +-- "italic", }, +-- } +-- +-- list.bolditalic = combine(list.bold, list.italic) +-- style_synonyms.list = list +-- +-- for category, synonyms in next, style_synonyms.list do +-- style_synonyms.set[category] = tabletohash(synonyms, true) +-- end +--end + +local regular_synonym = { + book = true, + normal = true, + plain = true, + regular = true, + roman = true, +} local type1_formats = { "tfm", "ofm", } @@ -2908,19 +2916,11 @@ do subfamily, splitstyle) - if fontstyle_name == "regular" or fontstyle_name == "book" then - return "r" - end - - if prefmodifiers == "regular" or prefmodifiers == "book" then - return "r" - end - - if subfamily == "regular" or subfamily == "book" then - return "r" - end - - if splitstyle == "regular" or splitstyle == "book" then + if regular_synonym [fontstyle_name] + or regular_synonym [prefmodifiers] + or regular_synonym [subfamily] + or regular_synonym [splitstyle] + then return "r" end -- cgit v1.2.3 From 36129aa2413bf76f7d9663e8de505e00eee5065f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 18:17:58 +0100 Subject: [db] adapt fuzzy matcher --- luaotfload-database.lua | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index aaba68d..c3256f0 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1421,12 +1421,13 @@ find_closest = function (name, limit) local by_distance = { } --- (int, string list) dict local distances = { } --- int list local cached = { } --- (string, int) dict - local mappings = data.mappings + local mappings = name_index.mappings local n_fonts = #mappings for n = 1, n_fonts do local current = mappings[n] - local cnames = current.sanitized + local names = current.names + local sanitized = names.sanitized --[[ This is simplistic but surpisingly fast. Matching is performed against the “fullname” field @@ -1436,10 +1437,13 @@ find_closest = function (name, limit) font name categories as well as whatever agrep does. --]] - if cnames then - local fullname, sfullname = current.fullname, cnames.fullname + if names then + local fullname = names.fullname + local sfullname = sanitized.info.fullname + or sanitized.english.fullname + or sanitized.metadata.fullname + local dist = cached[sfullname]--- maybe already calculated - local dist = cached[sfullname]--- maybe already calculated if not dist then dist = iterative_levenshtein(name, sfullname) cached[sfullname] = dist @@ -1589,10 +1593,6 @@ local organize_namedata = function (metadata, }, } --- print (fontnames.english.fullname, --- fontnames.metadata.fullname, --- fontnames.info.fullname) - if metadata.fontstyle_name then --- not present in all fonts, often differs from the preferred --- subfamily as well as subfamily fields, e.g. with LMSans10-BoldOblique: -- cgit v1.2.3 From e6613fb70ec97494b72970f9a53c10c69759dcd9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 18:38:04 +0100 Subject: [db] thow away obsolete name resolver Killing off one of the last remainders from the pre-TL2013 luaotfload. Good riddance ;-) --- luaotfload-database.lua | 310 ++++++------------------------------------------ 1 file changed, 38 insertions(+), 272 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index c3256f0..fec4efa 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -875,278 +875,6 @@ local add_to_match = function (found, size, face) return found, continue end ---[[doc-- - -Luatex-fonts, the font-loader package luaotfload imports, comes with -basic file location facilities (see luatex-fonts-syn.lua). -However, not only does the builtin functionality rely on Context’s font -name database, it is also too limited to be of more than basic use. -For this reason, luaotfload supplies its own resolvers that accesses -the font database created by the luaotfload-tool script. - ---doc]]-- - - ---- ---- the request specification has the fields: ---- ---- · features: table ---- · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig } ---- · ??? ---- · forced: string ---- · lookup: "name" ---- · method: string ---- · name: string ---- · resolved: string ---- · size: int ---- · specification: string (== ":" ) ---- · sub: string ---- ---- The “size” field deserves special attention: if its value is ---- negative, then it actually specifies a scalefactor of the ---- design size of the requested font. This happens e.g. if a font is ---- requested without an explicit “at size”. If the font is part of a ---- larger collection with different design sizes, this complicates ---- matters a bit: Normally, the resolver prefers fonts that have a ---- design size as close as possible to the requested size. If no ---- size specified, then the design size is implied. But which design ---- size should that be? Xetex appears to pick the “normal” (unmarked) ---- size: with Adobe fonts this would be the one that is neither ---- “caption” nor “subhead” nor “display” &c ... For fonts by Adobe this ---- seems to be the one that does not receive a “prefmodifiers” field. ---- (IOW Adobe uses the “prefmodifiers” field to encode the design size ---- in more or less human readable format.) However, this is not true ---- of LM and EB Garamond. As this matters only where there are ---- multiple design sizes to a given font/style combination, we put a ---- workaround in place that chooses that unmarked version. - ---- ---- the first return value of “resolve” is the file name of the ---- requested font (string) ---- the second is of type bool or string and indicates the subfont of a ---- ttc ---- ---- 'a -> 'a -> table -> (string * string | bool * bool) ---- - ---[===[ -resolve = function (_, _, specification) -- the 1st two parameters are used by ConTeXt - if not fonts_loaded then names.data = load_names() end - local data = names.data - - local name = sanitize_fontname (specification.name) - local style = sanitize_fontname (specification.style) or "regular" - - local askedsize - - if specification.optsize then - askedsize = tonumber(specification.optsize) - else - local specsize = specification.size - if specsize and specsize >= 0 then - askedsize = specsize / 65536 - end - end - - if type(data) ~= "table" then - --- this catches a case where load_names() doesn’t - --- return a database object, which can happen only - --- in case there is valid Lua code in the database, - --- but it’s not a table, e.g. it contains an integer. - if not fonts_reloaded then - return reload_db("invalid database; not a table", - resolve, nil, nil, specification) - end - --- unsucessfully reloaded; bail - return specification.name, false, false - end - - if not data.mappings then - if not fonts_reloaded then - return reload_db("invalid database; missing font mapping", - resolve, nil, nil, specification) - end - return specification.name, false, false - end - - local synonym_set = style_synonyms.set - local stylesynonyms = synonym_set[style] - local regularsynonyms = synonym_set.regular - - local exact = { } --> collect exact style matches - local synonymous = { } --> collect matching style synonyms - local fallback --> e.g. non-matching style (fontspec is anal about this) - local candidates = { } --> secondary results, incomplete matches - - for n, face in next, data.mappings do - local family, metafamily - local prefmodifiers, fontstyle_name, subfamily - local psname, fullname, fontname, pfullname - - local facenames = face.sanitized - if facenames then - family = facenames.family - subfamily = facenames.subfamily - fontstyle_name = facenames.fontstyle_name - prefmodifiers = facenames.prefmodifiers or fontstyle_name or subfamily - fullname = facenames.fullname - psname = facenames.psname - fontname = facenames.fontname - pfullname = facenames.pfullname - metafamily = facenames.metafamily - end - fontname = fontname or sanitize_fontname (face.fontname) - pfullname = pfullname or sanitize_fontname (face.fullname) - - if name == family - or name == metafamily - then - if style == prefmodifiers - or style == fontstyle_name - then - local continue - exact, continue = add_to_match(exact, askedsize, face) - if continue == false then break end - elseif style == subfamily then - exact = add_to_match(exact, askedsize, face) - elseif stylesynonyms and stylesynonyms[prefmodifiers] - or regularsynonyms[prefmodifiers] - then - --- treat synonyms for prefmodifiers as first-class - --- (needed to prioritize DejaVu Book over Condensed) - exact = add_to_match(exact, askedsize, face) - elseif name == fullname - or name == pfullname - or name == fontname - or name == psname - then - synonymous = add_to_match(synonymous, askedsize, face) - elseif stylesynonyms and stylesynonyms[subfamily] - or regularsynonyms[subfamily] - then - synonymous = add_to_match(synonymous, askedsize, face) - elseif prefmodifiers == "regular" - or subfamily == "regular" then - fallback = face - else --- mark as last straw but continue - candidates[#candidates+1] = face - end - else - if name == fullname - or name == pfullname - or name == fontname - or name == psname then - local continue - exact, continue = add_to_match(exact, askedsize, face) - if continue == false then break end - end - end - end - - local found - if next(exact) then - found = exact - else - found = synonymous - end - - --- this is a monster - if #found == 1 then - --- “found” is really synonymous with “registered in the db”. - local entry = found[1] - local success, filename, subfont - = get_font_file(data.files.full, entry) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - - elseif #found > 1 then - -- we found matching font(s) but not in the requested optical - -- sizes, so we loop through the matches to find the one with - -- least difference from the requested size. - local match - - if askedsize then --- choose by design size - local closest - local least = math.huge -- initial value is infinity - - for i, face in next, found do - local dsnsize = face.size and face.size [1] or 0 - local difference = mathabs (dsnsize - askedsize) - if difference < least then - closest = face - least = difference - end - end - - match = closest - else --- choose “unmarked” match, for Adobe fonts this - --- is the one without a “prefmodifiers” field. - match = found [1] --- fallback - for i, face in next, found do - if not face.sanitized.prefmodifiers then - match = face - break - end - end - end - - local success, filename, subfont - = get_font_file(data.files.full, match) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - - elseif fallback then - local success, filename, subfont - = get_font_file(data.files.full, fallback) - if success == true then - report("log", 0, "resolve", - "No exact match for request %s; using fallback", - specification.specification - ) - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - elseif next(candidates) then - --- pick the first candidate encountered - local entry = candidates[1] - local success, filename, subfont - = get_font_file(data.files.full, entry) - if success == true then - report("log", 0, "resolve", - "Font family='%s', subfamily='%s' found: %s", - name, style, filename - ) - return filename, subfont, true - end - end - - --- no font found so far - if not fonts_reloaded then - --- last straw: try reloading the database - return reload_db( - "unresolved font name: '" .. name .. "'", - resolve, nil, nil, specification - ) - end - - --- else, default to requested name - return specification.name, false, false -end --- resolve() -]===] - local choose_closest = function (distances) local closest = 2^51 local match @@ -1299,6 +1027,44 @@ end The return value is a pair consisting of the file name and the subfont index if appropriate.. + the request specification has the fields: + + · features: table + · normal: set of { ccmp clig itlc kern liga locl mark mkmk rlig } + · ??? + · forced: string + · lookup: "name" + · method: string + · name: string + · resolved: string + · size: int + · specification: string (== ":" ) + · sub: string + + The “size” field deserves special attention: if its value is + negative, then it actually specifies a scalefactor of the + design size of the requested font. This happens e.g. if a font is + requested without an explicit “at size”. If the font is part of a + larger collection with different design sizes, this complicates + matters a bit: Normally, the resolver prefers fonts that have a + design size as close as possible to the requested size. If no + size specified, then the design size is implied. But which design + size should that be? Xetex appears to pick the “normal” (unmarked) + size: with Adobe fonts this would be the one that is neither + “caption” nor “subhead” nor “display” &c ... For fonts by Adobe this + seems to be the one that does not receive a “prefmodifiers” field. + (IOW Adobe uses the “prefmodifiers” field to encode the design size + in more or less human readable format.) However, this is not true + of LM and EB Garamond. As this matters only where there are + multiple design sizes to a given font/style combination, we put a + workaround in place that chooses that unmarked version. + + The first return value of “resolve_name” is the file name of the + requested font (string). It can be passed to the fullname resolver + get_font_file(). + The second value is either “false” or an integer indicating the + subfont index in a TTC. + --doc]]-- --- table -> string * (int | bool) -- cgit v1.2.3 From de6ef7d6566f53b7554bf3861bb10712417b83fd Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 19:54:25 +0100 Subject: [db] strip extraneous information from mappings table --- luaotfload-database.lua | 171 ++++++++++++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 64 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index fec4efa..4632626 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -99,7 +99,8 @@ local name_index = nil -- upvalue for names.data local luaotfloadconfig = config.luaotfload --- always present luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" -luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" +luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" +luaotfloadconfig.strip = luaotfloadconfig.strip == true if luaotfloadconfig.update_live ~= false then --- this option allows for disabling updates @@ -699,16 +700,15 @@ local get_font_file = function (index) if not entry then return false end - local filedata = entry.file - local basename = filedata.base - if filedata.location == "texmf" then + local basename = entry.basename + if entry.location == "texmf" then if kpselookup(basename) then - return true, basename, filedata.subfont + return true, basename, entry.subfont end else --- system, local local fullname = name_index.files.full [index] if lfsisfile (fullname) then - return true, basename, filedata.subfont + return true, basename, entry.subfont end end return false @@ -1004,16 +1004,12 @@ end local resolve_fontname = function (specification, name) local mappings = name_index.mappings for i = 1, #mappings do - local face = mappings [i] - local sanitized = face.names.sanitized - local info = sanitized.info - local english = sanitized.english - if info.fontname == name - or info.fullname == name - or english.psname == name - or english.fullname == name + local face = mappings [i] + if face.fontname == name + or face.fullname == name + or face.psname == name then - return face.file.base, face.file.subfont + return face.basename, face.subfont end end return nil, nil @@ -1192,8 +1188,6 @@ find_closest = function (name, limit) for n = 1, n_fonts do local current = mappings[n] - local names = current.names - local sanitized = names.sanitized --[[ This is simplistic but surpisingly fast. Matching is performed against the “fullname” field @@ -1203,26 +1197,22 @@ find_closest = function (name, limit) font name categories as well as whatever agrep does. --]] - if names then - local fullname = names.fullname - local sfullname = sanitized.info.fullname - or sanitized.english.fullname - or sanitized.metadata.fullname - local dist = cached[sfullname]--- maybe already calculated - - if not dist then - dist = iterative_levenshtein(name, sfullname) - cached[sfullname] = dist - end - local namelst = by_distance[dist] - if not namelst then --- first entry - namelst = { fullname } - distances[#distances+1] = dist - else --- append - namelst[#namelst+1] = fullname - end - by_distance[dist] = namelst + local fullname = current.plainname + local sfullname = current.fullname + local dist = cached[sfullname]--- maybe already calculated + + if not dist then + dist = iterative_levenshtein(name, sfullname) + cached[sfullname] = dist end + local namelst = by_distance[dist] + if not namelst then --- first entry + namelst = { fullname } + distances[#distances+1] = dist + else --- append + namelst[#namelst+1] = fullname + end + by_distance[dist] = namelst end --- print the matches according to their distance @@ -2507,14 +2497,29 @@ local generate_filedata = function (mappings) local entry = mappings [index] local filedata = entry.file + local format + local location + local fullpath + local basename + local barename + local subfont + + if filedata then --- new entry + format = entry.format --- otf, afm, ... + location = filedata.location --- texmf, system, ... + fullpath = filedata.full + basename = filedata.base + barename = filenameonly (fullpath) + subfont = filedata.subfont + else + format = entry.format --- otf, afm, ... + location = entry.location --- texmf, system, ... + fullpath = entry.fullpath + basename = entry.basename + barename = filenameonly (fullpath) + subfont = entry.subfont + end - local format = entry.format --- otf, afm, ... - local location = filedata.location --- texmf, system, ... - - local fullname = filedata.full - local basename = filedata.base - local barename = filenameonly (fullname) - local subfont = filedata.subfont entry.index = index @@ -2578,9 +2583,9 @@ local generate_filedata = function (mappings) bare [location] [format] = inbare end - --- 3) add to fullname map + --- 3) add to fullpath map - full [index] = fullname + full [index] = fullpath end --- TODO adapt to new mechanism! @@ -2694,6 +2699,46 @@ do end end +local pull_values = function (entry) + local file = entry.file + local names = entry.names + local style = entry.style + local sanitized = names.sanitized + local english = sanitized.english + local info = sanitized.info + + --- pull file info ... + entry.basename = file.base + entry.fullpath = file.full + entry.location = file.location + entry.subfont = file.subfont + + --- pull name info ... + entry.psname = english.psname + entry.fontname = info.fontname + entry.fullname = english.fullname or info.fullname + entry.familyname = english.preffamily + or english.family + or info.familyname + entry.fontstyle_name = sanitized.fontstyle_name + entry.plainname = names.fullname + entry.prefmodifiers = english.prefmodifiers + entry.subfamily = english.subfamily + + --- pull style info ... + entry.italicangle = style.italicangle + entry.size = style.size + entry.splitstyle = style.split + entry.weight = style.weight + + if luaotfloadconfig.strip == true then + --if false then + entry.file = nil + entry.names = nil + entry.style = nil + end +end + local collect_families = function (mappings) report ("info", 2, "db", "Analyzing families, sizes, and styles.") @@ -2706,15 +2751,14 @@ local collect_families = function (mappings) for i = 1, #mappings do - local entry = mappings [i] - local file = entry.file - local names = entry.names - local style = entry.style + local entry = mappings [i] + + if entry.file then + pull_values (entry) + end - local location = file.location + local location = entry.location local format = entry.format - local english = names.sanitized.english - local info = names.sanitized.info local subtable = families [location] [format] if not subtable then @@ -2722,19 +2766,17 @@ local collect_families = function (mappings) families [location] [format] = subtable end - local familyname = english.preffamily - or english.family - or info.familyname - local fullname = english.fullname or info.fullname - local fontname = info.fontname + --local fullname = english.fullname or info.fullname + --local fontname = info.fontname - local fontstyle_name = names.sanitized.fontstyle_name - local prefmodifiers = english.prefmodifiers - local subfamily = english.subfamily + local familyname = entry.familyname + local fontstyle_name = entry.fontstyle_name + local prefmodifiers = entry.prefmodifiers + local subfamily = entry.subfamily - local weight = style.weight - local italicangle = style.italicangle - local splitstyle = style.split + local weight = entry.weight + local italicangle = entry.italicangle + local splitstyle = entry.splitstyle local modifier = pick_style (fontstyle_name, prefmodifiers, @@ -2786,7 +2828,7 @@ local collect_families = function (mappings) styletable.default = entry.index end - local size = style.size --- dsnsize * high * low + local size = entry.size --- dsnsize * high * low if size then styletable [#styletable + 1] = { size [1], @@ -2800,6 +2842,7 @@ local collect_families = function (mappings) end end + collectgarbage "collect" return families end -- cgit v1.2.3 From e83bdce6f53216620ae959fd55ba7025569d7e44 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 20:07:34 +0100 Subject: [tool] add --no-strip option --- luaotfload-tool.lua | 7 +++++++ luaotfload-tool.rst | 6 +++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 3bae175..f7ac215 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -112,6 +112,9 @@ luaotfloadconfig.index_file = luaotfloadconfig.index_file luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" luaotfloadconfig.reload = false +if not luaotfloadconfig.strip then + luaotfloadconfig.strip = true +end do -- we don’t have file.basename and the likes yet, so inline parser ftw local slash = P"/" @@ -214,6 +217,7 @@ This tool is part of the luaotfload package. Valid options are: -u --update update the database -n --no-reload suppress db update + --no-strip keep redundant information in db -f --force force re-indexing all fonts -c --compress gzip index file (text version only) -l --flush-lookups empty lookup cache of font requests @@ -1109,6 +1113,7 @@ local process_cmdline = function ( ) -- unit -> jobspec list = 1, log = 1, ["no-reload"] = "n", + ["no-strip"] = 0, ["skip-read"] = "R", ["prefer-texmf"] = "p", quiet = "q", @@ -1208,6 +1213,8 @@ local process_cmdline = function ( ) -- unit -> jobspec luaotfloadconfig.skip_read = true elseif v == "c" then luaotfloadconfig.compress = true + elseif v == "no-strip" then + luaotfloadconfig.strip = false end end diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst index 6971163..2db8481 100644 --- a/luaotfload-tool.rst +++ b/luaotfload-tool.rst @@ -20,7 +20,7 @@ SYNOPSIS **luaotfload-tool** --update [ --force ] [ --quiet ] [ --verbose ] [ --prefer-texmf ] [ --dry-run ] [ --formats=[+|-]EXTENSIONS ] - [ --compress ] + [ --compress ] [ --no-strip ] **luaotfload-tool** --find=FONTNAME [ --fuzzy ] [ --info ] [ --inspect ] [ --no-reload ] @@ -63,6 +63,10 @@ update mode all fonts. --no-reload, -n Suppress auto-updates to the database (e.g. when ``--find`` is passed an unknown name). +--no-strip Do not strip redundant information after + building the database. Warning: this will + inflate the index to about two to three times + the normal size. --compress Filter plain text version of font index through gzip. -- cgit v1.2.3 From a59332596c26d57b84a609749f7b677683bde029 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 20:10:48 +0100 Subject: [tool,man] adapt default fields for --list and --fields to stripped index format --- luaotfload-tool.lua | 2 +- luaotfload-tool.rst | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index f7ac215..737485f 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -960,7 +960,7 @@ actions.list = function (job) if not asked_fields then --- some defaults - asked_fields = { "names->fullname", "version", } + asked_fields = { "plainname", "version", } end if not name_index then diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst index 2db8481..37ef779 100644 --- a/luaotfload-tool.rst +++ b/luaotfload-tool.rst @@ -125,7 +125,9 @@ query mode 1) the character ``*``, selecting all entries; 2) a field of a database entry, for instance *version* or *format**, according to which - the output will be sorted; or + the output will be sorted. + Information in an unstripped database (see + the option ``--no-strip`` above) is nested: Subfields of a record can be addressed using the ``->`` separator, e. g. ``file->location``, ``style->units_per_em``, @@ -157,11 +159,13 @@ query mode --fields=FIELDS Comma-separated list of fields that should be printed. + Information in an unstripped database (see the + option ``--no-strip`` above) is nested: Subfields of a record can be addressed using the ``->`` separator, e. g. ``file->location``, ``style->units_per_em``, or ``names->sanitized->english->subfamily``. - The default is *names->fullname,version*. + The default is plainname,version*. (Only meaningful with ``--list``.) font and lookup caches -- cgit v1.2.3 From c05e323e497d7d8e3befb371d6b78e1f91be83d9 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 20:21:44 +0100 Subject: [main] add setting for db stripping to luaotfload.lua --- luaotfload.dtx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/luaotfload.dtx b/luaotfload.dtx index e19cdc3..5d80318 100644 --- a/luaotfload.dtx +++ b/luaotfload.dtx @@ -1550,6 +1550,9 @@ config.luaotfload.names_dir = config.luaotfload.names_dir or "names config.luaotfload.cache_dir = config.luaotfload.cache_dir or "fonts" config.luaotfload.index_file = config.luaotfload.index_file or "luaotfload-names.lua" config.luaotfload.formats = config.luaotfload.formats or "otf,ttf,ttc,dfont" +if not config.luaotfload.strip then + config.luaotfload.strip = true +end luaotfload.module = { name = "luaotfload", -- cgit v1.2.3 From b697b1fa3322d1ce40cc6b9149363b15ebe6f9f7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 20:59:46 +0100 Subject: [db] fix damage suffered during last merge --- luaotfload-database.lua | 6 ------ 1 file changed, 6 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 1deca07..ad954a2 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2526,12 +2526,6 @@ local generate_filedata = function (mappings) barename = filenameonly (fullpath) subfont = entry.subfont end - end - - local addmap = function (lst) - --- this will overwrite existing entries - for i=1, #lst do - local idx, base, bare, intexmf, full = unpack(lst[i]) entry.index = index -- cgit v1.2.3 From bf11a971192a6d8cb6aa4b23d0a1c48fd74a2538 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 22:15:58 +0100 Subject: [db] adapt type1 scanner --- luaotfload-database.lua | 117 +++++++++++++++++------------------------------- 1 file changed, 41 insertions(+), 76 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index ad954a2..e93a299 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -1355,7 +1355,8 @@ local organize_namedata = function (metadata, -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size if metadata.fontstyle_name then --- not present in all fonts, often differs from the preferred - --- subfamily as well as subfamily fields, e.g. with LMSans10-BoldOblique: + --- subfamily as well as subfamily fields, e.g. with + --- LMSans10-BoldOblique: --- subfamily: “Bold Italic” --- prefmodifiers: “10 Bold Oblique” --- fontstyle_name: “Bold Oblique” @@ -1364,9 +1365,6 @@ local organize_namedata = function (metadata, fontnames.fontstyle_name = name.name end end - - --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", english_names.prefmodifiers) - --print (fontnames.metadata.fontname, "|>", english_names.subfamily, "<>", fontnames.fontstyle_name) end return { @@ -1391,14 +1389,13 @@ local split_fontname = function (fontname) end end -local organize_styledata = function (fontname, metadata, english_names, info) +local organize_styledata = function (fontname, + metadata, + english_names, + info) local pfminfo = metadata.pfminfo local names = metadata.names --- print (">", pfminfo.avgwidth, --- (metadata.italicangle == info.italicangle) and "T" or --- string.format ("%f / %f", metadata.italicangle, info.italicangle), --- pfminfo.width, pfminfo.weight, info.weight) return { -- see http://www.microsoft.com/typography/OTSPEC/features_pt.htm#size size = get_size_info (metadata), @@ -1474,8 +1471,8 @@ end --- string -> int -> bool -> string -> fontentry t1_fullinfo = function (filename, _subfont, location, basename, format) - local namedata = { } - local metadata = load_font_file (filename) + local sanitized + local metadata = load_font_file (filename) local fontname = metadata.fontname local fullname = metadata.fullname @@ -1483,68 +1480,35 @@ t1_fullinfo = function (filename, _subfont, location, basename, format) local italicangle = metadata.italicangle local weight = metadata.weight --- string identifier - --- we have to improvise and detect whether we deal with - --- italics since pfb fonts don’t come with a “subfamily” - --- field - local style - if italicangle == 0 then - style = false - else - style = "italic" - end - - local style_synonyms_set = style_synonyms.set - if weight then - weight = sanitize_fontname (weight) - local tmp = "" - if style_synonyms_set.bold[weight] then - tmp = "bold" - end - if style then - style = tmp .. style - else - if style_synonyms_set.regular[weight] then - style = "regular" - else - style = tmp - end - end - end - - if not style or style == "" then - style = "regular" - --- else italic - end - - namedata.sanitized = sanitize_fontnames ({ + sanitized = sanitize_fontnames ({ fontname = fontname, psname = fullname, pfullname = fullname, metafamily = family, - family = familyname, + familyname = familyname, subfamily = weight, prefmodifiers = style, }) - namedata.fontname = fontname - namedata.fullname = fullname - namedata.familyname = familyname - - namedata.slant = italicangle - namedata.units_per_em = 1000 --- ps fonts standard - namedata.version = metadata.version - namedata.weight = metadata.pfminfo.weight --- integer - namedata.width = metadata.pfminfo.width - - namedata.size = false - - namedata.filename = filename --> sys - namedata.basename = basename --> texmf - namedata.format = format - namedata.location = location or "system" - namedata.subfont = false - - return namedata + return { + basename = basename, + fullpath = filename, + subfont = false, + location = location or "system", + format = format, + fullname = sanitized.fullname, + fontname = sanitized.fontname, + familyname = sanitized.familyname, + plainname = fullname, + psname = sanitized.fontname, + version = metadata.version, + size = false, + splitstyle = split_fontname (fontname), + fontstyle_name = sanitized.subfamily, + weight = { metadata.pfminfo.weight, + sanitized.subfamily }, + italicangle = italicangle, + } end local loaders = { @@ -1698,7 +1662,6 @@ local read_font_names = function (fullname, local format = stringlower (filesuffix (basename)) local loader = loaders [format] --- ot_fullinfo, t1_fullinfo - local loader = loaders[format] --- ot_fullinfo, t1_fullinfo if not loader then report ("both", 0, "db", "Unknown format: %q, skipping.", format) @@ -2091,8 +2054,8 @@ local scan_dir = function (dirname, currentnames, targetnames, "Would have been loading %q", fullname) else report_status ("both", "db", "Loading font %q", fullname) - local new = load_font (fullname, fontnames, - newfontnames, texmf) + local new = read_font_names (fullname, currentnames, + targetnames, texmf) if new == true then n_new = n_new + 1 end @@ -2627,13 +2590,14 @@ do subfamily) if italicangle ~= 0 then return true - elseif fontstyle_name and lpegmatch (italic, fontstyle_name) then + elseif fontstyle_name ~= nil and lpegmatch (italic, fontstyle_name) then return true - elseif prefmodifiers and lpegmatch (italic, prefmodifiers) then + elseif prefmodifiers ~= nil and lpegmatch (italic, prefmodifiers) then return lpegmatch (italic, prefmodifiers) - else - return lpegmatch (italic, subfamily) + elseif subfamily ~= nil and lpegmatch (italic, subfamily) then + return true end + return false end determine_bold = function (fontstyle_name, @@ -2642,13 +2606,14 @@ do subfamily) if weight [2] == "bold" then return true - elseif fontstyle_name and lpegmatch (bold, fontstyle_name) then + elseif fontstyle_name ~= nil and lpegmatch (bold, fontstyle_name) then return true - elseif prefmodifiers and lpegmatch (bold, prefmodifiers) then + elseif prefmodifiers ~= nil and lpegmatch (bold, prefmodifiers) then + return true + elseif subfamily ~= nil and lpegmatch (bold, subfamily) then return true - else - return lpegmatch (bold, subfamily) end + return false end local splitfontname = lpeg.splitat "-" -- cgit v1.2.3 From e0a62d13f16573b7461269bb022516634b7fce3d Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 22:16:42 +0100 Subject: [db] throw away some cruft (old synonyms table) --- luaotfload-database.lua | 52 ------------------------------------------------- 1 file changed, 52 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e93a299..291abe5 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -525,58 +525,6 @@ load_lookups = function ( ) return data end ---local style_synonyms = { set = { } } ---do --- local combine = function (ta, tb) --- local result = { } --- for i=1, #ta do --- for j=1, #tb do --- result[#result+1] = ta[i] .. tb[j] --- end --- end --- return result --- end --- --- --- read this: http://blogs.adobe.com/typblography/2008/05/indesign_font_conflicts.html --- --- tl;dr: font style synonyms are unreliable. --- --- --- --- Context matches font names against lists of known identifiers --- --- for weight, style, width, and variant, so that including --- --- the family name there are five dimensions for choosing a --- --- match. The sad thing is, while this is a decent heuristic it --- --- makes no sense to imitate it in luaotfload because the user --- --- interface must fit into the much more limited Xetex scheme that --- --- distinguishes between merely four style categories (variants): --- --- “regular”, “italic”, “bold”, and “bolditalic”. As a result, --- --- some of the styles are lumped together although they can differ --- --- significantly (like “medium” and “bold”). --- --- --- Xetex (XeTeXFontMgr.cpp) appears to recognize only “plain”, --- --- “normal”, and “roman” as synonyms for “regular”. --- local list = { --- regular = { "normal", "roman", --- "plain", "book", --- "light", "extralight", --- "ultralight", }, --- bold = { "demi", "demibold", --- "semibold", "boldregular", --- "medium", "mediumbold", --- "ultrabold", "extrabold", --- "heavy", "black", --- "bold", }, --- italic = { "regularitalic", "normalitalic", --- "oblique", "slanted", --- "italic", }, --- } --- --- list.bolditalic = combine(list.bold, list.italic) --- style_synonyms.list = list --- --- for category, synonyms in next, style_synonyms.list do --- style_synonyms.set[category] = tabletohash(synonyms, true) --- end ---end - local regular_synonym = { book = true, normal = true, -- cgit v1.2.3 From 947a07ebf509f7961673fdfdafb0cf1ea6faec4a Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 22:22:37 +0100 Subject: [db,tool] adapt --prefer-texmf --- luaotfload-database.lua | 17 ++++++----------- luaotfload-tool.lua | 6 ++++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 291abe5..868541b 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -154,6 +154,10 @@ local location_precedence = { "local", "system", "texmf", } +local set_location_precedence = function (precedence) + location_precedence = precedence +end + --[[doc-- We use the functions in the cache.* namespace that come with the fontloader (see luat-basics-gen). it’s safe to use for the most part @@ -2505,16 +2509,6 @@ local generate_filedata = function (mappings) full [index] = fullpath end - --- TODO adapt to new mechanism! --- if luaotfloadconfig.prioritize == "texmf" then --- report("both", 2, "db", "Preferring texmf fonts") --- addmap(sys) --- addmap(texmf) --- else --- sys --- addmap(texmf) --- addmap(sys) --- end - return files end @@ -2530,7 +2524,7 @@ local check_regular do local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") - local bold = match_synonyms (P"bold" + P"demi", P"heavy", P"black", P"ultra") + local bold = match_synonyms (P"bold" + P"demi" + P"heavy" + P"black" + P"ultra") determine_italic = function (fontstyle_name, italicangle, @@ -3323,6 +3317,7 @@ names.crude_file_lookup_verbose = crude_file_lookup_verbose names.read_blacklist = read_blacklist names.sanitize_fontname = sanitize_fontname names.getfilename = resolve_fullpath +names.set_location_precedence = set_location_precedence --- font cache names.purge_cache = purge_cache diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index 4f084ae..0cc26d2 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -1195,8 +1195,10 @@ local process_cmdline = function ( ) -- unit -> jobspec result.cache = optarg[n] elseif v == "D" then result.dry_run = true - elseif v == "p" then --- TODO adapt to new db structure - luaotfloadconfig.prioritize = "texmf" + elseif v == "p" then + names.set_location_precedence { + "local", "texmf", "system" + } elseif v == "b" then action_pending["blacklist"] = true elseif v == "diagnose" then -- cgit v1.2.3 From 06af053c14846af44b3306b28a8443e11a559cfe Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Tue, 5 Nov 2013 23:24:50 +0100 Subject: [db] stop matching bold synonyms as bold --- luaotfload-database.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 868541b..84f8474 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2524,7 +2524,7 @@ local check_regular do local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") - local bold = match_synonyms (P"bold" + P"demi" + P"heavy" + P"black" + P"ultra") + local bold = match_synonyms (P"bold") determine_italic = function (fontstyle_name, italicangle, -- cgit v1.2.3 From aa118e1f44ac27ed1d9046c45d8baa855fa2c107 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 02:04:47 +0100 Subject: [db] tweak font family matching Addresses https://github.com/lualatex/luaotfload/issues/118 but without crude hacks ;-) Now the test file processes perfectly: https://bitbucket.org/phg/lua-la-tex-tests/src/tip/ltx-fontspec-optical-size-3.tex --- luaotfload-database.lua | 208 +++++++++++++++++++++--------------------------- 1 file changed, 91 insertions(+), 117 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 84f8474..e439e2a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -530,11 +530,17 @@ load_lookups = function ( ) end local regular_synonym = { - book = true, - normal = true, - plain = true, - regular = true, - roman = true, + book = "r", + normal = "r", + plain = "r", + regular = "r", + roman = "r", +} + +local italic_synonym = { + oblique = true, + slanted = true, + italic = true, } local type1_formats = { "tfm", "ofm", } @@ -1294,7 +1300,7 @@ local organize_namedata = function (metadata, metadata = { fullname = metadata.fullname, fontname = metadata.fontname, - metafamily = metadata.familyname, + familyname = metadata.familyname, }, info = { @@ -2512,56 +2518,16 @@ local generate_filedata = function (mappings) return files end -local match_synonyms = function (pattern) - local nopattern = 1 - pattern - return nopattern^0 * pattern * Cc (true) + Cc (false) -end - local determine_italic local determine_bold local pick_style local check_regular do - local italic = match_synonyms (P"oblique" + P"slanted" + P"italic") - local bold = match_synonyms (P"bold") - - determine_italic = function (fontstyle_name, - italicangle, - prefmodifiers, - subfamily) - if italicangle ~= 0 then - return true - elseif fontstyle_name ~= nil and lpegmatch (italic, fontstyle_name) then - return true - elseif prefmodifiers ~= nil and lpegmatch (italic, prefmodifiers) then - return lpegmatch (italic, prefmodifiers) - elseif subfamily ~= nil and lpegmatch (italic, subfamily) then - return true - end - return false - end - - determine_bold = function (fontstyle_name, - weight, - prefmodifiers, - subfamily) - if weight [2] == "bold" then - return true - elseif fontstyle_name ~= nil and lpegmatch (bold, fontstyle_name) then - return true - elseif prefmodifiers ~= nil and lpegmatch (bold, prefmodifiers) then - return true - elseif subfamily ~= nil and lpegmatch (bold, subfamily) then - return true - end - return false - end - local splitfontname = lpeg.splitat "-" local choose_exact = function (field) - if field == "italic" or field == "oblique" then + if italic_synonym [field] then return "i" elseif field == "bold" then return "b" @@ -2577,17 +2543,18 @@ do subfamily, splitstyle) local style - if fontstyle_name ~= nil then - style = choose_exact (prefmodifiers) - end - if not style and prefmodifiers ~= nil then - style = choose_exact (prefmodifiers) + if fontstyle_name then + style = choose_exact (fontstyle_name) end if not style then - style = choose_exact (subfamily) + if prefmodifiers then + style = choose_exact (prefmodifiers) + elseif subfamily then + --style = choose_exact (subfamily) + end end - if not style and splitstyle ~= nil then - choose_exact (splitstyle) + if not style and splitstyle then + style = choose_exact (splitstyle) end return style end @@ -2600,12 +2567,14 @@ do subfamily, splitstyle) - if regular_synonym [fontstyle_name] - or regular_synonym [prefmodifiers] - or regular_synonym [subfamily] - or regular_synonym [splitstyle] - then - return "r" + if fontstyle_name then + return regular_synonym [fontstyle_name] + elseif prefmodifiers then + return regular_synonym [prefmodifiers] + elseif subfamily then + return regular_synonym [subfamily] + elseif splitstyle then + return regular_synonym [splitstyle] end return nil @@ -2619,6 +2588,7 @@ local pull_values = function (entry) local sanitized = names.sanitized local english = sanitized.english local info = sanitized.info + local metadata = sanitized.metadata --- pull file info ... entry.basename = file.base @@ -2630,12 +2600,15 @@ local pull_values = function (entry) entry.psname = english.psname entry.fontname = info.fontname entry.fullname = english.fullname or info.fullname - entry.familyname = english.preffamily - or english.family - or info.familyname + entry.prefmodifiers = english.prefmodifiers + local metafamily = metadata.familyname + local familyname = english.preffamily or english.family + entry.familyname = familyname + if familyname ~= metafamily then + entry.metafamily = metadata.familyname + end entry.fontstyle_name = sanitized.fontstyle_name entry.plainname = names.fullname - entry.prefmodifiers = english.prefmodifiers entry.subfamily = english.subfamily --- pull style info ... @@ -2645,13 +2618,43 @@ local pull_values = function (entry) entry.weight = style.weight if luaotfloadconfig.strip == true then - --if false then entry.file = nil entry.names = nil entry.style = nil end end +local add_family = function (name, subtable, modifier, entry) + local familytable = subtable [name] + if not familytable then + familytable = { } + subtable [name] = familytable + end + + --- the style table is treated as an unordered list + local styletable = familytable [modifier] + if not styletable then + styletable = { } + familytable [modifier] = styletable + end + + if not entry.prefmodifiers then --- default size for this style/family combo + styletable.default = entry.index + end + + local size = entry.size --- dsnsize * high * low + if size then + styletable [#styletable + 1] = { + size [1], + size [2], + size [3], + entry.index, + } + else + styletable.default = entry.index + end +end + local collect_families = function (mappings) report ("info", 2, "db", "Analyzing families, sizes, and styles.") @@ -2679,10 +2682,8 @@ local collect_families = function (mappings) families [location] [format] = subtable end - --local fullname = english.fullname or info.fullname - --local fontname = info.fontname - local familyname = entry.familyname + local metafamily = entry.metafamily local fontstyle_name = entry.fontstyle_name local prefmodifiers = entry.prefmodifiers local subfamily = entry.subfamily @@ -2696,24 +2697,24 @@ local collect_families = function (mappings) subfamily, splitstyle) - if not modifier then --- guess - local italic = determine_italic (fontstyle_name, - italicangle, - prefmodifiers, - subfamily) - local bold = determine_bold (fontstyle_name, - weight, - prefmodifiers, - subfamily) - if bold and italic then - modifier = "bi" - elseif bold then - modifier = "b" - elseif italic then - modifier = "i" - end - end - +-- if not modifier then --- guess +-- local italic = determine_italic (fontstyle_name, +-- italicangle, +-- prefmodifiers, +-- subfamily) +-- local bold = determine_bold (fontstyle_name, +-- weight, +-- prefmodifiers, +-- subfamily) +-- if bold and italic then +-- modifier = "bi" +-- elseif bold then +-- modifier = "b" +-- elseif italic then +-- modifier = "i" +-- end +-- end +-- if not modifier then --- regular, exact only modifier = check_regular (fontstyle_name, subfamily, @@ -2721,36 +2722,9 @@ local collect_families = function (mappings) end if modifier then - --- stub; here we will continue building a list of optical sizes - --- no size -> hash “normal” - --- other sizes -> indexed tuples { dsnsize, high, low, idx } - local familytable = subtable [familyname] - if not familytable then - familytable = { } - subtable [familyname] = familytable - end - - --- the style table is treated as an unordered list - local styletable = familytable [modifier] - if not styletable then - styletable = { } - familytable [modifier] = styletable - end - - if not prefmodifiers then --- default size for this style/family combo - styletable.default = entry.index - end - - local size = entry.size --- dsnsize * high * low - if size then - styletable [#styletable + 1] = { - size [1], - size [2], - size [3], - entry.index, - } - else - styletable.default = entry.index + add_family (familyname, subtable, modifier, entry) + if metafamily ~= nil then + add_family (metafamily, subtable, modifier, entry) end end end -- cgit v1.2.3 From 5c7df96b71cc281898067971ff96eb939f8d9d45 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 15:31:11 +0100 Subject: [db] adapt cached resolver --- luaotfload-database.lua | 119 +++++++++++++++++++++++++++--------------------- 1 file changed, 68 insertions(+), 51 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e439e2a..f103caa 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,8 +31,6 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table -local gzipload = gzip.load -local gzipsave = gzip.save local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -60,6 +58,7 @@ local tablesort = table.sort local texiowrite_nl = texio.write_nl local utf8gsub = unicode.utf8.gsub local utf8lower = unicode.utf8.lower +local zlibcompress = zlib.compress --- these come from Lualibs/Context local filebasename = file.basename @@ -94,9 +93,6 @@ fonts = fonts or { } fonts.names = fonts.names or { } fonts.definers = fonts.definers or { } -local names = fonts.names -local name_index = nil -- upvalue for names.data - local luaotfloadconfig = config.luaotfload --- always present luaotfloadconfig.resolver = luaotfloadconfig.resolver or "normal" luaotfloadconfig.formats = luaotfloadconfig.formats or "otf,ttf,ttc,dfont" @@ -108,6 +104,9 @@ if luaotfloadconfig.update_live ~= false then luaotfloadconfig.update_live = true end +local names = fonts.names +local name_index = nil --> upvalue for names.data +local lookup_cache = nil --> for names.lookups names.version = 2.4 names.data = nil --- contains the loaded database names.lookups = nil --- contains the lookup cache @@ -186,11 +185,11 @@ if not runasscript then end globals.prefix = prefix - local lookups = names.path.lookups + local lookup_path = names.path.lookups local index = names.path.index local lookups_file = filejoin (prefix, globals.lookups_file) local index_file = filejoin (prefix, globals.index_file) - lookups.lua, lookups.luc = make_luanames (lookups_file) + lookup_path.lua, lookup_path.luc = make_luanames (lookups_file) index.lua, index.luc = make_luanames (index_file) else --- running as script, inject some dummies caches = { } @@ -410,6 +409,33 @@ local initialize_namedata = function (formats) --- returns dbobj } end +--[[doc-- + + Since Luaotfload does not depend on the lualibs anymore we + have to put our own small wrappers for the gzip library in + place. + +--doc]]-- + +local load_gzipped = function (filename) + local gh = gzip.open (filename,"rb") + if gh then + local data = gh:read "*all" + gh:close () + return data + end +end + +local save_gzipped = function (filename, data) + local gh = ioopen (filename, "wb") + if gh then + local bin = zlibcompress (data, 9, nil, 15 + 16) + gh:write (bin) + gh:close () + return #bin + end +end + --- When loading a lua file we try its binary complement first, which --- is assumed to be located at an identical path, carrying the suffix --- .luc. @@ -417,15 +443,14 @@ end --- string -> (string * table) local load_lua_file = function (path) local foundname = filereplacesuffix (path, "luc") + local code = nil - if false then local fh = ioopen (foundname, "rb") -- try bin first if fh then local chunk = fh:read"*all" fh:close() code = load (chunk, "b") end -end if not code then --- fall back to text file foundname = filereplacesuffix (path, "lua") @@ -439,7 +464,7 @@ end if not code then --- probe gzipped file foundname = filereplacesuffix (path, "lua.gz") - local chunk = gzipload (foundname) + local chunk = load_gzipped (foundname) if chunk then code = load (chunk, "t") end @@ -515,7 +540,7 @@ load_names = function (dry_run) return data end ---- unit -> dbobj +--- unit -> unit load_lookups = function ( ) local foundname, data = load_lua_file(names.path.lookups.lua) if data then @@ -526,7 +551,7 @@ load_lookups = function ( ) "No lookup cache, creating empty.") data = { } end - return data + lookup_cache = data end local regular_synonym = { @@ -677,28 +702,18 @@ end --[[doc-- We need to verify if the result of a cached lookup actually exists in -the texmf or filesystem. +the texmf or filesystem. Again, due to the schizoprenic nature of the +font managment we have to check both the system path and the texmf. --doc]]-- local verify_font_file = function (basename) - if not name_index then name_index = load_names() end - local files = name_index.files - local idx = files.base[basename] - if not idx then - return false - end - - --- firstly, check filesystem - local fullname = files.full[idx] - if fullname and lfsisfile(fullname) then + local path = resolve_fullpath (basename) + if path and lfsisfile(path) then return true end - - --- secondly, locate via kpathsea if kpsefind_file(basename) then return true end - return false end @@ -733,8 +748,8 @@ Idk what the “spec” resolver is for. optsize, size spec: name, sub resolved, sub, name, forced -* name: contains both the name resolver from luatex-fonts and resolve() - below +* name: contains both the name resolver from luatex-fonts and + resolve_name() below From my reading of font-def.lua, what a resolver does is basically rewrite the “name” field of the specification record @@ -763,20 +778,20 @@ local hash_request = function (specification) end --- 'a -> 'a -> table -> (string * int|boolean * boolean) -resolve_cached = function (_, _, specification) - if not names.lookups then names.lookups = load_lookups() end +resolve_cached = function (specification) + if not lookup_cache then load_lookups () end local request = hash_request(specification) report("both", 4, "cache", "Looking for %q in cache ...", request) - local found = names.lookups[request] + local found = lookup_cache [request] --- case 1) cache positive ---------------------------------------- if found then --- replay fields from cache hit report("info", 4, "cache", "Found!") local basename = found[1] --- check the presence of the file in case it’s been removed - local success = verify_font_file(basename) + local success = verify_font_file (basename) if success == true then return basename, found[2], true end @@ -787,22 +802,24 @@ resolve_cached = function (_, _, specification) --- case 2) cache negative ---------------------------------------- --- first we resolve normally ... - local filename, subfont, success = resolve(nil, nil, specification) - if not success then return filename, subfont, false end + local filename, subfont = resolve_name (specification) + if not filename then + return nil, nil + end --- ... then we add the fields to the cache ... ... local entry = { filename, subfont } report("both", 4, "cache", "New entry: %s", request) - names.lookups[request] = entry + lookup_cache [request] = entry --- obviously, the updated cache needs to be stored. --- TODO this should trigger a save only once the --- document is compiled (finish_pdffile callback?) report("both", 5, "cache", "Saving updated cache") - local success = save_lookups() + local success = save_lookups () if not success then --- sad, but not critical - report("both", 0, "cache", "Could not write to cache") + report("both", 0, "cache", "Error writing cache.") end - return filename, subfont, true + return filename, subfont end --- this used to be inlined; with the lookup cache we don’t @@ -1061,13 +1078,15 @@ resolve_fullpath = function (fontname, ext) --- getfilename() for i = 1, #location_precedence do local location = location_precedence [i] local basenames = basedata [location] - local barenames = baredata [location] [ext] local idx if basenames ~= nil then idx = basenames [fontname] end - if not idx and barenames ~= nil then - idx = barenames [fontname] + if ext then + local barenames = baredata [location] [ext] + if not idx and barenames ~= nil then + idx = barenames [fontname] + end end if idx then return files.full [idx] @@ -2383,10 +2402,9 @@ end --- unit -> (bool, lookup_cache) flush_lookup_cache = function () - if not names.lookups then names.lookups = load_lookups() end - names.lookups = { } + lookup_cache = { } collectgarbage "collect" - return true, names.lookups + return true, lookup_cache end @@ -3038,7 +3056,7 @@ update_names = function (currentnames, force, dry_run) local success, _lookups = flush_lookup_cache () if success then - local success = names.save_lookups () + local success = save_lookups () if success then logs.names_report ("info", 2, "cache", "Lookup cache emptied") @@ -3051,13 +3069,12 @@ end --- unit -> bool save_lookups = function ( ) - local lookups = names.lookups local path = names.path.lookups local luaname, lucname = path.lua, path.luc if fileiswritable (luaname) and fileiswritable (lucname) then - tabletofile (luaname, lookups, true) + tabletofile (luaname, lookup_cache, true) osremove (lucname) - caches.compile (lookups, luaname, lucname) + caches.compile (lookup_cache, luaname, lucname) --- double check ... if lfsisfile (luaname) and lfsisfile (lucname) then report ("both", 3, "cache", "Lookup cache saved") @@ -3089,7 +3106,7 @@ save_names = function (currentnames) local gzname = luaname .. ".gz" if luaotfloadconfig.compress then local serialized = tableserialize (currentnames, true) - gzipsave (gzname, serialized) + save_gzipped (gzname, serialized) caches.compile (currentnames, "", lucname) else tabletofile (luaname, currentnames, true) @@ -3301,8 +3318,8 @@ names.show_cache = show_cache --- replace the resolver from luatex-fonts if luaotfloadconfig.resolver == "cached" then report("both", 2, "cache", "caching of name: lookups active") - names.resolve = resolve_cached - names.resolvespec = resolve_cached + names.resolvespec = resolve_cached + names.resolve_name = resolve_cached else names.resolvespec = resolve_name names.resolve_name = resolve_name -- cgit v1.2.3 From 4467a6f5150c49cd0992a624c294018cf02be07c Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 17:22:54 +0100 Subject: [db] remove dependency on lualibs for compression --- luaotfload-database.lua | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index f103caa..567149a 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -31,6 +31,7 @@ local fontloaderinfo = fontloader.info local fontloaderclose = fontloader.close local fontloaderopen = fontloader.open local fontloaderto_table = fontloader.to_table +local gzipopen = gzip.open local iolines = io.lines local ioopen = io.open local kpseexpand_path = kpse.expand_path @@ -415,10 +416,13 @@ end have to put our own small wrappers for the gzip library in place. + load_gzipped -- Read and decompress and entire gzipped file. + Returns the uncompressed content as a string. + --doc]]-- local load_gzipped = function (filename) - local gh = gzip.open (filename,"rb") + local gh = gzipopen (filename,"rb") if gh then local data = gh:read "*all" gh:close () @@ -426,13 +430,21 @@ local load_gzipped = function (filename) end end +--[[doc-- + + save_gzipped -- Compress and write a string to file. The return + value is the number of bytes written. Zlib parameters are: best + compression and default strategy. + +--doc]]-- + local save_gzipped = function (filename, data) - local gh = ioopen (filename, "wb") + local gh = gzipopen (filename, "wb9") if gh then - local bin = zlibcompress (data, 9, nil, 15 + 16) - gh:write (bin) + gh:write (data) + local bytes = gh:seek () gh:close () - return #bin + return bytes end end -- cgit v1.2.3 From 480327fcaf3d5da8f2766e7cf9fa6e2fbb59ceb5 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 17:52:17 +0100 Subject: [db] fix test for regular style --- luaotfload-database.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index 567149a..9101f94 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2747,6 +2747,7 @@ local collect_families = function (mappings) -- if not modifier then --- regular, exact only modifier = check_regular (fontstyle_name, + prefmodifiers, subfamily, splitstyle) end -- cgit v1.2.3 From 049edfd2afc26472a0f41df1c02a73cc1fd6e812 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 18:03:38 +0100 Subject: [db] remove cruft --- luaotfload-database.lua | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index e439e2a..0987508 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2518,8 +2518,6 @@ local generate_filedata = function (mappings) return files end -local determine_italic -local determine_bold local pick_style local check_regular @@ -2697,24 +2695,6 @@ local collect_families = function (mappings) subfamily, splitstyle) --- if not modifier then --- guess --- local italic = determine_italic (fontstyle_name, --- italicangle, --- prefmodifiers, --- subfamily) --- local bold = determine_bold (fontstyle_name, --- weight, --- prefmodifiers, --- subfamily) --- if bold and italic then --- modifier = "bi" --- elseif bold then --- modifier = "b" --- elseif italic then --- modifier = "i" --- end --- end --- if not modifier then --- regular, exact only modifier = check_regular (fontstyle_name, subfamily, -- cgit v1.2.3 From 19d57e135e09d9311d0e17c6f7c375a161b8c37f Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Wed, 6 Nov 2013 18:31:07 +0100 Subject: [db] do not add metafamilies to the family table --- luaotfload-database.lua | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/luaotfload-database.lua b/luaotfload-database.lua index c5b156c..d8d4a6f 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -2734,9 +2734,29 @@ local collect_families = function (mappings) if modifier then add_family (familyname, subtable, modifier, entry) - if metafamily ~= nil then - add_family (metafamily, subtable, modifier, entry) - end + --- registering the metafamilies is unreliable within the + --- same table as identifiers might interfere with an + --- unmarked style that lacks a metafamily, e.g. + --- + --- iwona condensed regular -> + --- family: iwonacond + --- metafamily: iwona + --- iwona regular -> + --- family: iwona + --- metafamily: ø + --- + --- Both would be registered as under the same family, + --- i.e. “iwona”, and depending on the loading order + --- the query “name:iwona” can resolve to the condensed + --- version instead of the actual unmarked one. The only + --- way around this would be to introduce a separate + --- table for metafamilies and do fallback queries on it. + --- At the moment this is not pressing enough to justify + --- further increasing the index size, maybe if need + --- arises from the user side. +-- if metafamily and metafamily ~= familyname then +-- add_family (metafamily, subtable, modifier, entry) +-- end end end -- cgit v1.2.3