Merge pull request #208 from phi-gamma/texlive2014

add bisect mode

4 files changed, 410 insertions, 30 deletions

diff --git a/src/luaotfload-database.lua b/src/luaotfload-database.lua
index 17cb4fd..936e380 100644
--- a/src/luaotfload-database.lua
+++ b/src/luaotfload-database.lua
@@ -2335,7 +2335,6 @@ local retrieve_namedata = function (files, currentnames, targetnames, dry_run)
 
     local nfiles    = #files
     local nnew      = 0
-    local max_fonts = luaotfloadconfig.max_fonts or 2^51
 
     report ("info", 1, "db", "Scanning %d collected font files ...", nfiles)
 
@@ -2344,7 +2343,7 @@ local retrieve_namedata = function (files, currentnames, targetnames, dry_run)
                        , system    = { 0, 0 }
                        }
     report_status_start (2, 4)
-    for i = 1, (nfiles < max_fonts) and nfiles or max_fonts do
+    for i = 1, nfiles do
         local fullname, location   = unpack (files[i])
         local count                = bylocation[location]
         count[1]                   = count[1] + 1
@@ -2920,18 +2919,90 @@ order_design_sizes = function (families)
     return families
 end
 
---- dbobj -> dbobj -> int -> int -> string * bool list
+--[[doc--
+
+    collect_font_filenames -- Scan the three search path categories for
+    font files. This constitutes the first pass of the update mode.
+
+--doc]]--
+
+--- unit -> string * bool list
 local collect_font_filenames = function ()
-    ---
+
+    report ("info", 4, "db", "Scanning the filesystem for font files.")
+
     local filenames = { }
+    local bisect    = luaotfloadconfig.bisect
+    local max_fonts = luaotfloadconfig.max_fonts or 2^51 --- XXX revisit for lua 5.3 wrt integers
+
     tableappend (filenames, collect_font_filenames_texmf  ())
     tableappend (filenames, collect_font_filenames_system ())
     if luaotfloadconfig.scan_local  == true then
         tableappend (filenames, collect_font_filenames_local  ())
     end
+    --- Now drop everything above max_fonts.
+    if max_fonts < #filenames then
+        filenames = { unpack (filenames, 1, max_fonts) }
+    end
+    --- And choose the requested slice if in bisect mode.
+    if bisect then
+        return { unpack (filenames, bisect[1], bisect[2]) }
+    end
     return filenames
 end
 
+--[[doc--
+
+    nth_font_file -- Return the filename of the nth font.
+
+--doc]]--
+
+--- int -> string
+local nth_font_filename = function (n)
+    report ("info", 4, "db", "Picking font file no. %d.", n)
+    if not p_blacklist then
+        read_blacklist ()
+    end
+    local filenames = collect_font_filenames ()
+    return filenames[n] and filenames[n][1] or "<error>"
+end
+
+--[[doc--
+
+    font_slice -- Return the fonts in the range from lo to hi.
+
+--doc]]--
+
+local font_slice = function (lo, hi)
+    report ("info", 4, "db", "Retrieving font files nos. %d--%d.", lo, hi)
+    if not p_blacklist then
+        read_blacklist ()
+    end
+    local filenames = collect_font_filenames ()
+    local result    = { }
+    for i = lo, hi do
+        result[#result + 1] = filenames[i][1]
+    end
+    return result
+end
+
+--[[doc
+
+    count_font_files -- Return the number of files found by
+    collect_font_filenames. This function is exported primarily
+    for use with luaotfload-tool.lua in bisect mode.
+
+--doc]]--
+
+--- unit -> int
+local count_font_files = function ()
+    report ("info", 4, "db", "Counting font files.")
+    if not p_blacklist then
+        read_blacklist ()
+    end
+    return #collect_font_filenames ()
+end
+
 --- dbobj -> stats
 
 local collect_statistics = function (mappings)
@@ -3459,6 +3530,9 @@ names.read_blacklist              = read_blacklist
 names.sanitize_fontname           = sanitize_fontname
 names.getfilename                 = resolve_fullpath
 names.set_location_precedence     = set_location_precedence
+names.count_font_files            = count_font_files
+names.nth_font_filename           = nth_font_filename
+names.font_slice                  = font_slice
 
 --- font cache
 names.purge_cache    = purge_cache
diff --git a/src/luaotfload-fontloader.lua b/src/luaotfload-fontloader.lua
index 196fc61..655aedf 100644
--- a/src/luaotfload-fontloader.lua
+++ b/src/luaotfload-fontloader.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 03/16/14 11:40:51
+-- merge date  : 03/22/14 15:31:38
 
 do -- begin closure to overcome local limits and interference
 
@@ -2574,9 +2574,11 @@ end
 if not number then number={} end 
 local stripper=patterns.stripzeros
 local function points(n)
+  n=tonumber(n)
   return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536))
 end
 local function basepoints(n)
+  n=tonumber(n)
   return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536))
 end
 number.points=points
@@ -6569,7 +6571,7 @@ local report_otf=logs.reporter("fonts","otf loading")
 local fonts=fonts
 local otf=fonts.handlers.otf
 otf.glists={ "gsub","gpos" }
-otf.version=2.754 
+otf.version=2.755 
 otf.cache=containers.define("fonts","otf",otf.version,true)
 local fontdata=fonts.hashes.identifiers
 local chardata=characters and characters.data 
diff --git a/src/luaotfload-main.lua b/src/luaotfload-main.lua
index 3c4c770..fe4e792 100644
--- a/src/luaotfload-main.lua
+++ b/src/luaotfload-main.lua
@@ -49,20 +49,22 @@ luaotfload.log                    = luaotfload.log or { }
 
 config                            = config or { }
 config.luaotfload                 = config.luaotfload or { }
-------.luaotfload.resolver        = config.luaotfload.resolver         or "normal"
-config.luaotfload.resolver        = config.luaotfload.resolver         or "cached"
-config.luaotfload.definer         = config.luaotfload.definer          or "patch"
-config.luaotfload.loglevel        = config.luaotfload.loglevel         or 2
-config.luaotfload.color_callback  = config.luaotfload.color_callback   or "pre_linebreak_filter"
-config.luaotfload.prioritize      = config.luaotfload.prioritize       or "sys"
-config.luaotfload.names_dir       = config.luaotfload.names_dir        or "names"
-config.luaotfload.cache_dir       = config.luaotfload.cache_dir        or "fonts"
-config.luaotfload.index_file      = config.luaotfload.index_file       or "luaotfload-names.lua"
-config.luaotfload.formats         = config.luaotfload.formats          or "otf,ttf,ttc,dfont"
-config.luaotfload.scan_local      = config.luaotfload.scan_local == true
-
-if config.luaotfload.strip == nil then
-    config.luaotfload.strip = true
+local luaotfloadconfig            = config.luaotfload
+----------------.resolver        = luaotfloadconfig.resolver         or "normal"
+luaotfloadconfig.resolver        = luaotfloadconfig.resolver         or "cached"
+luaotfloadconfig.definer         = luaotfloadconfig.definer          or "patch"
+luaotfloadconfig.bisect          = false --- useless when running TeX
+luaotfloadconfig.loglevel        = luaotfloadconfig.loglevel         or 2
+luaotfloadconfig.color_callback  = luaotfloadconfig.color_callback   or "pre_linebreak_filter"
+luaotfloadconfig.prioritize      = luaotfloadconfig.prioritize       or "sys"
+luaotfloadconfig.names_dir       = luaotfloadconfig.names_dir        or "names"
+luaotfloadconfig.cache_dir       = luaotfloadconfig.cache_dir        or "fonts"
+luaotfloadconfig.index_file      = luaotfloadconfig.index_file       or "luaotfload-names.lua"
+luaotfloadconfig.formats         = luaotfloadconfig.formats          or "otf,ttf,ttc,dfont"
+luaotfloadconfig.scan_local      = luaotfloadconfig.scan_local == true
+
+if luaotfloadconfig.strip == nil then
+    luaotfloadconfig.strip = true
 end
 
 luaotfload.module = {
@@ -151,7 +153,7 @@ loadmodule "log.lua"        --- messages; used to be part of -override
 local log             = luaotfload.log
 local report          = log.report
 
-log.set_loglevel(config.luaotfload.loglevel)
+log.set_loglevel(luaotfloadconfig.loglevel)
 
 --[[doc--
 
@@ -689,7 +691,7 @@ reset_callback "define_font"
 
 --doc]]--
 
-local font_definer = config.luaotfload.definer
+local font_definer = luaotfloadconfig.definer
 
 if font_definer == "generic"  then
   add_to_callback("define_font",
diff --git a/src/luaotfload-tool.lua b/src/luaotfload-tool.lua
index 9e75944..47e7ccc 100755
--- a/src/luaotfload-tool.lua
+++ b/src/luaotfload-tool.lua
@@ -48,6 +48,7 @@ kpse.set_program_name "luatex"
 local ioopen          = io.open
 local iowrite         = io.write
 local kpsefind_file   = kpse.find_file
+local mathfloor       = math.floor
 local next            = next
 local osdate          = os.date
 local ostype          = os.type
@@ -95,6 +96,7 @@ config                        = config or { }
 local config                  = config
 local luaotfloadconfig        = config.luaotfload or { }
 config.luaotfload             = luaotfloadconfig
+luaotfloadconfig.bisect       = false
 luaotfloadconfig.version      = luaotfloadconfig.version   or version
 luaotfloadconfig.names_dir    = luaotfloadconfig.names_dir or "names"
 luaotfloadconfig.cache_dir    = luaotfloadconfig.cache_dir or "fonts"
@@ -113,8 +115,12 @@ config.lualibs.prefer_merged    = true
 config.lualibs.load_extended    = true
 
 require "lualibs"
-local tabletohash               = table.tohash
+local iosavedata                = io.savedata
+local lfsisdir                  = lfs.isdir
+local lfsisfile                 = lfs.isfile
 local stringsplit               = string.split
+local tableserialize            = table.serialize
+local tabletohash               = table.tohash
 
 --[[doc--
 \fileent{luatex-basics-gen.lua} calls functions from the
@@ -256,7 +262,7 @@ local help_msg = function (version)
                          names_gzip,
                          names_bin,
                          caches.getwritablepath (
-                         luaotfloadconfig.cache_dir)))
+                            luaotfloadconfig.cache_dir)))
 end
 
 local about = [[
@@ -288,7 +294,7 @@ local head_adornchars = {
 }
 
 local textwidth         = 80
-local wd_leftcolumn     = math.floor(textwidth * .25)
+local wd_leftcolumn     = mathfloor(textwidth * .25)
 local key_fmt           = stringformat([[%%%ds]], wd_leftcolumn)
 local val_fmt           = [[%s]]
 local fieldseparator    = ":"
@@ -725,8 +731,8 @@ set.
 
 local action_sequence = {
     "loglevel",   "help",  "version", "diagnose",
-    "blacklist",  "cache", "flush",   "generate",
-    "list",       "query",
+    "blacklist",  "cache", "flush",   "bisect",
+    "generate",    "list", "query",
 }
 
 local action_pending  = tabletohash(action_sequence, false)
@@ -771,6 +777,297 @@ actions.generate = function (job)
     return false, false
 end
 
+-------------------------------------------------------------------------------
+--- bisect mode
+-------------------------------------------------------------------------------
+
+local bisect_status_path = caches.getwritablepath "bisect"
+local bisect_status_file = bisect_status_path .."/" .. "luaotfload-bisect-status.lua"
+local bisect_status_fmt  = [[
+--[==[-------------------------------------------------------------------------
+    This file is generated by Luaotfload. It can be safely deleted.
+    Creation date: %s.
+-------------------------------------------------------------------------]==]--
+
+%s
+
+--- vim:ft=lua:ts=8:et:sw=2
+]]
+
+--[[doc--
+
+    write_bisect_status -- Write the history of the current bisection to disk.
+
+--doc]]--
+
+--- state list -> bool
+local write_bisect_status = function (data)
+    local payload = tableserialize (data, true)
+    local status  = stringformat (bisect_status_fmt,
+                                  osdate ("%Y-%m-d %H:%M:%S", os.time ()),
+                                  payload)
+    if status and iosavedata (bisect_status_file, status) then
+        report ("info", 4, "bisect",
+                "Bisection state written to %s.", bisect_status_file)
+        return true
+    end
+    report ("info", 0, "bisect",
+            "Failed to write bisection state to %s.", bisect_status_file)
+    return false
+end
+
+--[[doc--
+
+    read_bisect_status -- Read the bisect log from disk.
+
+--doc]]--
+
+--- unit -> state list
+local read_bisect_status = function ()
+    report ("info", 4, "bisect", "Testing for status file: %q.", bisect_status_file)
+    if not lfsisfile (bisect_status_file) then
+        report ("info", 2, "bisect", "No such file: %q.", bisect_status_file)
+        report ("info", 0, "bisect", "Not in bisect mode.")
+        return false
+    end
+    report ("info", 4, "bisect", "Reading status file: %q.", bisect_status_file)
+    local success, status = pcall (dofile, bisect_status_file)
+    if not success then
+        report ("info", 0, "bisect", "Could not read status file.")
+        return false
+    end
+    return status
+end
+
+--[[doc--
+
+    bisect_start -- Begin a bisect session. Determines the number of
+    fonts and sets the initial high, low, and pivot values.
+
+--doc]]--
+
+local bisect_start = function ()
+    if lfsisfile (bisect_status_file) then
+        report ("info", 0, "bisect",
+                "Bisect session in progress.",
+                bisect_status_file)
+        report ("info", 0, "bisect",
+                "Use --bisect=stop to erase it before starting over.")
+        return false, false
+    end
+    report ("info", 2, "bisect",
+            "Starting bisection of font database %q.", bisect_status_file)
+    local n     = names.count_font_files ()
+    local pivot = mathfloor (n / 2)
+    local data  = { { 1, n, pivot } }
+    report ("info", 0, "bisect", "Initializing pivot to %d.", pivot)
+    if write_bisect_status (data) then
+        return true, false
+    end
+    return false, false
+end
+
+--[[doc--
+
+    bisect_stop -- Terminate bisection session by removing all state info.
+
+--doc]]--
+
+local bisect_stop = function ()
+    report ("info", 3, "bisect", "Erasing bisection state at %s.", bisect_status_file)
+    if lfsisfile (bisect_status_file) then
+        local success, msg = os.remove (bisect_status_file)
+        if not success then
+            report ("info", 2, "bisect",
+                    "Failed to erase file %s (%s).",
+                     bisect_status_file, msg)
+        end
+    end
+    if lfsisdir (bisect_status_path) then
+        local success, msg = os.remove (bisect_status_path)
+        if not success then
+            report ("info", 2, "bisect",
+                    "Failed to erase directory %s (%s).",
+                     bisect_status_path, msg)
+        end
+    end
+    if lfsisfile (bisect_status_file) then
+        return false, false
+    end
+    return true, false
+end
+
+--[[doc--
+
+    bisect_terminate -- Wrap up a bisect session by printing the
+    offending font and removing the state file.
+
+--doc]]--
+
+local bisect_terminate = function (nsteps, culprit)
+    report ("info", 1, "bisect",
+            "Bisection completed after %d steps.", nsteps)
+    report ("info", 0, "bisect",
+            "Bad file: %s.", names.nth_font_filename (culprit))
+    report ("info", 0, "bisect",
+            "Run with --bisect=stop to finish bisection.")
+    return true, false
+end
+
+--[[doc--
+
+    list_remainder -- Show remaining fonts in bisect slice.
+
+--doc]]--
+
+local list_remainder = function (lo, hi)
+    local fonts = names.font_slice (lo, hi)
+    report ("info", 0, "bisect", "%d fonts left.", hi - lo + 1)
+    for i = 1, #fonts do
+        report ("info", 1, "bisect", "   · %2d: %s", lo, fonts[i])
+        lo = lo + 1
+    end
+end
+
+--[[doc--
+
+    bisect_set -- Prepare the next bisection step by setting high, low,
+    and pivot to new values.
+
+    The “run” directive always picks the segment below the pivot so we
+    can rely on the “outcome parameter” to be referring to that.
+
+--doc]]--
+
+local bisect_set = function (outcome)
+    local status = read_bisect_status ()
+    if not status then
+        return false, false
+    end
+
+    local nsteps        = #status
+    local previous      = status[nsteps]
+    if previous == true then
+        --- Bisection already completed; we exit early through
+        --- bisect_terminate() to avoid further writes to the
+        --- state files that mess up step counting.
+        nsteps = nsteps - 1
+        return bisect_terminate (nsteps, status[nsteps][1])
+    end
+
+    local lo, hi, pivot = unpack (previous)
+
+    report ("info", 3, "bisect", "Previous step %d: lo=%d, hi=%d, pivot=%d.",
+            nsteps, lo, hi, pivot)
+
+    if outcome == "bad" then
+        hi = pivot
+        if lo >= hi then --- complete
+            status[nsteps + 1] = { lo, lo, lo }
+            status[nsteps + 1] = true
+            write_bisect_status (status)
+            return bisect_terminate (nsteps, lo)
+        end
+        pivot = mathfloor ((lo + hi) / 2)
+        report ("info", 0, "bisect",
+                "Continuing with the lower segment: lo=%d, hi=%d, pivot=%d.",
+                lo, hi, pivot)
+    elseif outcome == "good" then
+        lo = pivot + 1
+        if lo >= hi then --- complete
+            status[nsteps + 1] = { lo, lo, lo }
+            write_bisect_status (status)
+            status[nsteps + 1] = true
+            return bisect_terminate (nsteps, lo)
+        end
+        pivot = mathfloor ((lo + hi) / 2)
+        report ("info", 0, "bisect",
+                "Continuing with the upper segment: lo=%d, hi=%d, pivot=%d.",
+                lo, hi, pivot)
+    else -- can’t happen
+        report ("info", 0, "bisect", "What the hell?", lo, hi, pivot)
+        return false, false
+    end
+
+    status[nsteps + 1] = { lo, hi, pivot }
+    write_bisect_status (status)
+    if hi - lo <= 10 then
+        list_remainder (lo, hi)
+    end
+    return true, false
+end
+
+--[[doc--
+
+    bisect_status -- Output information about the current bisect session.
+
+--doc]]--
+
+local bisect_status = function ()
+    local status = read_bisect_status ()
+    if not status then
+        return false, false
+    end
+    local nsteps = #status
+    if nsteps > 1 then
+        for i = nsteps - 1, 1, -1 do
+            local step = status[i]
+            report ("info", 2, "bisect", "Step %d: lo=%d, hi=%d, pivot=%d.",
+                    i, unpack (step))
+        end
+    end
+    local current = status[nsteps]
+    report ("info", 0, "bisect", "Step %d: lo=%d, hi=%d, pivot=%d.",
+            nsteps, unpack (current))
+    return true, false
+end
+
+--[[doc--
+
+    bisect_run -- Run Luaotfload utilizing the current bisection state.
+    This should be combined with the --update mode, possibly with the
+    --force option.
+
+    Luaotfload always tests the segment below the pivot first.
+
+--doc]]--
+
+local bisect_run = function ()
+    local status = read_bisect_status ()
+    if not status then
+        return false, false
+    end
+    local nsteps        = #status
+    local currentstep   = nsteps + 1
+    local current       = status[nsteps]
+    local lo, hi, pivot = unpack (current)
+    report ("info", 3, "bisect", "Previous step %d: lo=%d, hi=%d, pivot=%d.",
+            nsteps, lo, hi, pivot)
+    report ("info", 1, "bisect", "Step %d: Testing fonts from %d to %d.",
+            currentstep, lo, pivot)
+    luaotfloadconfig.bisect = { lo, pivot }
+    return true, true
+end
+
+local bisect_modes = {
+    start   = bisect_start,
+    good    = function () return bisect_set "good" end,
+    bad     = function () return bisect_set "bad"  end,
+    stop    = bisect_stop,
+    status  = bisect_status,
+    run     = bisect_run,
+}
+
+actions.bisect = function (job)
+    local mode   = job.bisect
+    local runner = bisect_modes[mode]
+    if not runner then
+        report ("info", 0, "bisect", "Unknown directive %q.", mode)
+        return false, false
+    end
+    return runner (job)
+end
+
 actions.flush = function (job)
     local success = names.flush_lookup_cache()
     if success then
@@ -1076,6 +1373,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
         criterion    = "",
         query        = "",
         log_level    = 0, --- 2 is approx. the old behavior
+        bisect       = nil,
     }
 
     local long_options = {
@@ -1097,6 +1395,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
         ["local"]          = "L",
         log                = 1,
         ["max-fonts"]      = 1,
+        ["bisect"]         = 1,
         ["no-reload"]      = "n",
         ["no-strip"]       = 0,
         ["skip-read"]      = "R",
@@ -1211,6 +1510,9 @@ local process_cmdline = function ( ) -- unit -> jobspec
                     luaotfloadconfig.max_fonts = n
                 end
             end
+        elseif v == "bisect" then
+            result.bisect          = optarg[n]
+            action_pending.bisect = true
         end
     end
 
@@ -1239,16 +1541,16 @@ local main = function ( ) -- unit -> int
 
             if not success then
                 report (false, 0, "util",
-                        "Could not finish task", "%s", actionname)
+                        "Failed to execute task.", "%s", actionname)
                 retval = -1
                 exit   = true
             elseif not continue then
                 report (false, 3, "util",
-                        "Task completed, exiting", "%s", actionname)
+                        "Task completed, exiting.", "%s", actionname)
                 exit = true
             else
                 report (false, 3, "util",
-                        "Task completed successfully", "%s", actionname)
+                        "Task completed successfully.", "%s", actionname)
             end
         end
         if exit then break end