Merge pull request #110 from phi-gamma/master

v2.3a

1 files changed, 510 insertions, 38 deletions

diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua
index 4d25d53..4afd9d1 100755
--- a/luaotfload-tool.lua
+++ b/luaotfload-tool.lua
@@ -4,7 +4,7 @@
 --  DESCRIPTION:  database functionality
 -- REQUIREMENTS:  luaotfload 2.2
 --       AUTHOR:  Khaled Hosny, Élie Roux, Philipp Gesang
---      VERSION:  2.3
+--      VERSION:  2.3a
 --      LICENSE:  GPL v2
 --     MODIFIED:  2013-06-02 19:23:54+0200
 -----------------------------------------------------------------------
@@ -26,6 +26,8 @@ see the luaotfload documentation for more info. Report bugs to
 
 --doc]]--
 
+kpse.set_program_name "luatex"
+
 --[[doc--
 
     We test for Lua 5.1 by means of capability detection to see if
@@ -36,7 +38,28 @@ see the luaotfload documentation for more info. Report bugs to
 
 --doc]]--
 
-kpse.set_program_name "luatex"
+
+local ioopen          = io.open
+local iowrite         = io.write
+local kpsefind_file   = kpse.find_file
+local lfsattributes   = lfs.attributes
+local lfsisfile       = lfs.isfile
+local lfsreadlink     = lfs.readlink
+local md5sumhexa      = md5.sumhexa
+local next            = next
+local osdate          = os.date
+local osremove        = os.remove
+local ostype          = os.type
+local stringexplode   = string.explode
+local stringformat    = string.format
+local stringlower     = string.lower
+local stringrep       = string.rep
+local stringsub       = string.sub
+local tableconcat     = table.concat
+local texiowrite_nl   = texio.write_nl
+local texiowrite      = texio.write
+local tonumber        = tonumber
+local type            = type
 
 local runtime
 if _G.getfenv ~= nil then -- 5.1 or LJ
@@ -44,26 +67,19 @@ if _G.getfenv ~= nil then -- 5.1 or LJ
         runtime = { "jit", jit.version }
     else
         runtime = { "stock", _VERSION }
-        local oldscript = kpse.find_file "luaotfload-legacy-tool.lua"
+        local oldscript = kpsefind_file "luaotfload-legacy-tool.lua"
         return require (oldscript)
     end
 else -- 5.2
     runtime = { "stock", _VERSION }
 end
 
-local stringexplode   = string.explode
-local stringformat    = string.format
-local stringlower     = string.lower
-local stringrep       = string.rep
-local tableconcat     = table.concat
-local texiowrite_nl   = texio.write_nl
-local texiowrite      = texio.write
 
-local C, Ct, P, S  = lpeg.C, lpeg.Ct, lpeg.P, lpeg.S
+local C, Cg, Ct, P, S  = lpeg.C, lpeg.Cg, lpeg.Ct, lpeg.P, lpeg.S
 local lpegmatch    = lpeg.match
 
 local loader_file = "luatexbase.loader.lua"
-local loader_path = assert(kpse.find_file(loader_file, "lua"),
+local loader_path = assert(kpsefind_file(loader_file, "lua"),
                            "File '"..loader_file.."' not found")
 
 
@@ -93,7 +109,7 @@ local config                  = config
 config.luaotfload             = config.luaotfload or { }
 config.luaotfload.names_dir   = config.luaotfload.names_dir or "names"
 config.luaotfload.cache_dir   = config.luaotfload.cache_dir or "fonts"
-config.luaotfload.names_file  = config.luaotfload.names_file
+config.luaotfload.index_file  = config.luaotfload.index_file
                              or "luaotfload-names.lua"
 
 do -- we don’t have file.basename and the likes yet, so inline parser ftw
@@ -117,9 +133,16 @@ end
 config.lualibs                  = config.lualibs or { }
 config.lualibs.verbose          = false
 config.lualibs.prefer_merged    = true
-config.lualibs.load_extended    = false
+config.lualibs.load_extended    = true
 
 require "lualibs"
+--- dofile "util-jsn.lua" --- awaiting fix
+
+local lua_of_json               = utilities.json.tolua
+local ioloaddata                = io.loaddata
+local tabletohash               = table.tohash
+local fileiswritable            = file.iswritable
+local fileisreadable            = file.isreadable
 
 --[[doc--
 \fileent{luatex-basics-gen.lua} calls functions from the
@@ -143,17 +166,15 @@ local names    = fonts.names
 
 local sanitize_string = names.sanitize_string
 
---local db_src_out = names.path.dir.."/"..names.path.basename
-local names_plain = file.join (
-     caches.getwritablepath (config.luaotfload.names_dir),
-     config.luaotfload.names_file)
-local names_bin   = file.replacesuffix (names_plain, "luc")
+local pathdata      = names.path
+local names_plain   = pathdata.index.lua
+local names_bin     = pathdata.index.luc
 
 local help_messages = {
     ["luaotfload-tool"] = [[
 
-Usage: %s [OPTION]...
-    
+Usage: %s [OPTIONS...]
+
 Operations on the LuaTeX font database.
 
 This tool is part of the luaotfload package. Valid options are:
@@ -169,6 +190,8 @@ This tool is part of the luaotfload package. Valid options are:
 
   -V --version                 print version and exit
   -h --help                    print this message
+  --diagnose=CHECK             run a self test procedure; one of “files”,
+                               “permissions”, or “repository”
 
   --alias=<name>               force behavior of “luaotfload-tool” or legacy
                                “mkluatexfontdb”
@@ -214,7 +237,7 @@ The font cache will be written to
     mkluatexfontdb = [[
 
 Usage: %s [OPTION]...
-    
+
 Rebuild or update the LuaTeX font database.
 
 Valid options:
@@ -233,17 +256,26 @@ The font database will be saved to
    %s
 
 ]],
+    short = [[
+Usage: luaotfload-tool [--help] [--version] [--verbose=<lvl>]
+                       [--update] [--force] [--prefer-texmf]
+                       [--find=<font name>] [--fuzzy] [--info] [--inspect]
+                       [--list=<criterion>] [--fields=<field list>]
+                       [--cache=<directive>] [--flush-lookups]
+                       [--show-blacklist] [--diagnose=<procedure>]
+
+Enter 'luaotfload-tool --help' for a larger list of options.
+]]
 }
 
-local help_msg = function ( )
-    local template = help_messages[config.luaotfload.self]
-                  or help_messages["luaotfload-tool"]
-    texiowrite_nl(stringformat(template,
-                               config.luaotfload.self,
-                               names_plain,
-                               names_bin,
-                               caches.getwritablepath (
-                                 config.luaotfload.cache_dir)))
+local help_msg = function (version)
+    local template = help_messages[version]
+    iowrite(stringformat(template,
+                         config.luaotfload.self,
+                         names_plain,
+                         names_bin,
+                         caches.getwritablepath (
+                         config.luaotfload.cache_dir)))
 end
 
 local version_msg = function ( )
@@ -467,7 +499,7 @@ local display_general = function (fullinfo)
                 val = #fullinfo[key]
             end
         elseif mode == "d" then
-            val = os.date("%F %T", fullinfo[key])
+            val = osdate("%F %T", fullinfo[key])
         end
         if not val then
             val = "<none>"
@@ -648,10 +680,11 @@ set.
 --]]--
 
 local action_sequence = {
-    "loglevel", "help",     "version", "blacklist", "cache",
-    "flush",    "generate", "list",    "query",
+    "loglevel",  "help",  "version", "diagnose",
+    "blacklist", "cache", "flush",   "generate",
+    "list",      "query",
 }
-local action_pending  = table.tohash(action_sequence, false)
+local action_pending  = tabletohash(action_sequence, false)
 
 action_pending.loglevel = true  --- always set the loglevel
 action_pending.generate = false --- this is the default action
@@ -662,7 +695,7 @@ actions.loglevel = function (job)
     logs.set_loglevel(job.log_level)
     logs.names_report("info", 3, "util",
                       "Setting log level", "%d", job.log_level)
-    logs.names_report("log", 0, "util", "Lua=%s", _VERSION)
+    logs.names_report("log", 2, "util", "Lua=%s", _VERSION)
     return true, true
 end
 
@@ -672,7 +705,7 @@ actions.version = function (job)
 end
 
 actions.help = function (job)
-    help_msg()
+    help_msg (job.help_version or "luaotfload-tool")
     return true, false
 end
 
@@ -915,6 +948,437 @@ actions.list = function (job)
     return true, true
 end
 
+do
+    local out = function (...)
+        logs.names_report (false, 0, "diagnose", ...)
+    end
+
+    local verify_files = function (errcnt, info)
+        out "================ verify files ================="
+        local hashes = info.hashes
+        local notes  = info.notes
+        if not hashes or #hashes == 0 then
+            out ("FAILED: cannot read checksums from %s.", status_file)
+            return 1/0
+        elseif not notes then
+            out ("FAILED: cannot read commit metadata from %s.",
+                 status_file)
+            return 1/0
+        end
+
+        out ("Luaotfload revision %s.", notes.revision)
+        out ("Committed by %s.",        notes.committer)
+        out ("Timestamp %s.",           notes.timestamp)
+
+        local nhashes = #hashes
+        out ("Testing %d files for integrity.", nhashes)
+        for i = 1, nhashes do
+            local fname, canonicalsum = unpack (hashes[i])
+            local location = kpsefind_file (fname)
+                          or kpsefind_file (fname, "texmfscripts")
+            if not location then
+                errcnt = errcnt + 1
+                out ("FAILED: file %s missing.", fname)
+            else
+                out ("File: %s.", location)
+                local raw = ioloaddata (location)
+                if not raw then
+                    errcnt = errcnt + 1
+                    out ("FAILED: file %d not readable.", fname)
+                else
+                    local sum = md5sumhexa (raw)
+                    if sum ~= canonicalsum then
+                        errcnt = errcnt + 1
+                        out ("FAILED: checksum mismatch for file %s.",
+                             fname)
+                        out ("Expected %s.", canonicalsum)
+                        out ("Got      %s.", sum)
+                    else
+                        out ("Ok, %s passed.", fname)
+                    end
+                end
+            end
+        end
+        return errcnt
+    end
+
+    local get_tentative_attributes = function (file)
+        if not lfsisfile (file) then
+            local chan = ioopen (file, "w")
+            if chan then
+                chan:close ()
+                local attributes = lfsattributes (file)
+                os.remove (file)
+                return attributes
+            end
+        end
+    end
+
+    local p_permissions = Ct(Cg(Ct(C(1) * C(1) * C(1)), "u")
+                           * Cg(Ct(C(1) * C(1) * C(1)), "g")
+                           * Cg(Ct(C(1) * C(1) * C(1)), "o"))
+
+    local analyze_permissions = function (raw)
+        return lpegmatch (p_permissions, raw)
+    end
+
+    local get_permissions = function (t, location)
+        local attributes = lfsattributes (location)
+        if not attributes and t == "f" then
+            attributes = get_tentative_attributes (location)
+            if not attributes then
+                return false
+            end
+        end
+
+        local permissions
+
+        if fileisreadable (location) then
+            --- link handling appears to be unnecessary because
+            --- lfs.attributes() will return the information on
+            --- the link target.
+            if mode == "link" then --follow and repeat
+                location = lfsreadlink (location)
+                attributes = lfsattributes (location)
+            end
+        end
+
+        permissions = analyze_permissions (attributes.permissions)
+
+        return {
+            location    = location,
+            mode        = attributes.mode,
+            owner       = attributes.uid, --- useless on windows
+            permissions = permissions,
+            attributes  = attributes,
+        }
+    end
+
+    local check_conformance = function (spec, permissions, errcnt)
+        local uid = permissions.attributes.uid
+        local gid = permissions.attributes.gid
+        local raw = permissions.attributes.permissions
+
+        out ("Owner: %d, group %d, permissions %s.", uid, gid, raw)
+        if ostype == "unix" then
+            if uid == 0 or gid == 0 then
+                out "Owned by the superuser, permission conflict likely."
+                errcnt = errcnt + 1
+            end
+        end
+
+        local user = permissions.permissions.u
+        if spec.r == true then
+            if user[1] == "r" then
+                out "Readable: ok."
+            else
+                out "Not readable: permissions need fixing."
+                errcnt = errcnt + 1
+            end
+        end
+
+        if spec.w == true then
+            if user[2] == "w"
+            or  fileiswritable (permissions.location) then
+                out "Writable: ok."
+            else
+                out "Not writable: permissions need fixing."
+                errcnt = errcnt + 1
+            end
+        end
+
+        return errcnt
+    end
+
+    local path = names.path
+
+    local desired_permissions = {
+        { "d", {"r","w"}, function () return caches.getwritablepath () end },
+        { "d", {"r","w"}, path.globals.prefix },
+        { "f", {"r","w"}, path.index.lua },
+        { "f", {"r","w"}, path.index.luc },
+        { "f", {"r","w"}, path.lookups.lua },
+        { "f", {"r","w"}, path.lookups.luc },
+    }
+
+    local check_permissions = function (errcnt)
+        out [[=============== file permissions ==============]]
+        for i = 1, #desired_permissions do
+            local t, spec, path = unpack (desired_permissions[i])
+            if type (path) == "function" then
+                path = path ()
+            end
+
+            spec = tabletohash (spec)
+
+            out ("Checking permissions of %s.", path)
+
+            local permissions = get_permissions (t, path)
+            if permissions then
+                --inspect (permissions)
+                errcnt = check_conformance (spec, permissions, errcnt)
+            else
+                errcnt = errcnt + 1
+            end
+        end
+        return errcnt
+    end
+
+    local check_upstream
+
+    if kpsefind_file ("https.lua", "lua") == nil then
+        check_upstream = function (errcnt)
+            out       [[============= upstream repository =============
+                        Github API access requires the luasec library.
+                        WARNING: Cannot retrieve repository data.
+                        Grab it from <https://github.com/brunoos/luasec>
+                        and retry.]]
+            return errcnt
+        end
+    else
+    --- github api stuff begin
+        local https = require "ssl.https"
+
+        local gh_api_root     = [[https://api.github.com]]
+        local release_url     = [[https://github.com/lualatex/luaotfload/releases]]
+        local luaotfload_repo = [[lualatex/luaotfload]]
+        local user_agent      = [[lualatex/luaotfload integrity check]]
+        local shortbytes = 8
+
+        local gh_shortrevision = function (rev)
+            return stringsub (rev, 1, shortbytes)
+        end
+
+        local gh_encode_parameters = function (parameters)
+            local acc = {}
+            for field, value in next, parameters do
+                --- unsafe, non-urlencoded coz it’s all ascii chars
+                acc[#acc+1] = field .. "=" .. value
+            end
+            return "?" .. tableconcat (acc, "&")
+        end
+
+        local gh_make_url = function (components, parameters)
+            local url = tableconcat ({ gh_api_root,
+                                       unpack (components) },
+                                     "/")
+            if parameters then
+                url = url .. gh_encode_parameters (parameters)
+            end
+            return url
+        end
+
+        local alright = [[HTTP/1.1 200 OK]]
+
+        local gh_api_request = function (...)
+            local args    = {...}
+            local nargs   = #args
+            local final   = args[nargs]
+            local request = {
+                url     = "",
+                headers = { ["user-agent"] = user_agent },
+            }
+            if type (final) == "table" then
+                args[nargs] = nil
+                request = gh_make_url (args, final)
+            else
+                request = gh_make_url (args)
+            end
+
+            out ("Requesting <%s>.", request)
+            local response, code, headers, status
+                = https.request (request)
+            if status ~= alright then
+                out "Request failed!"
+                return false
+            end
+            return response
+        end
+
+        local gh_api_checklimit = function (headers)
+            local rawlimit  = gh_api_request "rate_limit"
+            local limitdata = lua_of_json (rawlimit)
+            if not limitdata and limitdata.rate then
+                out "Cannot parse API rate limit."
+                return false
+            end
+            limitdata = limitdata.rate
+
+            local limit = tonumber (limitdata.limit)
+            local left  = tonumber (limitdata.remaining)
+            local reset = tonumber (limitdata.reset)
+
+            out ("%d of %d Github API requests left.", left, limit)
+            if left == 0 then
+                out ("Cannot make any more API requests.")
+                out ("Try again later at %s.", osdate ("%F %T", reset))
+            end
+            return true
+        end
+
+        local gh_tags = function ()
+            out "Fetching tags from repository, please stand by."
+            local rawtags = gh_api_request ("repos",
+                                            luaotfload_repo,
+                                            "tags")
+            local taglist = lua_of_json (rawtags)
+            if not taglist or #taglist == 0 then
+                out "Cannot parse response."
+                return false
+            end
+
+            local ntags = #taglist
+            out ("Repository contains %d tags.", ntags)
+            local _idx, latest = next (taglist)
+            out ("The most recent release is %s (revision %s).",
+                 latest.name,
+            gh_shortrevision (latest.commit.sha))
+            return latest
+        end
+
+        local gh_compare = function (head, base)
+            if base == nil then
+                base = "HEAD"
+            end
+            out ("Fetching comparison between %s and %s, \z
+                  please stand by.",
+                 gh_shortrevision (head),
+                 gh_shortrevision (base))
+            local comparison = base .. "..." .. head
+            local rawstatus = gh_api_request ("repos",
+                                              luaotfload_repo,
+                                              "compare",
+                                              comparison)
+            local status = lua_of_json (rawstatus)
+            if not status then
+                out "Cannot parse response for status request."
+                return false
+            end
+            return status
+        end
+
+        local gh_news = function (since)
+            local compared  = gh_compare (since)
+            if not compared then
+                return false
+            end
+            local behind_by = compared.behind_by
+            local ahead_by  = compared.ahead_by
+            local status    = compared.status
+            out ("Comparison state: %s.", status)
+            if behind_by > 0 then
+                out ("Your Luaotfload is %d \z
+                      revisions behind upstream.",
+                     behind_by)
+                return behind_by
+            elseif status == "ahead" then
+                out "Since you are obviously from the future \z
+                     I assume you already know the repository state."
+            else
+                out "Everything up to date. \z
+                     Luaotfload is in sync with upstream."
+            end
+            return false
+        end
+
+        local gh_catchup = function (current, latest)
+            local compared = gh_compare (latest, current)
+            local ahead_by = tonumber (compared.ahead_by)
+            if ahead_by > 0 then
+                local permalink_url = compared.permalink_url
+                out ("Your Luaotfload is %d revisions \z
+                      behind the most recent release.",
+                     ahead_by)
+                out ("To view the commit log, visit <%s>.",
+                     permalink_url)
+                out ("You can grab an up to date tarball at <%s>.",
+                     release_url)
+                return true
+            else
+                out "There weren’t any new releases in the meantime."
+                out "Luaotfload is up to date."
+            end
+            return false
+        end
+
+        check_upstream = function (current)
+            out "============= upstream repository ============="
+            local _succ  = gh_api_checklimit ()
+            local behind = gh_news (current)
+            if behind then
+                local latest  = gh_tags ()
+                local _behind = gh_catchup (current,
+                                            latest.commit.sha,
+                                            latest.name)
+            end
+        end
+
+        --- trivium: diff since the first revision as pushed by Élie
+        --- in 2009
+        --- local firstrevision = "c3ccb3ee07e0a67171c24960966ae974e0dd8e98"
+        --- check_upstream (firstrevision)
+    end
+    --- github api stuff end
+
+    local anamneses   = { "files", "repository", "permissions" }
+    local status_file = "luaotfload-status"
+
+    actions.diagnose = function (job)
+        local errcnt = 0
+        local asked  = job.asked_diagnostics
+        if asked == "all" or asked == "thorough" then
+            asked = tabletohash (anamneses, true)
+        else
+            asked = lpegmatch(split_comma, asked)
+            asked = tabletohash (asked, true)
+        end
+
+        out "Loading file hashes."
+        local info = require (status_file)
+
+        if asked.files == true then
+            errcnt = verify_files (errcnt, info)
+        end
+        if asked.permissions == true then
+            errcnt = check_permissions (errcnt)
+        end
+        if asked.repository == true then
+            --errcnt = check_upstream (info.notes.revision)
+            check_upstream (info.notes.revision)
+        end
+
+
+        if errcnt == 0 then --> success
+            out ("Everything appears to be in order, \z
+                  you may sleep well.")
+            return true, false
+        end
+        out (         [[===============================================
+                                            WARNING
+                        ===============================================
+
+                        The diagnostic detected %d errors.
+
+                        This version of luaotfload may have been
+                        tampered with. Modified versions of the
+                        luaotfload source are unsupported. Read the log
+                        carefully and get a clean version from CTAN or
+                        github:
+
+                            × http://ctan.org/tex-archive/macros/luatex/generic/luaotfload
+                            × https://github.com/lualatex/luaotfload/releases
+
+                        If you are uncertain as to how to proceed, then
+                        ask on the lualatex mailing list:
+
+                            http://www.tug.org/mailman/listinfo/lualatex-dev
+
+                        ===============================================
+]],          errcnt)
+        return true, false
+    end
+end
+
 --- stuff to be carried out prior to exit
 
 local finalizers = { }
@@ -954,6 +1418,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
     local long_options = {
         alias              = 1,
         cache              = 1,
+        diagnose           = 1,
         ["dry-run"]        = "D",
         ["flush-lookups"]  = "l",
         fields             = 1,
@@ -1049,13 +1514,20 @@ local process_cmdline = function ( ) -- unit -> jobspec
             config.luaotfload.prioritize = "texmf"
         elseif v == "b" then
             action_pending["blacklist"] = true
+        elseif v == "diagnose" then
+            action_pending["diagnose"] = true
+            result.asked_diagnostics = optarg[n]
         end
     end
 
     if config.luaotfload.self == "mkluatexfontdb" then
+        result.help_version = "mkluatexfontdb"
         action_pending["generate"] = true
-        result.log_level = math.max(2, result.log_level)
+        result.log_level = math.max(1, result.log_level)
         logs.set_logout"stdout"
+    elseif nopts == 0 then
+        action_pending["help"] = true
+        result.help_version = "short"
     end
     return result
 end