Merge pull request #115 from phi-gamma/master

update docs; excise diagnostics from the runner script

7 files changed, 834 insertions, 632 deletions

diff --git a/filegraph.dot b/filegraph.dot
index c8a07d3..e412279 100644
--- a/filegraph.dot
+++ b/filegraph.dot
@@ -51,18 +51,55 @@ strict digraph luaotfload_files { //looks weird with circo ...
     merged_luatex_fonts -> font_age [label="luatex-fonts-enc.lua",
                                      ltail=cluster_merged]
 
+    fontdbutil -> fontdbutil_diagnostics [label="--diagnose"]
+
+    fontdbutil -> status [label="version information"]
+
+    fontdbutil_diagnostics -> status [constraint=no, label="hash files"]
+
+    merged_luatex_fonts -> characters [label="luaotfload-auxiliary.lua",
+                                       ltail=cluster_merged]
+
     luaotfload_libs -> font_names [label="luaotfload-database.lua"]
+    luaotfload_libs -> typo_krn   [label="luaotfload-extralibs.lua"]
+
+    mkstatus    -> status     [label="generates from distribution files",
+                               style=dashed]
 
     mkglyphlist -> font_age     [label="generates from glyphlist.txt",
                                  style=dashed]
 
-    subgraph { rank = same; mkglyphlist; fontdbutil; luaotfload }
+    mkcharacters -> characters     [label="generates from Context’s char-def.lua",
+                                    style=dashed]
+
+    subgraph { rank = same;
+               mkcharacters;
+               mkglyphlist;
+               mkstatus;
+               fontdbutil;
+               luaotfload }
 
 /* ····································································
  * main files
  * ································································· */
 
-    fontdbutil        [label  = "luaotfload-util\nmkluatexfontdb.lua",
+    fontdbutil        [label  = "luaotfload-tool.lua\nmkluatexfontdb.lua",
+                       shape  = rect,
+                       width  = "3.2cm",
+                       height = "1.2cm",
+                       color  = "#01012222",
+                       style  = "filled,rounded",
+                       penwidth=2]
+
+    fontdbutil_diagnostics [label  = "luaotfload-diagnostics.lua",
+                            shape  = rect,
+                            width  = "3.2cm",
+                            height = "1.2cm",
+                            color  = "#01012222",
+                            style  = "filled,rounded",
+                            penwidth=2]
+
+    mkstatus          [label  = "mkstatus",
                        shape  = rect,
                        width  = "3.2cm",
                        height = "1.2cm",
@@ -78,6 +115,14 @@ strict digraph luaotfload_files { //looks weird with circo ...
                        style  = "filled,rounded",
                        penwidth=2]
 
+    mkcharacters      [label  = "mkcharacters",
+                       shape  = rect,
+                       width  = "3.2cm",
+                       height = "1.2cm",
+                       color  = "#01012222",
+                       style  = "filled,rounded",
+                       penwidth=2]
+
     luaotfload        [label  = "luaotfload.lua",
                        shape  = rect,
                        width  = "3.2cm",
@@ -102,11 +147,27 @@ strict digraph luaotfload_files { //looks weird with circo ...
                        style  = "filled,rounded",
                        penwidth=2]
 
+    typo_krn [label  = "luaotfload-typo-krn.lua",
+              shape  = rect,
+              width  = "3.2cm",
+              height = "1.2cm",
+              color  = "#01012222",
+              style  = "filled,rounded",
+              penwidth=2]
+
 /* ····································································
  * luaotfload files
  * ································································· */
 
 
+    characters [style      = "filled,dashed",
+                shape      = rect,
+                width      = "3.2cm",
+                fillcolor  = "#01012222",
+                color      = grey40,
+                style      = "filled,dotted,rounded",
+                label      = "luaotfload-characters.lua"]
+
     font_age [style      = "filled,dashed",
               shape      = rect,
               width      = "3.2cm",
@@ -123,6 +184,14 @@ strict digraph luaotfload_files { //looks weird with circo ...
                 style      = "filled,dotted,rounded",
                 label      = "luaotfload-names.lua\nluaotfload-names.luc"]
 
+    status [style      = "filled,dashed",
+            shape      = rect,
+            width      = "3.2cm",
+            fillcolor  = "#01012222",
+            color      = grey40,
+            style      = "filled,dotted,rounded",
+            label      = "luaotfload-status.lua"]
+
     otfl_blacklist_cnf [style      = "filled,dashed",
                         shape      = rect,
                         width      = "3.2cm",
@@ -139,9 +208,10 @@ strict digraph luaotfload_files { //looks weird with circo ...
         label      = <
             <table cellborder="0" bgcolor="#FFFFFFAA">
                 <th> <td colspan="2"> <font point-size="12" face="Iwona Italic">Luaotfload Libraries</font> </td> </th>
-                <tr> <td>luaotfload-auxiliary.lua</td> <td>luaotfload-features.lua</td> </tr>
-                <tr> <td>luaotfload-override.lua</td>  <td>luaotfload-loaders.lua</td>  </tr>
-                <tr> <td>luaotfload-database.lua</td>  <td>luaotfload-color.lua</td>    </tr>
+                <tr> <td>luaotfload-auxiliary.lua</td>  <td>luaotfload-features.lua</td>    </tr>
+                <tr> <td>luaotfload-override.lua</td>   <td>luaotfload-loaders.lua</td>     </tr>
+                <tr> <td>luaotfload-database.lua</td>   <td>luaotfload-color.lua</td>       </tr>
+                <tr> <td>luaotfload-extralibs.lua</td>  <td>luaotfload-letterspace.lua</td> </tr>
             </table>
         >,
     ]
diff --git a/luaotfload-database.lua b/luaotfload-database.lua
index a49e575..a0a532e 100644
--- a/luaotfload-database.lua
+++ b/luaotfload-database.lua
@@ -117,9 +117,17 @@ end
 
 local report = logs.names_report
 
+names.patterns          = { }
+local patterns          = names.patterns
+
 local trailingslashes   = P"/"^1 * P(-1)
 local stripslashes      = C((1 - trailingslashes)^0)
-names.patterns          = { stripslashes = stripslashes }
+patterns.stripslashes   = stripslashes
+
+local comma             = P","
+local noncomma          = 1-comma
+local splitcomma        = Ct((C(noncomma^1) + comma)^1)
+patterns.splitcomma     = splitcomma
 
 --[[doc--
     We use the functions in the cache.* namespace that come with the
diff --git a/luaotfload-diagnostics.lua b/luaotfload-diagnostics.lua
new file mode 100644
index 0000000..db807de
--- /dev/null
+++ b/luaotfload-diagnostics.lua
@@ -0,0 +1,622 @@
+#!/usr/bin/env texlua
+-----------------------------------------------------------------------
+--         FILE:  luaotfload-diagnostics.lua
+--  DESCRIPTION:  functionality accessible by the --diagnose option
+-- REQUIREMENTS:  luaotfload-tool.lua
+--       AUTHOR:  Philipp Gesang (Phg), <phg42.2a@gmail.com>
+--      VERSION:  1.0
+--      CREATED:  2013-07-28 10:01:18+0200
+-----------------------------------------------------------------------
+--
+local names                    = fonts.names
+local status                   = config.luaotfload.status
+
+local kpse                     = require "kpse"
+local kpseexpand_path          = kpse.expand_path
+local kpseexpand_var           = kpse.expand_var
+local kpsefind_file            = kpse.find_file
+
+local lfs                      = require "lfs"
+local lfsattributes            = lfs.attributes
+local lfsisfile                = lfs.isfile
+local lfsreadlink              = lfs.readlink
+
+local md5                      = require "md5"
+local md5sumhexa               = md5.sumhexa
+
+local osgetenv                 = os.getenv
+local osname                   = os.name
+local osremove                 = os.remove
+local ostype                   = os.type
+local stringformat             = string.format
+local stringsub                = string.sub
+
+local fileisreadable           = file.isreadable
+local fileiswritable           = file.iswritable
+local filesplitpath            = file.splitpath
+local ioloaddata               = io.loaddata
+local lua_of_json              = utilities.json.tolua
+local tableconcat              = table.concat
+local tabletohash              = table.tohash
+
+local lpeg                     = require "lpeg"
+local C, Cg, Ct                = lpeg.C, lpeg.Cg, lpeg.Ct
+local lpegmatch                = lpeg.match
+
+local out = function (...)
+    logs.names_report (false, 0, "diagnose", ...)
+end
+
+local verify_files = function (errcnt, status)
+    out "================ verify files ================="
+    local hashes = status.hashes
+    local notes  = status.notes
+    if not hashes or #hashes == 0 then
+        out ("FAILED: cannot read checksums from %s.", status_file)
+        return 1/0
+    elseif not notes then
+        out ("FAILED: cannot read commit metadata from %s.",
+             status_file)
+        return 1/0
+    end
+
+    out ("Luaotfload revision %s.", notes.revision)
+    out ("Committed by %s.",        notes.committer)
+    out ("Timestamp %s.",           notes.timestamp)
+
+    local nhashes = #hashes
+    out ("Testing %d files for integrity.", nhashes)
+    for i = 1, nhashes do
+        local fname, canonicalsum = unpack (hashes[i])
+        local location = kpsefind_file (fname)
+                      or kpsefind_file (fname, "texmfscripts")
+        if not location then
+            errcnt = errcnt + 1
+            out ("FAILED: file %s missing.", fname)
+        else
+            out ("File: %s.", location)
+            local raw = ioloaddata (location)
+            if not raw then
+                errcnt = errcnt + 1
+                out ("FAILED: file %d not readable.", fname)
+            else
+                local sum = md5sumhexa (raw)
+                if sum ~= canonicalsum then
+                    errcnt = errcnt + 1
+                    out ("FAILED: checksum mismatch for file %s.",
+                         fname)
+                    out ("Expected %s.", canonicalsum)
+                    out ("Got      %s.", sum)
+                else
+                    out ("Ok, %s passed.", fname)
+                end
+            end
+        end
+    end
+    return errcnt
+end
+
+local get_tentative_attributes = function (file)
+    if not lfsisfile (file) then
+        local chan = ioopen (file, "w")
+        if chan then
+            chan:close ()
+            local attributes = lfsattributes (file)
+            os.remove (file)
+            return attributes
+        end
+    end
+end
+
+local p_permissions = Ct(Cg(Ct(C(1) * C(1) * C(1)), "u")
+                       * Cg(Ct(C(1) * C(1) * C(1)), "g")
+                       * Cg(Ct(C(1) * C(1) * C(1)), "o"))
+
+local analyze_permissions = function (raw)
+    return lpegmatch (p_permissions, raw)
+end
+
+local stripslashes = names.patterns.stripslashes
+
+local get_permissions = function (t, location)
+    if stringsub (location, #location) == "/" then
+        --- strip trailing slashes (lfs idiosyncrasy on Win)
+        location = lpegmatch (stripslashes, location)
+    end
+    local attributes = lfsattributes (location)
+
+    if not attributes and t == "f" then
+        attributes = get_tentative_attributes (location)
+        if not attributes then
+            return false
+        end
+    end
+
+    local permissions
+
+    if fileisreadable (location) then
+        --- link handling appears to be unnecessary because
+        --- lfs.attributes() will return the information on
+        --- the link target.
+        if mode == "link" then --follow and repeat
+            location = lfsreadlink (location)
+            attributes = lfsattributes (location)
+        end
+    end
+
+    permissions = analyze_permissions (attributes.permissions)
+
+    return {
+        location    = location,
+        mode        = attributes.mode,
+        owner       = attributes.uid, --- useless on windows
+        permissions = permissions,
+        attributes  = attributes,
+    }
+end
+
+local check_conformance = function (spec, permissions, errcnt)
+    local uid = permissions.attributes.uid
+    local gid = permissions.attributes.gid
+    local raw = permissions.attributes.permissions
+
+    out ("Owner: %d, group %d, permissions %s.", uid, gid, raw)
+    if ostype == "unix" then
+        if uid == 0 or gid == 0 then
+            out "Owned by the superuser, permission conflict likely."
+            errcnt = errcnt + 1
+        end
+    end
+
+    local user = permissions.permissions.u
+    if spec.r == true then
+        if user[1] == "r" then
+            out "Readable: ok."
+        else
+            out "Not readable: permissions need fixing."
+            errcnt = errcnt + 1
+        end
+    end
+
+    if spec.w == true then
+        if user[2] == "w"
+        or  fileiswritable (permissions.location) then
+            out "Writable: ok."
+        else
+            out "Not writable: permissions need fixing."
+            errcnt = errcnt + 1
+        end
+    end
+
+    return errcnt
+end
+
+local path = names.path
+
+local desired_permissions = {
+    { "d", {"r","w"}, function () return caches.getwritablepath () end },
+    { "d", {"r","w"}, path.globals.prefix },
+    { "f", {"r","w"}, path.index.lua },
+    { "f", {"r","w"}, path.index.luc },
+    { "f", {"r","w"}, path.lookups.lua },
+    { "f", {"r","w"}, path.lookups.luc },
+}
+
+local check_permissions = function (errcnt)
+    out [[=============== file permissions ==============]]
+    for i = 1, #desired_permissions do
+        local t, spec, path = unpack (desired_permissions[i])
+        if type (path) == "function" then
+            path = path ()
+        end
+
+        spec = tabletohash (spec)
+
+        out ("Checking permissions of %s.", path)
+
+        local permissions = get_permissions (t, path)
+        if permissions then
+            --inspect (permissions)
+            errcnt = check_conformance (spec, permissions, errcnt)
+        else
+            errcnt = errcnt + 1
+        end
+    end
+    return errcnt
+end
+
+local check_upstream
+
+if kpsefind_file ("https.lua", "lua") == nil then
+    check_upstream = function (errcnt)
+        out       [[============= upstream repository =============
+                    WARNING: Cannot retrieve repository data.
+                    Github API access requires the luasec library.
+                    Grab it from <https://github.com/brunoos/luasec>
+                    and retry.]]
+        return errcnt
+    end
+else
+--- github api stuff begin
+    local https = require "ssl.https"
+
+    local gh_api_root     = [[https://api.github.com]]
+    local release_url     = [[https://github.com/lualatex/luaotfload/releases]]
+    local luaotfload_repo = [[lualatex/luaotfload]]
+    local user_agent      = [[lualatex/luaotfload integrity check]]
+    local shortbytes = 8
+
+    local gh_shortrevision = function (rev)
+        return stringsub (rev, 1, shortbytes)
+    end
+
+    local gh_encode_parameters = function (parameters)
+        local acc = {}
+        for field, value in next, parameters do
+            --- unsafe, non-urlencoded coz it’s all ascii chars
+            acc[#acc+1] = field .. "=" .. value
+        end
+        return "?" .. tableconcat (acc, "&")
+    end
+
+    local gh_make_url = function (components, parameters)
+        local url = tableconcat ({ gh_api_root,
+                                   unpack (components) },
+                                 "/")
+        if parameters then
+            url = url .. gh_encode_parameters (parameters)
+        end
+        return url
+    end
+
+    local alright = [[HTTP/1.1 200 OK]]
+
+    local gh_api_request = function (...)
+        local args    = {...}
+        local nargs   = #args
+        local final   = args[nargs]
+        local request = {
+            url     = "",
+            headers = { ["user-agent"] = user_agent },
+        }
+        if type (final) == "table" then
+            args[nargs] = nil
+            request = gh_make_url (args, final)
+        else
+            request = gh_make_url (args)
+        end
+
+        out ("Requesting <%s>.", request)
+        local response, code, headers, status
+            = https.request (request)
+        if status ~= alright then
+            out "Request failed!"
+            return false
+        end
+        return response
+    end
+
+    local gh_api_checklimit = function (headers)
+        local rawlimit  = gh_api_request "rate_limit"
+        local limitdata = lua_of_json (rawlimit)
+        if not limitdata and limitdata.rate then
+            out "Cannot parse API rate limit."
+            return false
+        end
+        limitdata = limitdata.rate
+
+        local limit = tonumber (limitdata.limit)
+        local left  = tonumber (limitdata.remaining)
+        local reset = tonumber (limitdata.reset)
+
+        out ("%d of %d Github API requests left.", left, limit)
+        if left == 0 then
+            out ("Cannot make any more API requests.")
+            out ("Try again later at %s.", osdate ("%F %T", reset))
+        end
+        return true
+    end
+
+    local gh_tags = function ()
+        out "Fetching tags from repository, please stand by."
+        local rawtags = gh_api_request ("repos",
+                                        luaotfload_repo,
+                                        "tags")
+        local taglist = lua_of_json (rawtags)
+        if not taglist or #taglist == 0 then
+            out "Cannot parse response."
+            return false
+        end
+
+        local ntags = #taglist
+        out ("Repository contains %d tags.", ntags)
+        local _idx, latest = next (taglist)
+        out ("The most recent release is %s (revision %s).",
+             latest.name,
+        gh_shortrevision (latest.commit.sha))
+        return latest
+    end
+
+    local gh_compare = function (head, base)
+        if base == nil then
+            base = "HEAD"
+        end
+        out ("Fetching comparison between %s and %s, \z
+              please stand by.",
+             gh_shortrevision (head),
+             gh_shortrevision (base))
+        local comparison = base .. "..." .. head
+        local rawstatus = gh_api_request ("repos",
+                                          luaotfload_repo,
+                                          "compare",
+                                          comparison)
+        local status = lua_of_json (rawstatus)
+        if not status then
+            out "Cannot parse response for status request."
+            return false
+        end
+        return status
+    end
+
+    local gh_news = function (since)
+        local compared  = gh_compare (since)
+        if not compared then
+            return false
+        end
+        local behind_by = compared.behind_by
+        local ahead_by  = compared.ahead_by
+        local status    = compared.status
+        out ("Comparison state: %s.", status)
+        if behind_by > 0 then
+            out ("Your Luaotfload is %d \z
+                  revisions behind upstream.",
+                 behind_by)
+            return behind_by
+        elseif status == "ahead" then
+            out "Since you are obviously from the future \z
+                 I assume you already know the repository state."
+        else
+            out "Everything up to date. \z
+                 Luaotfload is in sync with upstream."
+        end
+        return false
+    end
+
+    local gh_catchup = function (current, latest)
+        local compared = gh_compare (latest, current)
+        local ahead_by = tonumber (compared.ahead_by)
+        if ahead_by > 0 then
+            local permalink_url = compared.permalink_url
+            out ("Your Luaotfload is %d revisions \z
+                  behind the most recent release.",
+                 ahead_by)
+            out ("To view the commit log, visit <%s>.",
+                 permalink_url)
+            out ("You can grab an up to date tarball at <%s>.",
+                 release_url)
+            return true
+        else
+            out "There weren't any new releases in the meantime."
+            out "Luaotfload is up to date."
+        end
+        return false
+    end
+
+    check_upstream = function (current)
+        out "============= upstream repository ============="
+        local _succ  = gh_api_checklimit ()
+        local behind = gh_news (current)
+        if behind then
+            local latest  = gh_tags ()
+            local _behind = gh_catchup (current,
+                                        latest.commit.sha,
+                                        latest.name)
+        end
+    end
+
+    --- trivium: diff since the first revision as pushed by Élie
+    --- in 2009
+    --- local firstrevision = "c3ccb3ee07e0a67171c24960966ae974e0dd8e98"
+    --- check_upstream (firstrevision)
+end
+--- github api stuff end
+
+local print_envvar = function (var)
+    local val = osgetenv (var)
+    if val then
+        out ("%20s: %q", stringformat ("$%s", var), val)
+        return val
+    else
+        out ("%20s: <unset>", stringformat ("$%s", var))
+    end
+end
+
+local print_path = function (var)
+    local val = osgetenv (var)
+    if val then
+        local paths = filesplitpath (val)
+        if paths then
+            local npaths = #paths
+            if npaths == 1 then
+                out ("%20s: %q", stringformat ("$%s", var), val)
+            elseif npaths > 1 then
+                out ("%20s: <%d items>", stringformat ("$%s", var), npaths)
+                for i = 1, npaths do
+                    out ("                   +: %q", paths[i])
+                end
+            else
+                out ("%20s: <empty>")
+            end
+        end
+    else
+        out ("%20s: <unset>", stringformat ("$%s", var))
+    end
+end
+
+local print_kpsevar = function (var)
+    var = "$" .. var
+    local val = kpseexpand_var (var)
+    if val and val ~= var then
+        out ("%20s: %q", var, val)
+        return val
+    else
+        out ("%20s: <empty or unset>", var)
+    end
+end
+
+local print_kpsepath = function (var)
+    var = "$" .. var
+    local val = kpseexpand_path (var)
+    if val and val ~= "" then
+        local paths = filesplitpath (val)
+        if paths then
+            local npaths = #paths
+            if npaths == 1 then
+                out ("%20s: %q", var, paths[1])
+            elseif npaths > 1 then
+                out ("%20s: <%d items>", var, npaths)
+                for i = 1, npaths do
+                    out ("                   +: %q", paths[i])
+                end
+            else
+                out ("%20s: <empty>")
+            end
+        end
+    else
+        out ("%20s: <empty or unset>", var)
+    end
+end
+
+--- this test first if a variable is set and then expands the
+--- paths; this is necessitated by the fact that expand-path will
+--- return the empty string both if the variable is unset and if
+--- the directory does not exist
+
+local print_kpsepathvar = function (var)
+    local vvar = "$" .. var
+    local val = kpseexpand_var (vvar)
+    if val and val ~= vvar then
+        out ("%20s: %q", vvar, val)
+        print_kpsepath (var)
+    else
+        out ("%20s: <empty or unset>", var)
+    end
+end
+
+local check_environment = function (errcnt)
+    out "============ environment settings ============="
+    out ("system: %s/%s", ostype, osname)
+    if ostype == "unix" and io.popen then
+        local chan = io.popen ("uname -a", "r")
+        if chan then
+            out ("info: %s", chan:read "*all")
+            chan:close ()
+        end
+    end
+
+    out "1) *shell environment*"
+    print_envvar "SHELL"
+    print_path "PATH"
+    print_path "OSFONTDIR"
+    print_envvar "USER"
+    if ostype == "windows" then
+        print_envvar "WINDIR"
+        print_envvar "CD"
+        print_path "TEMP"
+    elseif ostype == "unix" then
+        print_envvar "HOME"
+        print_envvar "PWD"
+        print_path "TMPDIR"
+    end
+
+    out "2) *kpathsea*"
+    print_kpsepathvar "OPENTYPEFONTS"
+    print_kpsepathvar "TTFONTS"
+
+    print_kpsepathvar "TEXMFCACHE"
+    print_kpsepathvar "TEXMFVAR"
+
+    --- the expansion of these can be quite large; as they aren’t
+    --- usually essential to luaotfload, we won’t dump every single
+    --- path
+    print_kpsevar "LUAINPUTS"
+    print_kpsevar "CLUAINPUTS"
+
+    return errcnt
+end
+
+local anamneses   = {
+    "environment",
+    "files",
+    "repository",
+    "permissions"
+}
+
+local splitcomma = names.patterns.splitcomma
+
+local diagnose = function (job)
+    local errcnt = 0
+    local asked  = job.asked_diagnostics
+    if asked == "all" or asked == "thorough" then
+        asked = tabletohash (anamneses, true)
+    else
+        asked = lpegmatch (splitcomma, asked)
+        asked = tabletohash (asked, true)
+    end
+
+    if asked.environment == true then
+        errcnt = check_environment (errcnt)
+        asked.environment = nil
+    end
+
+    if asked.files == true then
+        errcnt = verify_files (errcnt, status)
+        asked.files = nil
+    end
+
+    if asked.permissions == true then
+        errcnt = check_permissions (errcnt)
+        asked.permissions = nil
+    end
+
+    if asked.repository == true then
+        check_upstream (status.notes.revision)
+        asked.repository = nil
+    end
+
+    local rest = next (asked)
+    if rest ~= nil then --> something unknown
+        out ("Unknown diagnostic %q.", rest)
+    end
+    if errcnt == 0 then --> success
+        out ("Everything appears to be in order, \z
+              you may sleep well.")
+        return true, false
+    end
+    out (         [[===============================================
+                                        WARNING
+                    ===============================================
+
+                    The diagnostic detected %d errors.
+
+                    This version of luaotfload may have been
+                    tampered with. Modified versions of the
+                    luaotfload source are unsupported. Read the log
+                    carefully and get a clean version from CTAN or
+                    github:
+
+                        × http://ctan.org/tex-archive/macros/luatex/generic/luaotfload
+                        × https://github.com/lualatex/luaotfload/releases
+
+                    If you are uncertain as to how to proceed, then
+                    ask on the lualatex mailing list:
+
+                        http://www.tug.org/mailman/listinfo/lualatex-dev
+
+                    ===============================================
+]],          errcnt)
+    return true, false
+end
+
+return diagnose
+
diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua
index cd64a75..92483d6 100755
--- a/luaotfload-tool.lua
+++ b/luaotfload-tool.lua
@@ -4,11 +4,13 @@
 --  DESCRIPTION:  database functionality
 -- REQUIREMENTS:  luaotfload 2.2
 --       AUTHOR:  Khaled Hosny, Élie Roux, Philipp Gesang
---      VERSION:  2.3b
+--      VERSION:  2.4
 --      LICENSE:  GPL v2
---     MODIFIED:  2013-06-02 19:23:54+0200
+--     MODIFIED:  2013-07-28 13:12:04+0200
 -----------------------------------------------------------------------
 
+local version = "2.4" --- <int: major>.<int: minor><alpha: fixes>
+
 --[[doc--
 
 This file was originally written (as \fileent{mkluatexfontdb.lua}) by
@@ -42,23 +44,12 @@ kpse.set_program_name "luatex"
 local ioopen          = io.open
 local iowrite         = io.write
 local kpsefind_file   = kpse.find_file
-local kpseexpand_var  = kpse.expand_var
-local kpseexpand_path = kpse.expand_path
-local lfsattributes   = lfs.attributes
-local lfsisfile       = lfs.isfile
-local lfsreadlink     = lfs.readlink
-local md5sumhexa      = md5.sumhexa
 local next            = next
 local osdate          = os.date
-local osgetenv        = os.getenv
-local osremove        = os.remove
-local osname          = os.name
-local ostype          = os.type
 local stringexplode   = string.explode
 local stringformat    = string.format
 local stringlower     = string.lower
 local stringrep       = string.rep
-local stringsub       = string.sub
 local tableconcat     = table.concat
 local texiowrite_nl   = texio.write_nl
 local texiowrite      = texio.write
@@ -79,7 +70,7 @@ else -- 5.2
 end
 
 
-local C, Cg, Ct, P, S  = lpeg.C, lpeg.Cg, lpeg.Ct, lpeg.P, lpeg.S
+local C, Ct, P, S  = lpeg.C, lpeg.Ct, lpeg.P, lpeg.S
 local lpegmatch    = lpeg.match
 
 local loader_file = "luatexbase.loader.lua"
@@ -111,6 +102,7 @@ to be the more appropriate.
 config                        = config or { }
 local config                  = config
 config.luaotfload             = config.luaotfload or { }
+config.luaotfload.version     = config.luaotfload.version   or version
 config.luaotfload.names_dir   = config.luaotfload.names_dir or "names"
 config.luaotfload.cache_dir   = config.luaotfload.cache_dir or "fonts"
 config.luaotfload.index_file  = config.luaotfload.index_file
@@ -140,26 +132,33 @@ config.lualibs.prefer_merged    = true
 config.lualibs.load_extended    = true
 
 require "lualibs"
-
-local fileisreadable            = file.isreadable
-local fileiswritable            = file.iswritable
-local filesplitpath             = file.splitpath
-local ioloaddata                = io.loaddata
-local lua_of_json               = utilities.json.tolua
 local tabletohash               = table.tohash
 
 --[[doc--
 \fileent{luatex-basics-gen.lua} calls functions from the
 \luafunction{texio.*} library; too much for our taste.
 We intercept them with dummies.
+
+Also, it sets up dummies in place of the tables created by the Context
+libraries. Since we have loaded the lualibs already this would cause
+collateral damage for some libraries whose namespace would be
+overridden. We employ our usual backup-restore strategy to work around
+this. (Postponing the loading of the lualibs code is not an option
+because the functionality is needed by basics-gen itself.)
 --doc]]--
 
 local dummy_function = function ( ) end
-local backup_write, backup_write_nl  = texio.write, texio.write_nl
+local backup = {
+    write     = texio.write,
+    write_nl  = texio.write_nl,
+    utilities = utilities,
+}
 
 texio.write, texio.write_nl          = dummy_function, dummy_function
 require"luaotfload-basics-gen.lua"
-texio.write, texio.write_nl          = backup_write, backup_write_nl
+
+texio.write, texio.write_nl          = backup.write, backup.write_nl
+utilities                            = backup.utilities
 
 require"luaotfload-override.lua"  --- this populates the logs.* namespace
 require"luaotfload-database"
@@ -167,12 +166,11 @@ require"alt_getopt"
 
 local names = fonts.names
 
-local status_file = "luaotfload-status"
-local status      = require (status_file)
-
-local version  = "2.3b"
+local status_file                    = "luaotfload-status"
+local status                         = require (status_file)
+config.luaotfload.status             = status
 
-local sanitize_string = names.sanitize_string
+local sanitize_string                = names.sanitize_string
 
 local pathdata      = names.path
 local names_plain   = pathdata.index.lua
@@ -830,10 +828,6 @@ local get_fields get_fields = function (entry, fields, acc, n)
     return acc
 end
 
-local comma       = P","
-local noncomma    = 1-comma
-local split_comma = Ct((C(noncomma^1) + comma)^1)
-
 local separator = "\t" --- could be “,” for csv
 
 local format_fields format_fields = function (fields, acc, n)
@@ -869,12 +863,14 @@ set_primary_field = function (fields, addme, acc, n)
     return acc
 end
 
+local splitcomma = names.patterns.splitcomma
+
 actions.list = function (job)
     local criterion     = job.criterion
 
     local asked_fields  = job.asked_fields
     if asked_fields then
-        asked_fields = lpegmatch(split_comma, asked_fields)
+        asked_fields = lpegmatch(splitcomma, asked_fields)
     else
         --- some defaults
         asked_fields = { "fullname", "version", }
@@ -959,579 +955,10 @@ actions.list = function (job)
     return true, true
 end
 
-do
-    local out = function (...)
-        logs.names_report (false, 0, "diagnose", ...)
-    end
-
-    local verify_files = function (errcnt, status)
-        out "================ verify files ================="
-        local hashes = status.hashes
-        local notes  = status.notes
-        if not hashes or #hashes == 0 then
-            out ("FAILED: cannot read checksums from %s.", status_file)
-            return 1/0
-        elseif not notes then
-            out ("FAILED: cannot read commit metadata from %s.",
-                 status_file)
-            return 1/0
-        end
-
-        out ("Luaotfload revision %s.", notes.revision)
-        out ("Committed by %s.",        notes.committer)
-        out ("Timestamp %s.",           notes.timestamp)
-
-        local nhashes = #hashes
-        out ("Testing %d files for integrity.", nhashes)
-        for i = 1, nhashes do
-            local fname, canonicalsum = unpack (hashes[i])
-            local location = kpsefind_file (fname)
-                          or kpsefind_file (fname, "texmfscripts")
-            if not location then
-                errcnt = errcnt + 1
-                out ("FAILED: file %s missing.", fname)
-            else
-                out ("File: %s.", location)
-                local raw = ioloaddata (location)
-                if not raw then
-                    errcnt = errcnt + 1
-                    out ("FAILED: file %d not readable.", fname)
-                else
-                    local sum = md5sumhexa (raw)
-                    if sum ~= canonicalsum then
-                        errcnt = errcnt + 1
-                        out ("FAILED: checksum mismatch for file %s.",
-                             fname)
-                        out ("Expected %s.", canonicalsum)
-                        out ("Got      %s.", sum)
-                    else
-                        out ("Ok, %s passed.", fname)
-                    end
-                end
-            end
-        end
-        return errcnt
-    end
-
-    local get_tentative_attributes = function (file)
-        if not lfsisfile (file) then
-            local chan = ioopen (file, "w")
-            if chan then
-                chan:close ()
-                local attributes = lfsattributes (file)
-                os.remove (file)
-                return attributes
-            end
-        end
-    end
-
-    local p_permissions = Ct(Cg(Ct(C(1) * C(1) * C(1)), "u")
-                           * Cg(Ct(C(1) * C(1) * C(1)), "g")
-                           * Cg(Ct(C(1) * C(1) * C(1)), "o"))
-
-    local analyze_permissions = function (raw)
-        return lpegmatch (p_permissions, raw)
-    end
-
-    local stripslashes = names.patterns.stripslashes
-
-    local get_permissions = function (t, location)
-        if stringsub (location, #location) == "/" then
-            --- strip trailing slashes (lfs idiosyncrasy on Win)
-            location = lpegmatch (stripslashes, location)
-        end
-        local attributes = lfsattributes (location)
-
-        if not attributes and t == "f" then
-            attributes = get_tentative_attributes (location)
-            if not attributes then
-                return false
-            end
-        end
-
-        local permissions
-
-        if fileisreadable (location) then
-            --- link handling appears to be unnecessary because
-            --- lfs.attributes() will return the information on
-            --- the link target.
-            if mode == "link" then --follow and repeat
-                location = lfsreadlink (location)
-                attributes = lfsattributes (location)
-            end
-        end
-
-        permissions = analyze_permissions (attributes.permissions)
-
-        return {
-            location    = location,
-            mode        = attributes.mode,
-            owner       = attributes.uid, --- useless on windows
-            permissions = permissions,
-            attributes  = attributes,
-        }
-    end
-
-    local check_conformance = function (spec, permissions, errcnt)
-        local uid = permissions.attributes.uid
-        local gid = permissions.attributes.gid
-        local raw = permissions.attributes.permissions
-
-        out ("Owner: %d, group %d, permissions %s.", uid, gid, raw)
-        if ostype == "unix" then
-            if uid == 0 or gid == 0 then
-                out "Owned by the superuser, permission conflict likely."
-                errcnt = errcnt + 1
-            end
-        end
-
-        local user = permissions.permissions.u
-        if spec.r == true then
-            if user[1] == "r" then
-                out "Readable: ok."
-            else
-                out "Not readable: permissions need fixing."
-                errcnt = errcnt + 1
-            end
-        end
-
-        if spec.w == true then
-            if user[2] == "w"
-            or  fileiswritable (permissions.location) then
-                out "Writable: ok."
-            else
-                out "Not writable: permissions need fixing."
-                errcnt = errcnt + 1
-            end
-        end
-
-        return errcnt
-    end
-
-    local path = names.path
-
-    local desired_permissions = {
-        { "d", {"r","w"}, function () return caches.getwritablepath () end },
-        { "d", {"r","w"}, path.globals.prefix },
-        { "f", {"r","w"}, path.index.lua },
-        { "f", {"r","w"}, path.index.luc },
-        { "f", {"r","w"}, path.lookups.lua },
-        { "f", {"r","w"}, path.lookups.luc },
-    }
-
-    local check_permissions = function (errcnt)
-        out [[=============== file permissions ==============]]
-        for i = 1, #desired_permissions do
-            local t, spec, path = unpack (desired_permissions[i])
-            if type (path) == "function" then
-                path = path ()
-            end
-
-            spec = tabletohash (spec)
-
-            out ("Checking permissions of %s.", path)
-
-            local permissions = get_permissions (t, path)
-            if permissions then
-                --inspect (permissions)
-                errcnt = check_conformance (spec, permissions, errcnt)
-            else
-                errcnt = errcnt + 1
-            end
-        end
-        return errcnt
-    end
-
-    local check_upstream
-
-    if kpsefind_file ("https.lua", "lua") == nil then
-        check_upstream = function (errcnt)
-            out       [[============= upstream repository =============
-                        WARNING: Cannot retrieve repository data.
-                        Github API access requires the luasec library.
-                        Grab it from <https://github.com/brunoos/luasec>
-                        and retry.]]
-            return errcnt
-        end
-    else
-    --- github api stuff begin
-        local https = require "ssl.https"
-
-        local gh_api_root     = [[https://api.github.com]]
-        local release_url     = [[https://github.com/lualatex/luaotfload/releases]]
-        local luaotfload_repo = [[lualatex/luaotfload]]
-        local user_agent      = [[lualatex/luaotfload integrity check]]
-        local shortbytes = 8
-
-        local gh_shortrevision = function (rev)
-            return stringsub (rev, 1, shortbytes)
-        end
-
-        local gh_encode_parameters = function (parameters)
-            local acc = {}
-            for field, value in next, parameters do
-                --- unsafe, non-urlencoded coz it’s all ascii chars
-                acc[#acc+1] = field .. "=" .. value
-            end
-            return "?" .. tableconcat (acc, "&")
-        end
-
-        local gh_make_url = function (components, parameters)
-            local url = tableconcat ({ gh_api_root,
-                                       unpack (components) },
-                                     "/")
-            if parameters then
-                url = url .. gh_encode_parameters (parameters)
-            end
-            return url
-        end
-
-        local alright = [[HTTP/1.1 200 OK]]
-
-        local gh_api_request = function (...)
-            local args    = {...}
-            local nargs   = #args
-            local final   = args[nargs]
-            local request = {
-                url     = "",
-                headers = { ["user-agent"] = user_agent },
-            }
-            if type (final) == "table" then
-                args[nargs] = nil
-                request = gh_make_url (args, final)
-            else
-                request = gh_make_url (args)
-            end
-
-            out ("Requesting <%s>.", request)
-            local response, code, headers, status
-                = https.request (request)
-            if status ~= alright then
-                out "Request failed!"
-                return false
-            end
-            return response
-        end
-
-        local gh_api_checklimit = function (headers)
-            local rawlimit  = gh_api_request "rate_limit"
-            local limitdata = lua_of_json (rawlimit)
-            if not limitdata and limitdata.rate then
-                out "Cannot parse API rate limit."
-                return false
-            end
-            limitdata = limitdata.rate
-
-            local limit = tonumber (limitdata.limit)
-            local left  = tonumber (limitdata.remaining)
-            local reset = tonumber (limitdata.reset)
-
-            out ("%d of %d Github API requests left.", left, limit)
-            if left == 0 then
-                out ("Cannot make any more API requests.")
-                out ("Try again later at %s.", osdate ("%F %T", reset))
-            end
-            return true
-        end
-
-        local gh_tags = function ()
-            out "Fetching tags from repository, please stand by."
-            local rawtags = gh_api_request ("repos",
-                                            luaotfload_repo,
-                                            "tags")
-            local taglist = lua_of_json (rawtags)
-            if not taglist or #taglist == 0 then
-                out "Cannot parse response."
-                return false
-            end
-
-            local ntags = #taglist
-            out ("Repository contains %d tags.", ntags)
-            local _idx, latest = next (taglist)
-            out ("The most recent release is %s (revision %s).",
-                 latest.name,
-            gh_shortrevision (latest.commit.sha))
-            return latest
-        end
-
-        local gh_compare = function (head, base)
-            if base == nil then
-                base = "HEAD"
-            end
-            out ("Fetching comparison between %s and %s, \z
-                  please stand by.",
-                 gh_shortrevision (head),
-                 gh_shortrevision (base))
-            local comparison = base .. "..." .. head
-            local rawstatus = gh_api_request ("repos",
-                                              luaotfload_repo,
-                                              "compare",
-                                              comparison)
-            local status = lua_of_json (rawstatus)
-            if not status then
-                out "Cannot parse response for status request."
-                return false
-            end
-            return status
-        end
-
-        local gh_news = function (since)
-            local compared  = gh_compare (since)
-            if not compared then
-                return false
-            end
-            local behind_by = compared.behind_by
-            local ahead_by  = compared.ahead_by
-            local status    = compared.status
-            out ("Comparison state: %s.", status)
-            if behind_by > 0 then
-                out ("Your Luaotfload is %d \z
-                      revisions behind upstream.",
-                     behind_by)
-                return behind_by
-            elseif status == "ahead" then
-                out "Since you are obviously from the future \z
-                     I assume you already know the repository state."
-            else
-                out "Everything up to date. \z
-                     Luaotfload is in sync with upstream."
-            end
-            return false
-        end
-
-        local gh_catchup = function (current, latest)
-            local compared = gh_compare (latest, current)
-            local ahead_by = tonumber (compared.ahead_by)
-            if ahead_by > 0 then
-                local permalink_url = compared.permalink_url
-                out ("Your Luaotfload is %d revisions \z
-                      behind the most recent release.",
-                     ahead_by)
-                out ("To view the commit log, visit <%s>.",
-                     permalink_url)
-                out ("You can grab an up to date tarball at <%s>.",
-                     release_url)
-                return true
-            else
-                out "There weren't any new releases in the meantime."
-                out "Luaotfload is up to date."
-            end
-            return false
-        end
-
-        check_upstream = function (current)
-            out "============= upstream repository ============="
-            local _succ  = gh_api_checklimit ()
-            local behind = gh_news (current)
-            if behind then
-                local latest  = gh_tags ()
-                local _behind = gh_catchup (current,
-                                            latest.commit.sha,
-                                            latest.name)
-            end
-        end
-
-        --- trivium: diff since the first revision as pushed by Élie
-        --- in 2009
-        --- local firstrevision = "c3ccb3ee07e0a67171c24960966ae974e0dd8e98"
-        --- check_upstream (firstrevision)
-    end
-    --- github api stuff end
-
-    local print_envvar = function (var)
-        local val = osgetenv (var)
-        if val then
-            out ("%20s: %q", stringformat ("$%s", var), val)
-            return val
-        else
-            out ("%20s: <unset>", stringformat ("$%s", var))
-        end
-    end
-
-    local print_path = function (var)
-        local val = osgetenv (var)
-        if val then
-            local paths = filesplitpath (val)
-            if paths then
-                local npaths = #paths
-                if npaths == 1 then
-                    out ("%20s: %q", stringformat ("$%s", var), val)
-                elseif npaths > 1 then
-                    out ("%20s: <%d items>", stringformat ("$%s", var), npaths)
-                    for i = 1, npaths do
-                        out ("                   +: %q", paths[i])
-                    end
-                else
-                    out ("%20s: <empty>")
-                end
-            end
-        else
-            out ("%20s: <unset>", stringformat ("$%s", var))
-        end
-    end
-
-    local print_kpsevar = function (var)
-        var = "$" .. var
-        local val = kpseexpand_var (var)
-        if val and val ~= var then
-            out ("%20s: %q", var, val)
-            return val
-        else
-            out ("%20s: <empty or unset>", var)
-        end
-    end
-
-    local print_kpsepath = function (var)
-        var = "$" .. var
-        local val = kpseexpand_path (var)
-        if val and val ~= "" then
-            local paths = filesplitpath (val)
-            if paths then
-                local npaths = #paths
-                if npaths == 1 then
-                    out ("%20s: %q", var, paths[1])
-                elseif npaths > 1 then
-                    out ("%20s: <%d items>", var, npaths)
-                    for i = 1, npaths do
-                        out ("                   +: %q", paths[i])
-                    end
-                else
-                    out ("%20s: <empty>")
-                end
-            end
-        else
-            out ("%20s: <empty or unset>", var)
-        end
-    end
-
-    --- this test first if a variable is set and then expands the
-    --- paths; this is necessitated by the fact that expand-path will
-    --- return the empty string both if the variable is unset and if
-    --- the directory does not exist
-
-    local print_kpsepathvar = function (var)
-        local vvar = "$" .. var
-        local val = kpseexpand_var (vvar)
-        if val and val ~= vvar then
-            out ("%20s: %q", vvar, val)
-            print_kpsepath (var)
-        else
-            out ("%20s: <empty or unset>", var)
-        end
-    end
-
-    local check_environment = function (errcnt)
-        out "============ environment settings ============="
-        out ("system: %s/%s", ostype, osname)
-        if ostype == "unix" and io.popen then
-            local chan = io.popen ("uname -a", "r")
-            if chan then
-                out ("info: %s", chan:read "*all")
-                chan:close ()
-            end
-        end
-
-        out "1) *shell environment*"
-        print_envvar "SHELL"
-        print_path "PATH"
-        print_path "OSFONTDIR"
-        print_envvar "USER"
-        if ostype == "windows" then
-            print_envvar "WINDIR"
-            print_envvar "CD"
-            print_path "TEMP"
-        elseif ostype == "unix" then
-            print_envvar "HOME"
-            print_envvar "PWD"
-            print_path "TMPDIR"
-        end
-
-        out "2) *kpathsea*"
-        print_kpsepathvar "OPENTYPEFONTS"
-        print_kpsepathvar "TTFONTS"
-
-        print_kpsepathvar "TEXMFCACHE"
-        print_kpsepathvar "TEXMFVAR"
-
-        --- the expansion of these can be quite large; as they aren’t
-        --- usually essential to luaotfload, we won’t dump every single
-        --- path
-        print_kpsevar "LUAINPUTS"
-        print_kpsevar "CLUAINPUTS"
-
-        return errcnt
-    end
-
-    local anamneses   = {
-        "environment",
-        "files",
-        "repository",
-        "permissions"
-    }
-
-    actions.diagnose = function (job)
-        local errcnt = 0
-        local asked  = job.asked_diagnostics
-        if asked == "all" or asked == "thorough" then
-            asked = tabletohash (anamneses, true)
-        else
-            asked = lpegmatch(split_comma, asked)
-            asked = tabletohash (asked, true)
-        end
-
-        if asked.environment == true then
-            errcnt = check_environment (errcnt)
-            asked.environment = nil
-        end
-
-        if asked.files == true then
-            errcnt = verify_files (errcnt, status)
-            asked.files = nil
-        end
-
-        if asked.permissions == true then
-            errcnt = check_permissions (errcnt)
-            asked.permissions = nil
-        end
-
-        if asked.repository == true then
-            check_upstream (status.notes.revision)
-            asked.repository = nil
-        end
-
-        local rest = next (asked)
-        if rest ~= nil then --> something unknown
-            out ("Unknown diagnostic %q.", rest)
-        end
-        if errcnt == 0 then --> success
-            out ("Everything appears to be in order, \z
-                  you may sleep well.")
-            return true, false
-        end
-        out (         [[===============================================
-                                            WARNING
-                        ===============================================
-
-                        The diagnostic detected %d errors.
-
-                        This version of luaotfload may have been
-                        tampered with. Modified versions of the
-                        luaotfload source are unsupported. Read the log
-                        carefully and get a clean version from CTAN or
-                        github:
-
-                            × http://ctan.org/tex-archive/macros/luatex/generic/luaotfload
-                            × https://github.com/lualatex/luaotfload/releases
-
-                        If you are uncertain as to how to proceed, then
-                        ask on the lualatex mailing list:
-
-                            http://www.tug.org/mailman/listinfo/lualatex-dev
-
-                        ===============================================
-]],          errcnt)
-        return true, false
-    end
+actions.diagnose = function (job)
+    --- diagnostics are loaded on demand
+    local diagnose = require "luaotfload-diagnostics.lua"
+    return diagnose (job)
 end
 
 --- stuff to be carried out prior to exit
diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst
index e81aa8d..2e95381 100644
--- a/luaotfload-tool.rst
+++ b/luaotfload-tool.rst
@@ -135,9 +135,13 @@ miscellaneous
                            modifications;
                         2) ``permissions`` -> check permissions of
                            cache directories and files;
-                        3) ``repository`` -> check the git repository
+                        3) ``environment`` -> print relevant
+                            environment and kpse variables;
+                        4) ``repository`` -> check the git repository
                            for new releases.
 
+                        Procedures can be chained by concatenating with
+                        commas, e.g. ``--diagnose=files,permissions``.
                         Specify ``thorough`` to run all checks.
 
 FILES
diff --git a/luaotfload.dtx b/luaotfload.dtx
index 3892bd8..e42c14e 100644
--- a/luaotfload.dtx
+++ b/luaotfload.dtx
@@ -40,7 +40,7 @@
 \input docstrip.tex
 \Msg{************************************************************************}
 \Msg{* Installation}
-\Msg{* Package: luaotfload v2.3b OpenType layout system}
+\Msg{* Package: luaotfload v2.4 OpenType layout system}
 \Msg{************************************************************************}
 
 \keepsilent
@@ -111,7 +111,7 @@ and the derived files
 %<*driver>
 \NeedsTeXFormat{LaTeX2e}
 \ProvidesFile{luaotfload.drv}%
-  [2013/07/23 v2.3b OpenType layout system]%
+  [2013/07/28 v2.4 OpenType layout system]%
 \documentclass{ltxdoc}
 \usepackage{metalogo,multicol,mdwlist,fancyvrb,xspace}
 \usepackage[x11names]{xcolor}
@@ -231,7 +231,7 @@ and the derived files
 % \GetFileInfo{luaotfload.drv}
 %
 % \title{The \identifier{luaotfload} package}
-% \date{2013/07/23 v2.3b}
+% \date{2013/07/28 v2.4}
 % \author{Elie Roux · Khaled Hosny · Philipp Gesang\\
 %         Home:      \url{https://github.com/lualatex/luaotfload}\\
 %         Support:   \email{lualatex-dev@tug.org}}
@@ -1005,6 +1005,14 @@ and the derived files
 %   In \TEX Live: \fileent{texmf-dist/doc/luatex/base/luatexref-t.pdf}.
 % }
 %
+% For a much more detailed report about a given font try the \verb|-I| option
+% instead (\verb|--inspect|).
+% \begin{quote}
+%   \begin{verbatim}
+%     luaotfload-tool  -I --find="Iwona Light Italic"
+%   \end{verbatim}
+% \end{quote}
+%
 % \verb|luaotfload-tool --help| will list the available command line
 % switches, including some not discussed in detail here.
 % For a full documentation of \identifier{luaotfload-tool} and its
@@ -1383,6 +1391,7 @@ and the derived files
 %
 % \section{Troubleshooting}
 %
+% \subsection {Database Generation}
 % If you encounter problems with some fonts, please first update to the latest
 % version of this package before reporting a bug, as
 % \identifier{luaotfload} is under active development and still a
@@ -1392,18 +1401,42 @@ and the derived files
 % tracker for submitting bug reports, feature requests and the likes
 % requests and the likes.
 %
-% Errors during database generation can be traced by increasing
-% verbosity levels and redirecting log output to \fileent{stdout}:
+% Bug reports are more likely to be addressed if they contain the output of
 %
-% \begin{verbatim}
-%   luaotfload-tool -fuvvv --log=stdout
-% \end{verbatim}
+% \begin{quote}
+%   \begin{verbatim}
+%     luaotfload-tool --diagnose=environment,files,permissions
+%   \end{verbatim}
+% \end{quote}
+%
+% \noindent Consult the man page for a description of these options.
+%
+% Errors during database generation can be traced by increasing the
+% verbosity level and redirecting log output to \fileent{stdout}:
 %
-% If this fails, the font last printed to the terminal is likely to be
-% the culprit.
+% \begin{quote}
+%   \begin{verbatim}
+%     luaotfload-tool -fuvvv --log=stdout
+%   \end{verbatim}
+% \end{quote}
+%
+% \noindent or to a file in \fileent{/tmp}:
+%
+% \begin{quote}
+%   \begin{verbatim}
+%     luaotfload-tool -fuvvv --log=file
+%   \end{verbatim}
+% \end{quote}
+%
+% \noindent In the latter case, invoke the \verb|tail(1)| utility on the file
+% for live monitoring of the progress.
+%
+% If database generation fails, the font last printed to the terminal or log
+% file is likely to be the culprit.
 % Please specify it when reporting a bug, and blacklist it for the time
 % being (see above, page \pageref{font-blacklist}).
 %
+% \subsection {Font Features}
 % A common problem is the lack of features for some
 % \OpenType fonts even when specified.
 % This can be related to the fact that some fonts do not provide
@@ -1414,10 +1447,22 @@ and the derived files
 % fix it.
 % For example with \verb|latn|:
 %
-% \begin{verbatim}
-%   \font\test=file:MyFont.otf:script=latn;+liga;
-% \end{verbatim}
+% \begin{quote}
+%   \begin{verbatim}
+%     \font\test=file:MyFont.otf:script=latn;+liga;
+%   \end{verbatim}
+% \end{quote}
+%
+% You can get a list of features that a font defines for scripts and languages
+% by querying it in \fileent{luaotfload-tool}:
 %
+% \begin{quote}
+%   \begin{verbatim}
+%     luaotfload-tool --find="Iwona" --inspect
+%   \end{verbatim}
+% \end{quote}
+%
+% \subsection {\LUATEX Programming}
 % Another strategy that helps avoiding problems is to not access raw \LUATEX
 % internals directly.
 % Some of them, even though they are dangerous to access, have not been
@@ -1426,14 +1471,39 @@ and the derived files
 % \luafunction{aux} namespace over direct manipulation of font objects.
 % For example, raw access to the \luafunction{font.fonts} table like:
 %
-% \begin{verbatim}
-%   local somefont = font.fonts[2]
-% \end{verbatim}
+% \begin{quote}
+%   \begin{verbatim}
+%     local somefont = font.fonts[2]
+%   \end{verbatim}
+% \end{quote}
 %
-% can render already defined fonts unusable.
+% \noindent can render already defined fonts unusable.
 % Instead, the function \luafunction{font.getfont()} should be used because
 % it has been replaced by a safe variant.
 %
+% However, \luafunction{font.getfont()} only covers fonts handled by the font
+% loader, e.~g. \identifier{OpenType} and \identifier{TrueType} fonts, but
+% not \abbrev{tfm} or \abbrev{ofm}.
+% Should you absolutely require access to all fonts known to \LUATEX, including
+% the virtual and autogenerated ones, then you need to query both
+% \luafunction{font.getfont()} and \luafunction{font.fonts}.
+% In this case, best define you own accessor:
+%
+% \begin{quote}
+%   \begin{verbatim}
+%     local unsafe_getfont = function (id)
+%         local tfmdata = font.getfont (id)
+%         if not tfmdata then
+%             tfmdata = font.fonts[id]
+%         end
+%         return tfmdata
+%     end
+%
+%     --- use like getfont()
+%     local somefont = unsafe_getfont (2)
+%   \end{verbatim}
+% \end{quote}
+%
 % \part{Implementation}
 %
 % \section{\fileent{luaotfload.lua}}
@@ -1478,8 +1548,8 @@ config.luaotfload.index_file      = config.luaotfload.index_file       or "luaot
 
 luaotfload.module = {
     name          = "luaotfload",
-    version       = 2.3002,
-    date          = "2013/07/23",
+    version       = 2.40001,
+    date          = "2013/07/28",
     description   = "OpenType layout system.",
     author        = "Elie Roux & Hans Hagen",
     copyright     = "Elie Roux",
@@ -2070,7 +2140,7 @@ luaotfload.aux.start_rewrite_fontname () --- to be migrated to fontspec
 \else
   \NeedsTeXFormat{LaTeX2e}
   \ProvidesPackage{luaotfload}%
-    [2013/07/23 v2.3b OpenType layout system]
+    [2013/07/28 v2.4 OpenType layout system]
   \RequirePackage{luatexbase}
 \fi
 \ifnum\luatexversion<76
diff --git a/mkstatus b/mkstatus
index 97d9f04..16d0b5d 100755
--- a/mkstatus
+++ b/mkstatus
@@ -39,6 +39,7 @@ local names = {
   "luaotfload-characters.lua",
   "luaotfload-colors.lua",
   "luaotfload-database.lua",
+  "luaotfload-diagnostics.lua",
   "luaotfload-extralibs.lua",
   "luaotfload-features.lua",
   "luaotfload-fonts-cbk.lua",