diff options
| -rw-r--r-- | filegraph.dot | 80 | ||||
| -rw-r--r-- | luaotfload-database.lua | 10 | ||||
| -rw-r--r-- | luaotfload-diagnostics.lua | 622 | ||||
| -rwxr-xr-x | luaotfload-tool.lua | 637 | ||||
| -rw-r--r-- | luaotfload-tool.rst | 6 | ||||
| -rw-r--r-- | luaotfload.dtx | 110 | ||||
| -rwxr-xr-x | mkstatus | 1 |
7 files changed, 834 insertions, 632 deletions
diff --git a/filegraph.dot b/filegraph.dot index c8a07d3..e412279 100644 --- a/filegraph.dot +++ b/filegraph.dot @@ -51,18 +51,55 @@ strict digraph luaotfload_files { //looks weird with circo ... merged_luatex_fonts -> font_age [label="luatex-fonts-enc.lua", ltail=cluster_merged] + fontdbutil -> fontdbutil_diagnostics [label="--diagnose"] + + fontdbutil -> status [label="version information"] + + fontdbutil_diagnostics -> status [constraint=no, label="hash files"] + + merged_luatex_fonts -> characters [label="luaotfload-auxiliary.lua", + ltail=cluster_merged] + luaotfload_libs -> font_names [label="luaotfload-database.lua"] + luaotfload_libs -> typo_krn [label="luaotfload-extralibs.lua"] + + mkstatus -> status [label="generates from distribution files", + style=dashed] mkglyphlist -> font_age [label="generates from glyphlist.txt", style=dashed] - subgraph { rank = same; mkglyphlist; fontdbutil; luaotfload } + mkcharacters -> characters [label="generates from Context’s char-def.lua", + style=dashed] + + subgraph { rank = same; + mkcharacters; + mkglyphlist; + mkstatus; + fontdbutil; + luaotfload } /* ···································································· * main files * ································································· */ - fontdbutil [label = "luaotfload-util\nmkluatexfontdb.lua", + fontdbutil [label = "luaotfload-tool.lua\nmkluatexfontdb.lua", + shape = rect, + width = "3.2cm", + height = "1.2cm", + color = "#01012222", + style = "filled,rounded", + penwidth=2] + + fontdbutil_diagnostics [label = "luaotfload-diagnostics.lua", + shape = rect, + width = "3.2cm", + height = "1.2cm", + color = "#01012222", + style = "filled,rounded", + penwidth=2] + + mkstatus [label = "mkstatus", shape = rect, width = "3.2cm", height = "1.2cm", @@ -78,6 +115,14 @@ strict digraph luaotfload_files { //looks weird with circo ... style = "filled,rounded", penwidth=2] + mkcharacters [label = "mkcharacters", + shape = rect, + width = "3.2cm", + height = "1.2cm", + color = "#01012222", + style = "filled,rounded", + penwidth=2] + luaotfload [label = "luaotfload.lua", shape = rect, width = "3.2cm", @@ -102,11 +147,27 @@ strict digraph luaotfload_files { //looks weird with circo ... style = "filled,rounded", penwidth=2] + typo_krn [label = "luaotfload-typo-krn.lua", + shape = rect, + width = "3.2cm", + height = "1.2cm", + color = "#01012222", + style = "filled,rounded", + penwidth=2] + /* ···································································· * luaotfload files * ································································· */ + characters [style = "filled,dashed", + shape = rect, + width = "3.2cm", + fillcolor = "#01012222", + color = grey40, + style = "filled,dotted,rounded", + label = "luaotfload-characters.lua"] + font_age [style = "filled,dashed", shape = rect, width = "3.2cm", @@ -123,6 +184,14 @@ strict digraph luaotfload_files { //looks weird with circo ... style = "filled,dotted,rounded", label = "luaotfload-names.lua\nluaotfload-names.luc"] + status [style = "filled,dashed", + shape = rect, + width = "3.2cm", + fillcolor = "#01012222", + color = grey40, + style = "filled,dotted,rounded", + label = "luaotfload-status.lua"] + otfl_blacklist_cnf [style = "filled,dashed", shape = rect, width = "3.2cm", @@ -139,9 +208,10 @@ strict digraph luaotfload_files { //looks weird with circo ... label = < <table cellborder="0" bgcolor="#FFFFFFAA"> <th> <td colspan="2"> <font point-size="12" face="Iwona Italic">Luaotfload Libraries</font> </td> </th> - <tr> <td>luaotfload-auxiliary.lua</td> <td>luaotfload-features.lua</td> </tr> - <tr> <td>luaotfload-override.lua</td> <td>luaotfload-loaders.lua</td> </tr> - <tr> <td>luaotfload-database.lua</td> <td>luaotfload-color.lua</td> </tr> + <tr> <td>luaotfload-auxiliary.lua</td> <td>luaotfload-features.lua</td> </tr> + <tr> <td>luaotfload-override.lua</td> <td>luaotfload-loaders.lua</td> </tr> + <tr> <td>luaotfload-database.lua</td> <td>luaotfload-color.lua</td> </tr> + <tr> <td>luaotfload-extralibs.lua</td> <td>luaotfload-letterspace.lua</td> </tr> </table> >, ] diff --git a/luaotfload-database.lua b/luaotfload-database.lua index a49e575..a0a532e 100644 --- a/luaotfload-database.lua +++ b/luaotfload-database.lua @@ -117,9 +117,17 @@ end local report = logs.names_report +names.patterns = { } +local patterns = names.patterns + local trailingslashes = P"/"^1 * P(-1) local stripslashes = C((1 - trailingslashes)^0) -names.patterns = { stripslashes = stripslashes } +patterns.stripslashes = stripslashes + +local comma = P"," +local noncomma = 1-comma +local splitcomma = Ct((C(noncomma^1) + comma)^1) +patterns.splitcomma = splitcomma --[[doc-- We use the functions in the cache.* namespace that come with the diff --git a/luaotfload-diagnostics.lua b/luaotfload-diagnostics.lua new file mode 100644 index 0000000..db807de --- /dev/null +++ b/luaotfload-diagnostics.lua @@ -0,0 +1,622 @@ +#!/usr/bin/env texlua +----------------------------------------------------------------------- +-- FILE: luaotfload-diagnostics.lua +-- DESCRIPTION: functionality accessible by the --diagnose option +-- REQUIREMENTS: luaotfload-tool.lua +-- AUTHOR: Philipp Gesang (Phg), <phg42.2a@gmail.com> +-- VERSION: 1.0 +-- CREATED: 2013-07-28 10:01:18+0200 +----------------------------------------------------------------------- +-- +local names = fonts.names +local status = config.luaotfload.status + +local kpse = require "kpse" +local kpseexpand_path = kpse.expand_path +local kpseexpand_var = kpse.expand_var +local kpsefind_file = kpse.find_file + +local lfs = require "lfs" +local lfsattributes = lfs.attributes +local lfsisfile = lfs.isfile +local lfsreadlink = lfs.readlink + +local md5 = require "md5" +local md5sumhexa = md5.sumhexa + +local osgetenv = os.getenv +local osname = os.name +local osremove = os.remove +local ostype = os.type +local stringformat = string.format +local stringsub = string.sub + +local fileisreadable = file.isreadable +local fileiswritable = file.iswritable +local filesplitpath = file.splitpath +local ioloaddata = io.loaddata +local lua_of_json = utilities.json.tolua +local tableconcat = table.concat +local tabletohash = table.tohash + +local lpeg = require "lpeg" +local C, Cg, Ct = lpeg.C, lpeg.Cg, lpeg.Ct +local lpegmatch = lpeg.match + +local out = function (...) + logs.names_report (false, 0, "diagnose", ...) +end + +local verify_files = function (errcnt, status) + out "================ verify files =================" + local hashes = status.hashes + local notes = status.notes + if not hashes or #hashes == 0 then + out ("FAILED: cannot read checksums from %s.", status_file) + return 1/0 + elseif not notes then + out ("FAILED: cannot read commit metadata from %s.", + status_file) + return 1/0 + end + + out ("Luaotfload revision %s.", notes.revision) + out ("Committed by %s.", notes.committer) + out ("Timestamp %s.", notes.timestamp) + + local nhashes = #hashes + out ("Testing %d files for integrity.", nhashes) + for i = 1, nhashes do + local fname, canonicalsum = unpack (hashes[i]) + local location = kpsefind_file (fname) + or kpsefind_file (fname, "texmfscripts") + if not location then + errcnt = errcnt + 1 + out ("FAILED: file %s missing.", fname) + else + out ("File: %s.", location) + local raw = ioloaddata (location) + if not raw then + errcnt = errcnt + 1 + out ("FAILED: file %d not readable.", fname) + else + local sum = md5sumhexa (raw) + if sum ~= canonicalsum then + errcnt = errcnt + 1 + out ("FAILED: checksum mismatch for file %s.", + fname) + out ("Expected %s.", canonicalsum) + out ("Got %s.", sum) + else + out ("Ok, %s passed.", fname) + end + end + end + end + return errcnt +end + +local get_tentative_attributes = function (file) + if not lfsisfile (file) then + local chan = ioopen (file, "w") + if chan then + chan:close () + local attributes = lfsattributes (file) + os.remove (file) + return attributes + end + end +end + +local p_permissions = Ct(Cg(Ct(C(1) * C(1) * C(1)), "u") + * Cg(Ct(C(1) * C(1) * C(1)), "g") + * Cg(Ct(C(1) * C(1) * C(1)), "o")) + +local analyze_permissions = function (raw) + return lpegmatch (p_permissions, raw) +end + +local stripslashes = names.patterns.stripslashes + +local get_permissions = function (t, location) + if stringsub (location, #location) == "/" then + --- strip trailing slashes (lfs idiosyncrasy on Win) + location = lpegmatch (stripslashes, location) + end + local attributes = lfsattributes (location) + + if not attributes and t == "f" then + attributes = get_tentative_attributes (location) + if not attributes then + return false + end + end + + local permissions + + if fileisreadable (location) then + --- link handling appears to be unnecessary because + --- lfs.attributes() will return the information on + --- the link target. + if mode == "link" then --follow and repeat + location = lfsreadlink (location) + attributes = lfsattributes (location) + end + end + + permissions = analyze_permissions (attributes.permissions) + + return { + location = location, + mode = attributes.mode, + owner = attributes.uid, --- useless on windows + permissions = permissions, + attributes = attributes, + } +end + +local check_conformance = function (spec, permissions, errcnt) + local uid = permissions.attributes.uid + local gid = permissions.attributes.gid + local raw = permissions.attributes.permissions + + out ("Owner: %d, group %d, permissions %s.", uid, gid, raw) + if ostype == "unix" then + if uid == 0 or gid == 0 then + out "Owned by the superuser, permission conflict likely." + errcnt = errcnt + 1 + end + end + + local user = permissions.permissions.u + if spec.r == true then + if user[1] == "r" then + out "Readable: ok." + else + out "Not readable: permissions need fixing." + errcnt = errcnt + 1 + end + end + + if spec.w == true then + if user[2] == "w" + or fileiswritable (permissions.location) then + out "Writable: ok." + else + out "Not writable: permissions need fixing." + errcnt = errcnt + 1 + end + end + + return errcnt +end + +local path = names.path + +local desired_permissions = { + { "d", {"r","w"}, function () return caches.getwritablepath () end }, + { "d", {"r","w"}, path.globals.prefix }, + { "f", {"r","w"}, path.index.lua }, + { "f", {"r","w"}, path.index.luc }, + { "f", {"r","w"}, path.lookups.lua }, + { "f", {"r","w"}, path.lookups.luc }, +} + +local check_permissions = function (errcnt) + out [[=============== file permissions ==============]] + for i = 1, #desired_permissions do + local t, spec, path = unpack (desired_permissions[i]) + if type (path) == "function" then + path = path () + end + + spec = tabletohash (spec) + + out ("Checking permissions of %s.", path) + + local permissions = get_permissions (t, path) + if permissions then + --inspect (permissions) + errcnt = check_conformance (spec, permissions, errcnt) + else + errcnt = errcnt + 1 + end + end + return errcnt +end + +local check_upstream + +if kpsefind_file ("https.lua", "lua") == nil then + check_upstream = function (errcnt) + out [[============= upstream repository ============= + WARNING: Cannot retrieve repository data. + Github API access requires the luasec library. + Grab it from <https://github.com/brunoos/luasec> + and retry.]] + return errcnt + end +else +--- github api stuff begin + local https = require "ssl.https" + + local gh_api_root = [[https://api.github.com]] + local release_url = [[https://github.com/lualatex/luaotfload/releases]] + local luaotfload_repo = [[lualatex/luaotfload]] + local user_agent = [[lualatex/luaotfload integrity check]] + local shortbytes = 8 + + local gh_shortrevision = function (rev) + return stringsub (rev, 1, shortbytes) + end + + local gh_encode_parameters = function (parameters) + local acc = {} + for field, value in next, parameters do + --- unsafe, non-urlencoded coz it’s all ascii chars + acc[#acc+1] = field .. "=" .. value + end + return "?" .. tableconcat (acc, "&") + end + + local gh_make_url = function (components, parameters) + local url = tableconcat ({ gh_api_root, + unpack (components) }, + "/") + if parameters then + url = url .. gh_encode_parameters (parameters) + end + return url + end + + local alright = [[HTTP/1.1 200 OK]] + + local gh_api_request = function (...) + local args = {...} + local nargs = #args + local final = args[nargs] + local request = { + url = "", + headers = { ["user-agent"] = user_agent }, + } + if type (final) == "table" then + args[nargs] = nil + request = gh_make_url (args, final) + else + request = gh_make_url (args) + end + + out ("Requesting <%s>.", request) + local response, code, headers, status + = https.request (request) + if status ~= alright then + out "Request failed!" + return false + end + return response + end + + local gh_api_checklimit = function (headers) + local rawlimit = gh_api_request "rate_limit" + local limitdata = lua_of_json (rawlimit) + if not limitdata and limitdata.rate then + out "Cannot parse API rate limit." + return false + end + limitdata = limitdata.rate + + local limit = tonumber (limitdata.limit) + local left = tonumber (limitdata.remaining) + local reset = tonumber (limitdata.reset) + + out ("%d of %d Github API requests left.", left, limit) + if left == 0 then + out ("Cannot make any more API requests.") + out ("Try again later at %s.", osdate ("%F %T", reset)) + end + return true + end + + local gh_tags = function () + out "Fetching tags from repository, please stand by." + local rawtags = gh_api_request ("repos", + luaotfload_repo, + "tags") + local taglist = lua_of_json (rawtags) + if not taglist or #taglist == 0 then + out "Cannot parse response." + return false + end + + local ntags = #taglist + out ("Repository contains %d tags.", ntags) + local _idx, latest = next (taglist) + out ("The most recent release is %s (revision %s).", + latest.name, + gh_shortrevision (latest.commit.sha)) + return latest + end + + local gh_compare = function (head, base) + if base == nil then + base = "HEAD" + end + out ("Fetching comparison between %s and %s, \z + please stand by.", + gh_shortrevision (head), + gh_shortrevision (base)) + local comparison = base .. "..." .. head + local rawstatus = gh_api_request ("repos", + luaotfload_repo, + "compare", + comparison) + local status = lua_of_json (rawstatus) + if not status then + out "Cannot parse response for status request." + return false + end + return status + end + + local gh_news = function (since) + local compared = gh_compare (since) + if not compared then + return false + end + local behind_by = compared.behind_by + local ahead_by = compared.ahead_by + local status = compared.status + out ("Comparison state: %s.", status) + if behind_by > 0 then + out ("Your Luaotfload is %d \z + revisions behind upstream.", + behind_by) + return behind_by + elseif status == "ahead" then + out "Since you are obviously from the future \z + I assume you already know the repository state." + else + out "Everything up to date. \z + Luaotfload is in sync with upstream." + end + return false + end + + local gh_catchup = function (current, latest) + local compared = gh_compare (latest, current) + local ahead_by = tonumber (compared.ahead_by) + if ahead_by > 0 then + local permalink_url = compared.permalink_url + out ("Your Luaotfload is %d revisions \z + behind the most recent release.", + ahead_by) + out ("To view the commit log, visit <%s>.", + permalink_url) + out ("You can grab an up to date tarball at <%s>.", + release_url) + return true + else + out "There weren't any new releases in the meantime." + out "Luaotfload is up to date." + end + return false + end + + check_upstream = function (current) + out "============= upstream repository =============" + local _succ = gh_api_checklimit () + local behind = gh_news (current) + if behind then + local latest = gh_tags () + local _behind = gh_catchup (current, + latest.commit.sha, + latest.name) + end + end + + --- trivium: diff since the first revision as pushed by Élie + --- in 2009 + --- local firstrevision = "c3ccb3ee07e0a67171c24960966ae974e0dd8e98" + --- check_upstream (firstrevision) +end +--- github api stuff end + +local print_envvar = function (var) + local val = osgetenv (var) + if val then + out ("%20s: %q", stringformat ("$%s", var), val) + return val + else + out ("%20s: <unset>", stringformat ("$%s", var)) + end +end + +local print_path = function (var) + local val = osgetenv (var) + if val then + local paths = filesplitpath (val) + if paths then + local npaths = #paths + if npaths == 1 then + out ("%20s: %q", stringformat ("$%s", var), val) + elseif npaths > 1 then + out ("%20s: <%d items>", stringformat ("$%s", var), npaths) + for i = 1, npaths do + out (" +: %q", paths[i]) + end + else + out ("%20s: <empty>") + end + end + else + out ("%20s: <unset>", stringformat ("$%s", var)) + end +end + +local print_kpsevar = function (var) + var = "$" .. var + local val = kpseexpand_var (var) + if val and val ~= var then + out ("%20s: %q", var, val) + return val + else + out ("%20s: <empty or unset>", var) + end +end + +local print_kpsepath = function (var) + var = "$" .. var + local val = kpseexpand_path (var) + if val and val ~= "" then + local paths = filesplitpath (val) + if paths then + local npaths = #paths + if npaths == 1 then + out ("%20s: %q", var, paths[1]) + elseif npaths > 1 then + out ("%20s: <%d items>", var, npaths) + for i = 1, npaths do + out (" +: %q", paths[i]) + end + else + out ("%20s: <empty>") + end + end + else + out ("%20s: <empty or unset>", var) + end +end + +--- this test first if a variable is set and then expands the +--- paths; this is necessitated by the fact that expand-path will +--- return the empty string both if the variable is unset and if +--- the directory does not exist + +local print_kpsepathvar = function (var) + local vvar = "$" .. var + local val = kpseexpand_var (vvar) + if val and val ~= vvar then + out ("%20s: %q", vvar, val) + print_kpsepath (var) + else + out ("%20s: <empty or unset>", var) + end +end + +local check_environment = function (errcnt) + out "============ environment settings =============" + out ("system: %s/%s", ostype, osname) + if ostype == "unix" and io.popen then + local chan = io.popen ("uname -a", "r") + if chan then + out ("info: %s", chan:read "*all") + chan:close () + end + end + + out "1) *shell environment*" + print_envvar "SHELL" + print_path "PATH" + print_path "OSFONTDIR" + print_envvar "USER" + if ostype == "windows" then + print_envvar "WINDIR" + print_envvar "CD" + print_path "TEMP" + elseif ostype == "unix" then + print_envvar "HOME" + print_envvar "PWD" + print_path "TMPDIR" + end + + out "2) *kpathsea*" + print_kpsepathvar "OPENTYPEFONTS" + print_kpsepathvar "TTFONTS" + + print_kpsepathvar "TEXMFCACHE" + print_kpsepathvar "TEXMFVAR" + + --- the expansion of these can be quite large; as they aren’t + --- usually essential to luaotfload, we won’t dump every single + --- path + print_kpsevar "LUAINPUTS" + print_kpsevar "CLUAINPUTS" + + return errcnt +end + +local anamneses = { + "environment", + "files", + "repository", + "permissions" +} + +local splitcomma = names.patterns.splitcomma + +local diagnose = function (job) + local errcnt = 0 + local asked = job.asked_diagnostics + if asked == "all" or asked == "thorough" then + asked = tabletohash (anamneses, true) + else + asked = lpegmatch (splitcomma, asked) + asked = tabletohash (asked, true) + end + + if asked.environment == true then + errcnt = check_environment (errcnt) + asked.environment = nil + end + + if asked.files == true then + errcnt = verify_files (errcnt, status) + asked.files = nil + end + + if asked.permissions == true then + errcnt = check_permissions (errcnt) + asked.permissions = nil + end + + if asked.repository == true then + check_upstream (status.notes.revision) + asked.repository = nil + end + + local rest = next (asked) + if rest ~= nil then --> something unknown + out ("Unknown diagnostic %q.", rest) + end + if errcnt == 0 then --> success + out ("Everything appears to be in order, \z + you may sleep well.") + return true, false + end + out ( [[=============================================== + WARNING + =============================================== + + The diagnostic detected %d errors. + + This version of luaotfload may have been + tampered with. Modified versions of the + luaotfload source are unsupported. Read the log + carefully and get a clean version from CTAN or + github: + + × http://ctan.org/tex-archive/macros/luatex/generic/luaotfload + × https://github.com/lualatex/luaotfload/releases + + If you are uncertain as to how to proceed, then + ask on the lualatex mailing list: + + http://www.tug.org/mailman/listinfo/lualatex-dev + + =============================================== +]], errcnt) + return true, false +end + +return diagnose + diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua index cd64a75..92483d6 100755 --- a/luaotfload-tool.lua +++ b/luaotfload-tool.lua @@ -4,11 +4,13 @@ -- DESCRIPTION: database functionality -- REQUIREMENTS: luaotfload 2.2 -- AUTHOR: Khaled Hosny, Élie Roux, Philipp Gesang --- VERSION: 2.3b +-- VERSION: 2.4 -- LICENSE: GPL v2 --- MODIFIED: 2013-06-02 19:23:54+0200 +-- MODIFIED: 2013-07-28 13:12:04+0200 ----------------------------------------------------------------------- +local version = "2.4" --- <int: major>.<int: minor><alpha: fixes> + --[[doc-- This file was originally written (as \fileent{mkluatexfontdb.lua}) by @@ -42,23 +44,12 @@ kpse.set_program_name "luatex" local ioopen = io.open local iowrite = io.write local kpsefind_file = kpse.find_file -local kpseexpand_var = kpse.expand_var -local kpseexpand_path = kpse.expand_path -local lfsattributes = lfs.attributes -local lfsisfile = lfs.isfile -local lfsreadlink = lfs.readlink -local md5sumhexa = md5.sumhexa local next = next local osdate = os.date -local osgetenv = os.getenv -local osremove = os.remove -local osname = os.name -local ostype = os.type local stringexplode = string.explode local stringformat = string.format local stringlower = string.lower local stringrep = string.rep -local stringsub = string.sub local tableconcat = table.concat local texiowrite_nl = texio.write_nl local texiowrite = texio.write @@ -79,7 +70,7 @@ else -- 5.2 end -local C, Cg, Ct, P, S = lpeg.C, lpeg.Cg, lpeg.Ct, lpeg.P, lpeg.S +local C, Ct, P, S = lpeg.C, lpeg.Ct, lpeg.P, lpeg.S local lpegmatch = lpeg.match local loader_file = "luatexbase.loader.lua" @@ -111,6 +102,7 @@ to be the more appropriate. config = config or { } local config = config config.luaotfload = config.luaotfload or { } +config.luaotfload.version = config.luaotfload.version or version config.luaotfload.names_dir = config.luaotfload.names_dir or "names" config.luaotfload.cache_dir = config.luaotfload.cache_dir or "fonts" config.luaotfload.index_file = config.luaotfload.index_file @@ -140,26 +132,33 @@ config.lualibs.prefer_merged = true config.lualibs.load_extended = true require "lualibs" - -local fileisreadable = file.isreadable -local fileiswritable = file.iswritable -local filesplitpath = file.splitpath -local ioloaddata = io.loaddata -local lua_of_json = utilities.json.tolua local tabletohash = table.tohash --[[doc-- \fileent{luatex-basics-gen.lua} calls functions from the \luafunction{texio.*} library; too much for our taste. We intercept them with dummies. + +Also, it sets up dummies in place of the tables created by the Context +libraries. Since we have loaded the lualibs already this would cause +collateral damage for some libraries whose namespace would be +overridden. We employ our usual backup-restore strategy to work around +this. (Postponing the loading of the lualibs code is not an option +because the functionality is needed by basics-gen itself.) --doc]]-- local dummy_function = function ( ) end -local backup_write, backup_write_nl = texio.write, texio.write_nl +local backup = { + write = texio.write, + write_nl = texio.write_nl, + utilities = utilities, +} texio.write, texio.write_nl = dummy_function, dummy_function require"luaotfload-basics-gen.lua" -texio.write, texio.write_nl = backup_write, backup_write_nl + +texio.write, texio.write_nl = backup.write, backup.write_nl +utilities = backup.utilities require"luaotfload-override.lua" --- this populates the logs.* namespace require"luaotfload-database" @@ -167,12 +166,11 @@ require"alt_getopt" local names = fonts.names -local status_file = "luaotfload-status" -local status = require (status_file) - -local version = "2.3b" +local status_file = "luaotfload-status" +local status = require (status_file) +config.luaotfload.status = status -local sanitize_string = names.sanitize_string +local sanitize_string = names.sanitize_string local pathdata = names.path local names_plain = pathdata.index.lua @@ -830,10 +828,6 @@ local get_fields get_fields = function (entry, fields, acc, n) return acc end -local comma = P"," -local noncomma = 1-comma -local split_comma = Ct((C(noncomma^1) + comma)^1) - local separator = "\t" --- could be “,” for csv local format_fields format_fields = function (fields, acc, n) @@ -869,12 +863,14 @@ set_primary_field = function (fields, addme, acc, n) return acc end +local splitcomma = names.patterns.splitcomma + actions.list = function (job) local criterion = job.criterion local asked_fields = job.asked_fields if asked_fields then - asked_fields = lpegmatch(split_comma, asked_fields) + asked_fields = lpegmatch(splitcomma, asked_fields) else --- some defaults asked_fields = { "fullname", "version", } @@ -959,579 +955,10 @@ actions.list = function (job) return true, true end -do - local out = function (...) - logs.names_report (false, 0, "diagnose", ...) - end - - local verify_files = function (errcnt, status) - out "================ verify files =================" - local hashes = status.hashes - local notes = status.notes - if not hashes or #hashes == 0 then - out ("FAILED: cannot read checksums from %s.", status_file) - return 1/0 - elseif not notes then - out ("FAILED: cannot read commit metadata from %s.", - status_file) - return 1/0 - end - - out ("Luaotfload revision %s.", notes.revision) - out ("Committed by %s.", notes.committer) - out ("Timestamp %s.", notes.timestamp) - - local nhashes = #hashes - out ("Testing %d files for integrity.", nhashes) - for i = 1, nhashes do - local fname, canonicalsum = unpack (hashes[i]) - local location = kpsefind_file (fname) - or kpsefind_file (fname, "texmfscripts") - if not location then - errcnt = errcnt + 1 - out ("FAILED: file %s missing.", fname) - else - out ("File: %s.", location) - local raw = ioloaddata (location) - if not raw then - errcnt = errcnt + 1 - out ("FAILED: file %d not readable.", fname) - else - local sum = md5sumhexa (raw) - if sum ~= canonicalsum then - errcnt = errcnt + 1 - out ("FAILED: checksum mismatch for file %s.", - fname) - out ("Expected %s.", canonicalsum) - out ("Got %s.", sum) - else - out ("Ok, %s passed.", fname) - end - end - end - end - return errcnt - end - - local get_tentative_attributes = function (file) - if not lfsisfile (file) then - local chan = ioopen (file, "w") - if chan then - chan:close () - local attributes = lfsattributes (file) - os.remove (file) - return attributes - end - end - end - - local p_permissions = Ct(Cg(Ct(C(1) * C(1) * C(1)), "u") - * Cg(Ct(C(1) * C(1) * C(1)), "g") - * Cg(Ct(C(1) * C(1) * C(1)), "o")) - - local analyze_permissions = function (raw) - return lpegmatch (p_permissions, raw) - end - - local stripslashes = names.patterns.stripslashes - - local get_permissions = function (t, location) - if stringsub (location, #location) == "/" then - --- strip trailing slashes (lfs idiosyncrasy on Win) - location = lpegmatch (stripslashes, location) - end - local attributes = lfsattributes (location) - - if not attributes and t == "f" then - attributes = get_tentative_attributes (location) - if not attributes then - return false - end - end - - local permissions - - if fileisreadable (location) then - --- link handling appears to be unnecessary because - --- lfs.attributes() will return the information on - --- the link target. - if mode == "link" then --follow and repeat - location = lfsreadlink (location) - attributes = lfsattributes (location) - end - end - - permissions = analyze_permissions (attributes.permissions) - - return { - location = location, - mode = attributes.mode, - owner = attributes.uid, --- useless on windows - permissions = permissions, - attributes = attributes, - } - end - - local check_conformance = function (spec, permissions, errcnt) - local uid = permissions.attributes.uid - local gid = permissions.attributes.gid - local raw = permissions.attributes.permissions - - out ("Owner: %d, group %d, permissions %s.", uid, gid, raw) - if ostype == "unix" then - if uid == 0 or gid == 0 then - out "Owned by the superuser, permission conflict likely." - errcnt = errcnt + 1 - end - end - - local user = permissions.permissions.u - if spec.r == true then - if user[1] == "r" then - out "Readable: ok." - else - out "Not readable: permissions need fixing." - errcnt = errcnt + 1 - end - end - - if spec.w == true then - if user[2] == "w" - or fileiswritable (permissions.location) then - out "Writable: ok." - else - out "Not writable: permissions need fixing." - errcnt = errcnt + 1 - end - end - - return errcnt - end - - local path = names.path - - local desired_permissions = { - { "d", {"r","w"}, function () return caches.getwritablepath () end }, - { "d", {"r","w"}, path.globals.prefix }, - { "f", {"r","w"}, path.index.lua }, - { "f", {"r","w"}, path.index.luc }, - { "f", {"r","w"}, path.lookups.lua }, - { "f", {"r","w"}, path.lookups.luc }, - } - - local check_permissions = function (errcnt) - out [[=============== file permissions ==============]] - for i = 1, #desired_permissions do - local t, spec, path = unpack (desired_permissions[i]) - if type (path) == "function" then - path = path () - end - - spec = tabletohash (spec) - - out ("Checking permissions of %s.", path) - - local permissions = get_permissions (t, path) - if permissions then - --inspect (permissions) - errcnt = check_conformance (spec, permissions, errcnt) - else - errcnt = errcnt + 1 - end - end - return errcnt - end - - local check_upstream - - if kpsefind_file ("https.lua", "lua") == nil then - check_upstream = function (errcnt) - out [[============= upstream repository ============= - WARNING: Cannot retrieve repository data. - Github API access requires the luasec library. - Grab it from <https://github.com/brunoos/luasec> - and retry.]] - return errcnt - end - else - --- github api stuff begin - local https = require "ssl.https" - - local gh_api_root = [[https://api.github.com]] - local release_url = [[https://github.com/lualatex/luaotfload/releases]] - local luaotfload_repo = [[lualatex/luaotfload]] - local user_agent = [[lualatex/luaotfload integrity check]] - local shortbytes = 8 - - local gh_shortrevision = function (rev) - return stringsub (rev, 1, shortbytes) - end - - local gh_encode_parameters = function (parameters) - local acc = {} - for field, value in next, parameters do - --- unsafe, non-urlencoded coz it’s all ascii chars - acc[#acc+1] = field .. "=" .. value - end - return "?" .. tableconcat (acc, "&") - end - - local gh_make_url = function (components, parameters) - local url = tableconcat ({ gh_api_root, - unpack (components) }, - "/") - if parameters then - url = url .. gh_encode_parameters (parameters) - end - return url - end - - local alright = [[HTTP/1.1 200 OK]] - - local gh_api_request = function (...) - local args = {...} - local nargs = #args - local final = args[nargs] - local request = { - url = "", - headers = { ["user-agent"] = user_agent }, - } - if type (final) == "table" then - args[nargs] = nil - request = gh_make_url (args, final) - else - request = gh_make_url (args) - end - - out ("Requesting <%s>.", request) - local response, code, headers, status - = https.request (request) - if status ~= alright then - out "Request failed!" - return false - end - return response - end - - local gh_api_checklimit = function (headers) - local rawlimit = gh_api_request "rate_limit" - local limitdata = lua_of_json (rawlimit) - if not limitdata and limitdata.rate then - out "Cannot parse API rate limit." - return false - end - limitdata = limitdata.rate - - local limit = tonumber (limitdata.limit) - local left = tonumber (limitdata.remaining) - local reset = tonumber (limitdata.reset) - - out ("%d of %d Github API requests left.", left, limit) - if left == 0 then - out ("Cannot make any more API requests.") - out ("Try again later at %s.", osdate ("%F %T", reset)) - end - return true - end - - local gh_tags = function () - out "Fetching tags from repository, please stand by." - local rawtags = gh_api_request ("repos", - luaotfload_repo, - "tags") - local taglist = lua_of_json (rawtags) - if not taglist or #taglist == 0 then - out "Cannot parse response." - return false - end - - local ntags = #taglist - out ("Repository contains %d tags.", ntags) - local _idx, latest = next (taglist) - out ("The most recent release is %s (revision %s).", - latest.name, - gh_shortrevision (latest.commit.sha)) - return latest - end - - local gh_compare = function (head, base) - if base == nil then - base = "HEAD" - end - out ("Fetching comparison between %s and %s, \z - please stand by.", - gh_shortrevision (head), - gh_shortrevision (base)) - local comparison = base .. "..." .. head - local rawstatus = gh_api_request ("repos", - luaotfload_repo, - "compare", - comparison) - local status = lua_of_json (rawstatus) - if not status then - out "Cannot parse response for status request." - return false - end - return status - end - - local gh_news = function (since) - local compared = gh_compare (since) - if not compared then - return false - end - local behind_by = compared.behind_by - local ahead_by = compared.ahead_by - local status = compared.status - out ("Comparison state: %s.", status) - if behind_by > 0 then - out ("Your Luaotfload is %d \z - revisions behind upstream.", - behind_by) - return behind_by - elseif status == "ahead" then - out "Since you are obviously from the future \z - I assume you already know the repository state." - else - out "Everything up to date. \z - Luaotfload is in sync with upstream." - end - return false - end - - local gh_catchup = function (current, latest) - local compared = gh_compare (latest, current) - local ahead_by = tonumber (compared.ahead_by) - if ahead_by > 0 then - local permalink_url = compared.permalink_url - out ("Your Luaotfload is %d revisions \z - behind the most recent release.", - ahead_by) - out ("To view the commit log, visit <%s>.", - permalink_url) - out ("You can grab an up to date tarball at <%s>.", - release_url) - return true - else - out "There weren't any new releases in the meantime." - out "Luaotfload is up to date." - end - return false - end - - check_upstream = function (current) - out "============= upstream repository =============" - local _succ = gh_api_checklimit () - local behind = gh_news (current) - if behind then - local latest = gh_tags () - local _behind = gh_catchup (current, - latest.commit.sha, - latest.name) - end - end - - --- trivium: diff since the first revision as pushed by Élie - --- in 2009 - --- local firstrevision = "c3ccb3ee07e0a67171c24960966ae974e0dd8e98" - --- check_upstream (firstrevision) - end - --- github api stuff end - - local print_envvar = function (var) - local val = osgetenv (var) - if val then - out ("%20s: %q", stringformat ("$%s", var), val) - return val - else - out ("%20s: <unset>", stringformat ("$%s", var)) - end - end - - local print_path = function (var) - local val = osgetenv (var) - if val then - local paths = filesplitpath (val) - if paths then - local npaths = #paths - if npaths == 1 then - out ("%20s: %q", stringformat ("$%s", var), val) - elseif npaths > 1 then - out ("%20s: <%d items>", stringformat ("$%s", var), npaths) - for i = 1, npaths do - out (" +: %q", paths[i]) - end - else - out ("%20s: <empty>") - end - end - else - out ("%20s: <unset>", stringformat ("$%s", var)) - end - end - - local print_kpsevar = function (var) - var = "$" .. var - local val = kpseexpand_var (var) - if val and val ~= var then - out ("%20s: %q", var, val) - return val - else - out ("%20s: <empty or unset>", var) - end - end - - local print_kpsepath = function (var) - var = "$" .. var - local val = kpseexpand_path (var) - if val and val ~= "" then - local paths = filesplitpath (val) - if paths then - local npaths = #paths - if npaths == 1 then - out ("%20s: %q", var, paths[1]) - elseif npaths > 1 then - out ("%20s: <%d items>", var, npaths) - for i = 1, npaths do - out (" +: %q", paths[i]) - end - else - out ("%20s: <empty>") - end - end - else - out ("%20s: <empty or unset>", var) - end - end - - --- this test first if a variable is set and then expands the - --- paths; this is necessitated by the fact that expand-path will - --- return the empty string both if the variable is unset and if - --- the directory does not exist - - local print_kpsepathvar = function (var) - local vvar = "$" .. var - local val = kpseexpand_var (vvar) - if val and val ~= vvar then - out ("%20s: %q", vvar, val) - print_kpsepath (var) - else - out ("%20s: <empty or unset>", var) - end - end - - local check_environment = function (errcnt) - out "============ environment settings =============" - out ("system: %s/%s", ostype, osname) - if ostype == "unix" and io.popen then - local chan = io.popen ("uname -a", "r") - if chan then - out ("info: %s", chan:read "*all") - chan:close () - end - end - - out "1) *shell environment*" - print_envvar "SHELL" - print_path "PATH" - print_path "OSFONTDIR" - print_envvar "USER" - if ostype == "windows" then - print_envvar "WINDIR" - print_envvar "CD" - print_path "TEMP" - elseif ostype == "unix" then - print_envvar "HOME" - print_envvar "PWD" - print_path "TMPDIR" - end - - out "2) *kpathsea*" - print_kpsepathvar "OPENTYPEFONTS" - print_kpsepathvar "TTFONTS" - - print_kpsepathvar "TEXMFCACHE" - print_kpsepathvar "TEXMFVAR" - - --- the expansion of these can be quite large; as they aren’t - --- usually essential to luaotfload, we won’t dump every single - --- path - print_kpsevar "LUAINPUTS" - print_kpsevar "CLUAINPUTS" - - return errcnt - end - - local anamneses = { - "environment", - "files", - "repository", - "permissions" - } - - actions.diagnose = function (job) - local errcnt = 0 - local asked = job.asked_diagnostics - if asked == "all" or asked == "thorough" then - asked = tabletohash (anamneses, true) - else - asked = lpegmatch(split_comma, asked) - asked = tabletohash (asked, true) - end - - if asked.environment == true then - errcnt = check_environment (errcnt) - asked.environment = nil - end - - if asked.files == true then - errcnt = verify_files (errcnt, status) - asked.files = nil - end - - if asked.permissions == true then - errcnt = check_permissions (errcnt) - asked.permissions = nil - end - - if asked.repository == true then - check_upstream (status.notes.revision) - asked.repository = nil - end - - local rest = next (asked) - if rest ~= nil then --> something unknown - out ("Unknown diagnostic %q.", rest) - end - if errcnt == 0 then --> success - out ("Everything appears to be in order, \z - you may sleep well.") - return true, false - end - out ( [[=============================================== - WARNING - =============================================== - - The diagnostic detected %d errors. - - This version of luaotfload may have been - tampered with. Modified versions of the - luaotfload source are unsupported. Read the log - carefully and get a clean version from CTAN or - github: - - × http://ctan.org/tex-archive/macros/luatex/generic/luaotfload - × https://github.com/lualatex/luaotfload/releases - - If you are uncertain as to how to proceed, then - ask on the lualatex mailing list: - - http://www.tug.org/mailman/listinfo/lualatex-dev - - =============================================== -]], errcnt) - return true, false - end +actions.diagnose = function (job) + --- diagnostics are loaded on demand + local diagnose = require "luaotfload-diagnostics.lua" + return diagnose (job) end --- stuff to be carried out prior to exit diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst index e81aa8d..2e95381 100644 --- a/luaotfload-tool.rst +++ b/luaotfload-tool.rst @@ -135,9 +135,13 @@ miscellaneous modifications; 2) ``permissions`` -> check permissions of cache directories and files; - 3) ``repository`` -> check the git repository + 3) ``environment`` -> print relevant + environment and kpse variables; + 4) ``repository`` -> check the git repository for new releases. + Procedures can be chained by concatenating with + commas, e.g. ``--diagnose=files,permissions``. Specify ``thorough`` to run all checks. FILES diff --git a/luaotfload.dtx b/luaotfload.dtx index 3892bd8..e42c14e 100644 --- a/luaotfload.dtx +++ b/luaotfload.dtx @@ -40,7 +40,7 @@ \input docstrip.tex \Msg{************************************************************************} \Msg{* Installation} -\Msg{* Package: luaotfload v2.3b OpenType layout system} +\Msg{* Package: luaotfload v2.4 OpenType layout system} \Msg{************************************************************************} \keepsilent @@ -111,7 +111,7 @@ and the derived files %<*driver> \NeedsTeXFormat{LaTeX2e} \ProvidesFile{luaotfload.drv}% - [2013/07/23 v2.3b OpenType layout system]% + [2013/07/28 v2.4 OpenType layout system]% \documentclass{ltxdoc} \usepackage{metalogo,multicol,mdwlist,fancyvrb,xspace} \usepackage[x11names]{xcolor} @@ -231,7 +231,7 @@ and the derived files % \GetFileInfo{luaotfload.drv} % % \title{The \identifier{luaotfload} package} -% \date{2013/07/23 v2.3b} +% \date{2013/07/28 v2.4} % \author{Elie Roux · Khaled Hosny · Philipp Gesang\\ % Home: \url{https://github.com/lualatex/luaotfload}\\ % Support: \email{lualatex-dev@tug.org}} @@ -1005,6 +1005,14 @@ and the derived files % In \TEX Live: \fileent{texmf-dist/doc/luatex/base/luatexref-t.pdf}. % } % +% For a much more detailed report about a given font try the \verb|-I| option +% instead (\verb|--inspect|). +% \begin{quote} +% \begin{verbatim} +% luaotfload-tool -I --find="Iwona Light Italic" +% \end{verbatim} +% \end{quote} +% % \verb|luaotfload-tool --help| will list the available command line % switches, including some not discussed in detail here. % For a full documentation of \identifier{luaotfload-tool} and its @@ -1383,6 +1391,7 @@ and the derived files % % \section{Troubleshooting} % +% \subsection {Database Generation} % If you encounter problems with some fonts, please first update to the latest % version of this package before reporting a bug, as % \identifier{luaotfload} is under active development and still a @@ -1392,18 +1401,42 @@ and the derived files % tracker for submitting bug reports, feature requests and the likes % requests and the likes. % -% Errors during database generation can be traced by increasing -% verbosity levels and redirecting log output to \fileent{stdout}: +% Bug reports are more likely to be addressed if they contain the output of % -% \begin{verbatim} -% luaotfload-tool -fuvvv --log=stdout -% \end{verbatim} +% \begin{quote} +% \begin{verbatim} +% luaotfload-tool --diagnose=environment,files,permissions +% \end{verbatim} +% \end{quote} +% +% \noindent Consult the man page for a description of these options. +% +% Errors during database generation can be traced by increasing the +% verbosity level and redirecting log output to \fileent{stdout}: % -% If this fails, the font last printed to the terminal is likely to be -% the culprit. +% \begin{quote} +% \begin{verbatim} +% luaotfload-tool -fuvvv --log=stdout +% \end{verbatim} +% \end{quote} +% +% \noindent or to a file in \fileent{/tmp}: +% +% \begin{quote} +% \begin{verbatim} +% luaotfload-tool -fuvvv --log=file +% \end{verbatim} +% \end{quote} +% +% \noindent In the latter case, invoke the \verb|tail(1)| utility on the file +% for live monitoring of the progress. +% +% If database generation fails, the font last printed to the terminal or log +% file is likely to be the culprit. % Please specify it when reporting a bug, and blacklist it for the time % being (see above, page \pageref{font-blacklist}). % +% \subsection {Font Features} % A common problem is the lack of features for some % \OpenType fonts even when specified. % This can be related to the fact that some fonts do not provide @@ -1414,10 +1447,22 @@ and the derived files % fix it. % For example with \verb|latn|: % -% \begin{verbatim} -% \font\test=file:MyFont.otf:script=latn;+liga; -% \end{verbatim} +% \begin{quote} +% \begin{verbatim} +% \font\test=file:MyFont.otf:script=latn;+liga; +% \end{verbatim} +% \end{quote} +% +% You can get a list of features that a font defines for scripts and languages +% by querying it in \fileent{luaotfload-tool}: % +% \begin{quote} +% \begin{verbatim} +% luaotfload-tool --find="Iwona" --inspect +% \end{verbatim} +% \end{quote} +% +% \subsection {\LUATEX Programming} % Another strategy that helps avoiding problems is to not access raw \LUATEX % internals directly. % Some of them, even though they are dangerous to access, have not been @@ -1426,14 +1471,39 @@ and the derived files % \luafunction{aux} namespace over direct manipulation of font objects. % For example, raw access to the \luafunction{font.fonts} table like: % -% \begin{verbatim} -% local somefont = font.fonts[2] -% \end{verbatim} +% \begin{quote} +% \begin{verbatim} +% local somefont = font.fonts[2] +% \end{verbatim} +% \end{quote} % -% can render already defined fonts unusable. +% \noindent can render already defined fonts unusable. % Instead, the function \luafunction{font.getfont()} should be used because % it has been replaced by a safe variant. % +% However, \luafunction{font.getfont()} only covers fonts handled by the font +% loader, e.~g. \identifier{OpenType} and \identifier{TrueType} fonts, but +% not \abbrev{tfm} or \abbrev{ofm}. +% Should you absolutely require access to all fonts known to \LUATEX, including +% the virtual and autogenerated ones, then you need to query both +% \luafunction{font.getfont()} and \luafunction{font.fonts}. +% In this case, best define you own accessor: +% +% \begin{quote} +% \begin{verbatim} +% local unsafe_getfont = function (id) +% local tfmdata = font.getfont (id) +% if not tfmdata then +% tfmdata = font.fonts[id] +% end +% return tfmdata +% end +% +% --- use like getfont() +% local somefont = unsafe_getfont (2) +% \end{verbatim} +% \end{quote} +% % \part{Implementation} % % \section{\fileent{luaotfload.lua}} @@ -1478,8 +1548,8 @@ config.luaotfload.index_file = config.luaotfload.index_file or "luaot luaotfload.module = { name = "luaotfload", - version = 2.3002, - date = "2013/07/23", + version = 2.40001, + date = "2013/07/28", description = "OpenType layout system.", author = "Elie Roux & Hans Hagen", copyright = "Elie Roux", @@ -2070,7 +2140,7 @@ luaotfload.aux.start_rewrite_fontname () --- to be migrated to fontspec \else \NeedsTeXFormat{LaTeX2e} \ProvidesPackage{luaotfload}% - [2013/07/23 v2.3b OpenType layout system] + [2013/07/28 v2.4 OpenType layout system] \RequirePackage{luatexbase} \fi \ifnum\luatexversion<76 @@ -39,6 +39,7 @@ local names = { "luaotfload-characters.lua", "luaotfload-colors.lua", "luaotfload-database.lua", + "luaotfload-diagnostics.lua", "luaotfload-extralibs.lua", "luaotfload-features.lua", "luaotfload-fonts-cbk.lua", |