[mkimport] directly package fontloader

Fix issue #305

No round-trip through mtx-package is required anymore. This assumes
“util-mrg.lua” which comes with Context is present in the path.

As as side-effect we can inject custom annotations into the file that
reduce possible confusion for users.

1 files changed, 193 insertions, 54 deletions

diff --git a/scripts/mkimport b/scripts/mkimport
index abe1608..9d443ea 100755
--- a/scripts/mkimport
+++ b/scripts/mkimport
@@ -30,6 +30,11 @@ local md5 = require "md5"
 local os  = require "os"
 
 require "lualibs"
+require "util-mrg"
+
+assert (utilities and utilities.merger and utilities.merger.compact
+        and type (utilities.merger.compact) == "function",
+        "Whoa, util-mrg.lua is not accessible! How do you expect this to work‽")
 
 local filedirname      = file.dirname
 local fileiswritable   = file.is_writable
@@ -91,7 +96,6 @@ local prefixes = {
 --doc]]--
 
 local loader_merge_name  = "luaotfload-package.lua"
-local loader_output_name = "luaotfload-package-merged.lua"
 local loader_target_name = "fontloader-%s.lua"
 local loader_orig_dir    = "/src/fontloader/"
 local loader_target_dir  = "/build/"
@@ -245,6 +249,107 @@ local imports = {
   }, --[[ [context] ]]
 } --[[ [imports] ]]
 
+local package = {
+
+  optional = { --- components not included in the default package
+
+--- The original initialization sequence by Hans Hagen, see the file
+--- luatex-fonts.lua for details:
+---
+---   [01] l-lua.lua
+---   [02] l-lpeg.lua
+---   [03] l-function.lua
+---   [04] l-string.lua
+---   [05] l-table.lua
+---   [06] l-io.lua
+---   [07] l-file.lua
+---   [08] l-boolean.lua
+---   [09] l-math.lua
+---   [10] util-str.lua
+---   [11] luatex-basics-gen.lua
+---   [12] data-con.lua
+---   [13] luatex-basics-nod.lua
+---   [14] font-ini.lua
+---   [15] font-con.lua
+---   [16] luatex-fonts-enc.lua
+---   [17] font-cid.lua
+---   [18] font-map.lua
+---   [19] luatex-fonts-syn.lua
+---   [20] font-tfm.lua
+---   [21] font-afm.lua
+---   [22] font-afk.lua
+---   [23] luatex-fonts-tfm.lua
+---   [24] font-oti.lua
+---   [25] font-otf.lua
+---   [26] font-otb.lua
+---   [27] luatex-fonts-inj.lua
+---   [28] luatex-fonts-ota.lua
+---   [29] luatex-fonts-otn.lua
+---   [30] font-otp.lua
+---   [31] luatex-fonts-lua.lua
+---   [32] font-def.lua
+---   [33] luatex-fonts-def.lua
+---   [34] luatex-fonts-ext.lua
+---   [35] luatex-fonts-cbk.lua
+---
+--- Of these, nos. 01--10 are provided by the Lualibs. Keeping them
+--- around in the Luaotfload fontloader is therefore unnecessary.
+--- Packaging needs to account for this difference.
+
+    "l-lua",
+    "l-lpeg",
+    "l-function",
+    "l-string",
+    "l-table",
+    "l-io",
+    "l-file",
+    "l-boolean",
+    "l-math",
+    "util-str",
+
+--- Another file containing auxiliary definitions must be present
+--- prior to initialization of the configuration.
+
+    "luatex-basics-gen",
+
+  }, --[[ [package.optional] ]]
+
+--- The files below constitute the “fontloader proper”. Some of the
+--- functionality like file resolvers is overloaded later by
+--- Luaotfload. Consequently, the resulting package is pretty
+--- bare-bones and not usable independently.
+
+  required = {
+
+    "data-con",
+    "basics-nod",
+    "font-ini",
+    "font-con",
+    "fonts-enc",
+    "font-cid",
+    "font-map",
+    "fonts-syn",
+    "font-tfm",
+    "font-afm",
+    "font-afk",
+    "fonts-tfm",
+    "font-oti",
+    "font-otf",
+    "font-otb",
+    "fonts-inj",
+    "fonts-ota",
+    "fonts-otn",
+    "font-otp",
+    "fonts-lua",
+    "font-def",
+    "fonts-def",
+    "fonts-ext",
+    "fonts-cbk",
+
+  }, --[[ [package.required] ]]
+
+} --[[ [package] ]]
+
 local hash_file = function (fname)
   if not lfsisfile (fname) then
     die ("cannot find %s.", fname)
@@ -663,10 +768,9 @@ local build_paths = function (argv)
 
   local orig_dir    = lfscurrentdir ()
   local base_dir    = orig_dir .. loader_orig_dir
-  local target_name = orig_dir .. loader_target_dir
-                   .. stringformat (loader_target_name, os.date ("%F"))
+  local loader_name = stringformat (loader_target_name, os.date ("%F"))
+  local target_name = orig_dir .. loader_target_dir .. loader_name
   local merge_name  = base_dir .. loader_merge_name
-  local output_name = base_dir .. loader_output_name
 
   if #argv >= 2 then
     local fname = argv[2]
@@ -677,7 +781,6 @@ local build_paths = function (argv)
     end
     base_dir    = dir
     merge_name  = fname
-    output_name = dir .. loader_output_name
   end
 
   if #argv == 3 then
@@ -696,28 +799,93 @@ local build_paths = function (argv)
     base_dir    = base_dir,
     merge_name  = merge_name,
     target_name = target_name,
-    output_name = output_name,
+    loader_name = loader_name,
   }
   return ret
 end
 
---[[doc--
+local luaotfload_header = [==[
+--[[info-----------------------------------------------------------------------
+  Luaotfload fontloader package
+  build %s by %s@%s
+-------------------------------------------------------------------------------
 
-  Packaging works as follows:
+  © %s Hans Hagen / Pragma ADE
 
-      * Files are looked up the usual way, allowing us to override the
-        distribution-supplied scripts with our own alternatives in the
-        local path.
+  The code in this file is provided under the GPL v2.0 license. See the
+  file COPYING in the Luaotfload repository for details.
 
-      * The merged package is written to the same directory as the
-        packaging script (not ``$PWD``).
+  Report bugs to github.com/lualatex/luaotfload
 
-  There is some room for improvements: Instead of reading a file with
-  fixed content from disk, the merge script could be composed
-  on-the-fly from a list of files and then written to memory (not sure
-  though if we can access shm_open or memfd and the likes from Lua).
+  This file has been assembled from components taken from Context. See
+  the Luaotfload documentation for details:
 
---doc]]--
+      $ texdoc luaotfload
+      $ man 1 luaotfload-tool
+      $ man 5 luaotfload.conf
+
+  Included files:
+
+%s
+
+--info]]-----------------------------------------------------------------------
+
+]==]
+
+local make_header = function (files)
+  local filelist = { }
+  for i = 1, #files do
+    local f = files[i]
+    local _void, ourname = derive_ourname (f, kind_merged)
+    filelist [#filelist + 1] = stringformat ("    · %s", ourname)
+  end
+  return stringformat (luaotfload_header,
+                       os.date "%F %T",
+                       os.getenv "USER" or "anon",
+                       os.getenv "HOSTNAME" or "void",
+                       os.date "%Y",
+                       table.concat (filelist, "\n"))
+end
+
+
+local scope_start         = "\ndo  --- [luaotfload, %s scope for “%s”] ---\n\n"
+local scope_stop          = "\nend --- [luaotfload, %s scope for “%s”] ---\n\n"
+local luaotfload_modeline = "\n--- vim:ft=lua:sw=2:ts=8:et:tw=79\n"
+
+local assemble_fontloader = function (tgt, dst)
+  status ("packaging fontloader as “%s”.", emphasis (tgt))
+  local required  = package.required
+  local compact   = utilities.merger.compact
+  local sequence  = { [1] = make_header (required) }
+  for i = 1, #required do
+    local cur = required [i]
+    local subdir, ourname = derive_ourname (cur, kind_merged)
+    local ourpath = file.join (fontloader_subdir, subdir, ourname)
+    local data = io.loaddata (ourpath)
+    if not data then
+      bad ("file “%s” cannot be loaded from “%s”", ourname, ourpath)
+      return false
+    end
+    sequence[#sequence + 1] = stringformat (scope_start, tgt, cur)
+    sequence[#sequence + 1] = compact (data)
+    sequence[#sequence + 1] = stringformat (scope_stop , tgt, cur)
+  end
+  sequence[#sequence + 1] = luaotfload_modeline
+  local raw = table.concat (sequence)
+  local _void, chunk, err = pcall (loadstring, raw)
+  print(_void, chunk)
+  if chunk == nil then
+    bad ("packaging result not well-formed")
+    bad ("error message: “%s”", err)
+    bad ("dumping to fontloader-junk.lua")
+    io.savedata ("fontloader-junk.lua", raw)
+    return false
+  end
+  io.savedata (dst, raw)
+  status ("amalgamated %d files, written to “%s”.",
+          #required, dst)
+  return dst
+end
 
 local package = function (argv)
   local t0    = osgettimeofday ()
@@ -725,21 +893,19 @@ local package = function (argv)
 
   status ("assuming fontloader source in      %s", paths.base_dir)
   status ("reading merge instructions from    %s", paths.merge_name)
-  status ("mtx-package result at              %s", paths.output_name)
   status ("writing output to                  %s", paths.target_name)
 
   --- check preconditions
 
   if not lfsisdir (paths.base_dir)          then die ("directory %s does not exist", emphasis (paths.base_dir   )) end
   if not lfsisfile (paths.merge_name)       then die ("missing merge file at %s",    emphasis (paths.merge_name )) end
-  if not fileiswritable (paths.output_name) then die ("cannot write to %s",          emphasis (paths.output_name)) end
   if not fileiswritable (paths.target_name) then die ("cannot write to %s",          emphasis (paths.target_name)) end
 ---- not lfschdir (paths.base_dir)          then die ("failed to cd into %s",        emphasis (paths.base_dir   )) end
 
-  if lfsisfile (paths.output_name) then
+  if lfsisfile (paths.target_name) then
     status ("output file already exists at “%s”, unlinking",
-            paths.output_name)
-    local ret, err = os.remove (paths.output_name)
+            paths.target_name)
+    local ret, err = os.remove (paths.target_name)
     if ret == nil then
       if not lfschdir (paths.orig_dir) then
         status ("warning: failed to cd retour into %s",
@@ -748,18 +914,11 @@ local package = function (argv)
       die ("failed to remove existing merge package")
     end
   end
-    --die ("missing merge file at %s",    emphasis (paths.merge_name )) end
-
   --- perform merge
 
-  local cmd = { "mtxrun", "--script", "package", "--merge", paths.merge_name }
-  local shl = tableconcat (cmd, " ")
-
-  status ("invoking %s as “%s”", emphasis "mtx-package", shl)
+  local ret = assemble_fontloader (paths.loader_name, paths.target_name)
 
-  local fh = iopopen (shl, "r")
-
-  if not fh then
+  if not ret then
     if not lfschdir (paths.orig_dir) then
       status ("warning: failed to cd retour into %s",
               emphasis (paths.orig_dir))
@@ -767,15 +926,6 @@ local package = function (argv)
     die ("merge failed; failed to invoke mtxrun")
   end
 
-  local junk = fh.read (fh, "*all")
-  if not junk then
-    status ("warning: received no output from mtxrun; this is strange")
-  end
-
-  fh.close (fh)
-
-  if debug then print (junk) end
-
   --- clean up
 
   if not lfschdir (paths.orig_dir) then
@@ -785,24 +935,13 @@ local package = function (argv)
 
   --- check postconditions
 
-  if not lfsisfile (paths.output_name) then
-    die ("merge failed; package not found at " .. paths.output_name)
+  if not lfsisfile (ret) then
+    die ("merge failed; package not found at “%s”", paths.output_name)
   end
 
   --- at this point we know that mtxrun was invoked correctly and the
   --- result file has been created
 
-  if lfsisfile (paths.target_name) then
-    status ("target file %s exists, overwriting", emphasis (paths.target_name))
-  end
-
-  local res, err = osrename (paths.output_name, paths.target_name)
-
-  if res == nil then
-    die ("merge failed; failed to move package from %s to %s",
-         paths.output_name, paths.target_name)
-  end
-
   status ("merge complete; operation finished in %.0f ms",
           (osgettimeofday() - t0) * 1000)
   status ("a fresh fontloader at %s is ready to roll", paths.target_name)