Merge pull request #306 from phi-gamma/master

improved packaging

6 files changed, 210 insertions, 188 deletions

diff --git a/Makefile b/Makefile
index f23c6b8..a131f3a 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,6 @@ GRAPH		= filegraph
 DOCSRC		= $(addprefix $(DOCSRCDIR)/$(NAME), -main.tex -latex.tex)
 LOADERSRC	= $(wildcard $(FONTLOADERSRCDIR)/*.lua)
 LOADERSRC	+= $(wildcard $(FONTLOADERSRCDIR)/*.tex)
-LOADERSRC	+= $(PACKAGEDIR)/luaotfload-package.lua
 GRAPHSRC	= $(DOCSRCDIR)/$(GRAPH).dot
 MANSRC		= $(DOCSRCDIR)/$(TOOLNAME).rst $(DOCSRCDIR)/$(CONFNAME).rst
 
@@ -99,8 +98,7 @@ DO_GLYPHS	= $(LUA) $(GLYPHSCRIPT) > /dev/null
 DO_CHARS	= $(LUA) $(CHARSCRIPT)  > /dev/null
 DO_STATUS	= $(LUA) $(STATUSSCRIPT) --fontloader=$(LOADER) >/dev/null
 DO_IMPORT	= $(LUA) $(IMPORTSCRIPT) import >/dev/null
-DO_PACKAGE	= $(LUA) $(IMPORTSCRIPT) package \
-		  $(PACKAGEDIR)/luaotfload-package.lua $(LOADER) >/dev/null
+DO_PACKAGE	= $(LUA) $(IMPORTSCRIPT) package $(LOADER) >/dev/null
 
 define check-lua-files
 @echo validating syntax
@@ -200,7 +198,7 @@ $(TDS_ZIP): $(DOCS) $(ALL_STATUS) check
 
 sign: $(CTAN_ZIPSIG)
 
-.PHONY: install manifest clean mrproper show showtargets check import news
+.PHONY: package install manifest clean mrproper show showtargets check import news
 
 install:
 	@echo "	××××××××××××××××××××××××××××××××"
diff --git a/doc/filegraph.dot b/doc/filegraph.dot
index 9773ebd..de60823 100644
--- a/doc/filegraph.dot
+++ b/doc/filegraph.dot
@@ -54,7 +54,7 @@ strict digraph luaotfload_files { //looks weird with circo ...
     fontdbutil -> fontdbutil_diagnostics [label="--diagnose"]
     fontdbutil -> status [label="version information"]
 
-    luaotfload_package -> fontloader [label="merges", style=dashed]
+    mkimport -> fontloader [label="merges", style=dashed]
 
     merged_luatex_fonts -> characters [label="luaotfload-auxiliary.lua",
                                        ltail=cluster_merged]
@@ -108,14 +108,6 @@ strict digraph luaotfload_files { //looks weird with circo ...
                        style  = "filled,rounded",
                        penwidth=2]
 
-    luaotfload_package [label  = "luaotfload-package.lua",
-                        shape  = rect,
-                        width  = "3.2cm",
-                        height = "1.2cm",
-                        color  = "#01012222",
-                        style  = "filled,rounded",
-                        penwidth=2]
-
     fontdbutil_diagnostics [label  = "luaotfload-diagnostics.lua",
                             shape  = rect,
                             width  = "3.2cm",
diff --git a/doc/luaotfload-main.tex b/doc/luaotfload-main.tex
index 967f5a2..ca1bc96 100644
--- a/doc/luaotfload-main.tex
+++ b/doc/luaotfload-main.tex
@@ -32,7 +32,7 @@
 \beginfrontmatter
 
   \setdocumenttitle  {The \identifier{luaotfload} package}
-  \setdocumentdate   {2015/12/09 v2.6}
+  \setdocumentdate   {2015/12/12 v2.6}
   \setdocumentauthor {Elie Roux · Khaled Hosny · Philipp Gesang\\
                       Home:    \hyperlink {https://github.com/lualatex/luaotfload}\\
                       Support: \email     {lualatex-dev@tug.org}}
@@ -1076,8 +1076,8 @@ format.
 It integrates the font loader, written entirely in \LUA, as distributed
 in the \identifier{\LUATEX-Fonts} package.
 %
-The original \LUA source files have been combined using the
-\fileent{mtx-package} script into a single, self-contained blob. In
+The original \LUA source files have been combined using the \CONTEXT
+packaging library into a single, self-contained blob. In
 this form the font loader depends only on the \identifier{lualibs}
 package and requires only minor adaptions to integrate into
 \identifier{luaotfload}.
@@ -1106,17 +1106,14 @@ See figure \ref{file-graph} on page \pageref{file-graph} for a
 graphical representation of the dependencies.
 %
 \label{package}%
-Through the script \fileent{luaotfload-package.lua} the \CONTEXT
-utility \fileent{mtx-package} is invoked to create the
-\identifier{luaotfload} fontloader as a merged (amalgamated) source
-file.\footnote{%
-  \fileent{mtx-package}, a \LUA source code merging tool developed by
-  Hans Hagen, is
-  \hyperlink [part of \CONTEXT]{https://bitbucket.org/phg/context-mirror/src/726a663be481042003566d4614266b940b5a0c91/scripts/context/lua/mtx-package.lua?at=beta}
-  and requires \fileent{mtxrun}.
-  Run
-  \inlinecode {mtxrun --script package --help}
-  to display further information.
+Through the script \fileent{mkimport} a \CONTEXT library
+is invoked to create the \identifier{luaotfload} fontloader as a merged
+(amalgamated) source file.\footnote{%
+  In \CONTEXT, this facility can be accessed by means of a
+  \hyperlink[script]{https://bitbucket.org/phg/context-mirror/src/beta/scripts/context/lua/mtx-package.lua?at=beta}
+  which is integrated into \fileent{mtxrun} as a subcommand.
+  Run \inlinecode {mtxrun --script package --help} to display further
+  information.
   For the actual merging code see the file
   \fileent{util-mrg.lua} that is part of \CONTEXT.
 }
@@ -1334,7 +1331,7 @@ The subcommand for importing takes the prefix of the desired \CONTEXT
 Whereas the command for packaging requires a path to the
 \emphasis{package description file} and the output name to be passed.
 \beginlisting
-./scripts/mkimport package ./src/fontloader/luaotfload-package.lua fontloader-custom.lua
+./scripts/mkimport package fontloader-custom.lua
 \endlisting
 
 From the toplevel makefile, the targets \inlinecode{import} and
diff --git a/scripts/mkimport b/scripts/mkimport
index abe1608..b3c71fe 100755
--- a/scripts/mkimport
+++ b/scripts/mkimport
@@ -30,10 +30,16 @@ local md5 = require "md5"
 local os  = require "os"
 
 require "lualibs"
+require "util-mrg"
+
+assert (utilities and utilities.merger and utilities.merger.compact
+        and type (utilities.merger.compact) == "function",
+        "Whoa, util-mrg.lua is not accessible! How do you expect this to work‽")
 
 local filedirname      = file.dirname
 local fileiswritable   = file.is_writable
 local ioloaddata       = io.loaddata
+local iosavedata       = io.savedata
 local iopopen          = io.popen
 local iowrite          = io.write
 local lfschdir         = lfs.chdir
@@ -90,8 +96,6 @@ local prefixes = {
 
 --doc]]--
 
-local loader_merge_name  = "luaotfload-package.lua"
-local loader_output_name = "luaotfload-package-merged.lua"
 local loader_target_name = "fontloader-%s.lua"
 local loader_orig_dir    = "/src/fontloader/"
 local loader_target_dir  = "/build/"
@@ -245,6 +249,107 @@ local imports = {
   }, --[[ [context] ]]
 } --[[ [imports] ]]
 
+local package = {
+
+  optional = { --- components not included in the default package
+
+--- The original initialization sequence by Hans Hagen, see the file
+--- luatex-fonts.lua for details:
+---
+---   [01] l-lua.lua
+---   [02] l-lpeg.lua
+---   [03] l-function.lua
+---   [04] l-string.lua
+---   [05] l-table.lua
+---   [06] l-io.lua
+---   [07] l-file.lua
+---   [08] l-boolean.lua
+---   [09] l-math.lua
+---   [10] util-str.lua
+---   [11] luatex-basics-gen.lua
+---   [12] data-con.lua
+---   [13] luatex-basics-nod.lua
+---   [14] font-ini.lua
+---   [15] font-con.lua
+---   [16] luatex-fonts-enc.lua
+---   [17] font-cid.lua
+---   [18] font-map.lua
+---   [19] luatex-fonts-syn.lua
+---   [20] font-tfm.lua
+---   [21] font-afm.lua
+---   [22] font-afk.lua
+---   [23] luatex-fonts-tfm.lua
+---   [24] font-oti.lua
+---   [25] font-otf.lua
+---   [26] font-otb.lua
+---   [27] luatex-fonts-inj.lua
+---   [28] luatex-fonts-ota.lua
+---   [29] luatex-fonts-otn.lua
+---   [30] font-otp.lua
+---   [31] luatex-fonts-lua.lua
+---   [32] font-def.lua
+---   [33] luatex-fonts-def.lua
+---   [34] luatex-fonts-ext.lua
+---   [35] luatex-fonts-cbk.lua
+---
+--- Of these, nos. 01--10 are provided by the Lualibs. Keeping them
+--- around in the Luaotfload fontloader is therefore unnecessary.
+--- Packaging needs to account for this difference.
+
+    "l-lua",
+    "l-lpeg",
+    "l-function",
+    "l-string",
+    "l-table",
+    "l-io",
+    "l-file",
+    "l-boolean",
+    "l-math",
+    "util-str",
+
+--- Another file containing auxiliary definitions must be present
+--- prior to initialization of the configuration.
+
+    "luatex-basics-gen",
+
+  }, --[[ [package.optional] ]]
+
+--- The files below constitute the “fontloader proper”. Some of the
+--- functionality like file resolvers is overloaded later by
+--- Luaotfload. Consequently, the resulting package is pretty
+--- bare-bones and not usable independently.
+
+  required = {
+
+    "data-con",
+    "basics-nod",
+    "font-ini",
+    "font-con",
+    "fonts-enc",
+    "font-cid",
+    "font-map",
+    "fonts-syn",
+    "font-tfm",
+    "font-afm",
+    "font-afk",
+    "fonts-tfm",
+    "font-oti",
+    "font-otf",
+    "font-otb",
+    "fonts-inj",
+    "fonts-ota",
+    "fonts-otn",
+    "font-otp",
+    "fonts-lua",
+    "font-def",
+    "fonts-def",
+    "fonts-ext",
+    "fonts-cbk",
+
+  }, --[[ [package.required] ]]
+
+} --[[ [package] ]]
+
 local hash_file = function (fname)
   if not lfsisfile (fname) then
     die ("cannot find %s.", fname)
@@ -663,10 +768,8 @@ local build_paths = function (argv)
 
   local orig_dir    = lfscurrentdir ()
   local base_dir    = orig_dir .. loader_orig_dir
-  local target_name = orig_dir .. loader_target_dir
-                   .. stringformat (loader_target_name, os.date ("%F"))
-  local merge_name  = base_dir .. loader_merge_name
-  local output_name = base_dir .. loader_output_name
+  local loader_name = stringformat (loader_target_name, os.date ("%F"))
+  local target_name = orig_dir .. loader_target_dir .. loader_name
 
   if #argv >= 2 then
     local fname = argv[2]
@@ -676,8 +779,7 @@ local build_paths = function (argv)
            argv[2])
     end
     base_dir    = dir
-    merge_name  = fname
-    output_name = dir .. loader_output_name
+    target_name  = fname
   end
 
   if #argv == 3 then
@@ -696,28 +798,93 @@ local build_paths = function (argv)
     base_dir    = base_dir,
     merge_name  = merge_name,
     target_name = target_name,
-    output_name = output_name,
+    loader_name = loader_name,
   }
   return ret
 end
 
---[[doc--
+local luaotfload_header = [==[
+--[[info-----------------------------------------------------------------------
+  Luaotfload fontloader package
+  build %s by %s@%s
+-------------------------------------------------------------------------------
 
-  Packaging works as follows:
+  © %s PRAGMA ADE / ConTeXt Development Team
 
-      * Files are looked up the usual way, allowing us to override the
-        distribution-supplied scripts with our own alternatives in the
-        local path.
+  The code in this file is provided under the GPL v2.0 license. See the
+  file COPYING in the Luaotfload repository for details.
 
-      * The merged package is written to the same directory as the
-        packaging script (not ``$PWD``).
+  Report bugs to github.com/lualatex/luaotfload
 
-  There is some room for improvements: Instead of reading a file with
-  fixed content from disk, the merge script could be composed
-  on-the-fly from a list of files and then written to memory (not sure
-  though if we can access shm_open or memfd and the likes from Lua).
+  This file has been assembled from components taken from Context. See
+  the Luaotfload documentation for details:
 
---doc]]--
+      $ texdoc luaotfload
+      $ man 1 luaotfload-tool
+      $ man 5 luaotfload.conf
+
+  Included files:
+
+%s
+
+--info]]-----------------------------------------------------------------------
+
+]==]
+
+local make_header = function (files)
+  local filelist = { }
+  for i = 1, #files do
+    local f = files[i]
+    local _void, ourname = derive_ourname (f, kind_merged)
+    filelist [#filelist + 1] = stringformat ("    · %s", ourname)
+  end
+  return stringformat (luaotfload_header,
+                       os.date "%F %T",
+                       os.getenv "USER" or "anon",
+                       os.getenv "HOSTNAME" or "void",
+                       os.date "%Y",
+                       table.concat (filelist, "\n"))
+end
+
+
+local scope_start         = "\ndo  --- [luaotfload, %s scope for “%s” %s] ---\n\n"
+local scope_stop          = "\nend --- [luaotfload, %s scope for “%s”] ---\n\n"
+local luaotfload_modeline = "\n--- vim:ft=lua:sw=2:ts=8:et:tw=79\n"
+
+local assemble_fontloader = function (tgt, dst)
+  status ("packaging fontloader as “%s”.", emphasis (tgt))
+  local required  = package.required
+  local compact   = utilities.merger.compact
+  local sequence  = { [1] = make_header (required) }
+  for i = 1, #required do
+    local cur = required [i]
+    local subdir, ourname = derive_ourname (cur, kind_merged)
+    local ourpath = file.join (fontloader_subdir, subdir, ourname)
+    local data = ioloaddata (ourpath)
+    if not data then
+      bad ("file “%s” cannot be loaded from “%s”", ourname, ourpath)
+      return false
+    end
+    sequence[#sequence + 1] = stringformat (scope_start, tgt, cur, md5sumhexa (data))
+    sequence[#sequence + 1] = compact (data)
+    sequence[#sequence + 1] = stringformat (scope_stop , tgt, cur)
+  end
+  sequence[#sequence + 1] = luaotfload_modeline
+  local raw = table.concat (sequence)
+  local _void, chunk, err = pcall (loadstring, raw)
+  print(_void, chunk)
+  if chunk == nil then
+    bad ("packaging result not well-formed")
+    bad ("error message: “%s”", err)
+    bad ("dumping to fontloader-junk.lua")
+    iosavedata ("fontloader-junk.lua", raw)
+    return false
+  end
+  iosavedata (dst, raw)
+  status ("amalgamated %d files, written to “%s”.",
+          #required, dst)
+  return dst
+end
 
 local package = function (argv)
   local t0    = osgettimeofday ()
@@ -725,21 +892,18 @@ local package = function (argv)
 
   status ("assuming fontloader source in      %s", paths.base_dir)
   status ("reading merge instructions from    %s", paths.merge_name)
-  status ("mtx-package result at              %s", paths.output_name)
   status ("writing output to                  %s", paths.target_name)
 
   --- check preconditions
 
   if not lfsisdir (paths.base_dir)          then die ("directory %s does not exist", emphasis (paths.base_dir   )) end
-  if not lfsisfile (paths.merge_name)       then die ("missing merge file at %s",    emphasis (paths.merge_name )) end
-  if not fileiswritable (paths.output_name) then die ("cannot write to %s",          emphasis (paths.output_name)) end
   if not fileiswritable (paths.target_name) then die ("cannot write to %s",          emphasis (paths.target_name)) end
 ---- not lfschdir (paths.base_dir)          then die ("failed to cd into %s",        emphasis (paths.base_dir   )) end
 
-  if lfsisfile (paths.output_name) then
+  if lfsisfile (paths.target_name) then
     status ("output file already exists at “%s”, unlinking",
-            paths.output_name)
-    local ret, err = os.remove (paths.output_name)
+            paths.target_name)
+    local ret, err = os.remove (paths.target_name)
     if ret == nil then
       if not lfschdir (paths.orig_dir) then
         status ("warning: failed to cd retour into %s",
@@ -748,18 +912,11 @@ local package = function (argv)
       die ("failed to remove existing merge package")
     end
   end
-    --die ("missing merge file at %s",    emphasis (paths.merge_name )) end
-
   --- perform merge
 
-  local cmd = { "mtxrun", "--script", "package", "--merge", paths.merge_name }
-  local shl = tableconcat (cmd, " ")
-
-  status ("invoking %s as “%s”", emphasis "mtx-package", shl)
+  local ret = assemble_fontloader (paths.loader_name, paths.target_name)
 
-  local fh = iopopen (shl, "r")
-
-  if not fh then
+  if not ret then
     if not lfschdir (paths.orig_dir) then
       status ("warning: failed to cd retour into %s",
               emphasis (paths.orig_dir))
@@ -767,15 +924,6 @@ local package = function (argv)
     die ("merge failed; failed to invoke mtxrun")
   end
 
-  local junk = fh.read (fh, "*all")
-  if not junk then
-    status ("warning: received no output from mtxrun; this is strange")
-  end
-
-  fh.close (fh)
-
-  if debug then print (junk) end
-
   --- clean up
 
   if not lfschdir (paths.orig_dir) then
@@ -785,24 +933,13 @@ local package = function (argv)
 
   --- check postconditions
 
-  if not lfsisfile (paths.output_name) then
-    die ("merge failed; package not found at " .. paths.output_name)
+  if not lfsisfile (ret) then
+    die ("merge failed; package not found at “%s”", paths.output_name)
   end
 
   --- at this point we know that mtxrun was invoked correctly and the
   --- result file has been created
 
-  if lfsisfile (paths.target_name) then
-    status ("target file %s exists, overwriting", emphasis (paths.target_name))
-  end
-
-  local res, err = osrename (paths.output_name, paths.target_name)
-
-  if res == nil then
-    die ("merge failed; failed to move package from %s to %s",
-         paths.output_name, paths.target_name)
-  end
-
   status ("merge complete; operation finished in %.0f ms",
           (osgettimeofday() - t0) * 1000)
   status ("a fresh fontloader at %s is ready to roll", paths.target_name)
diff --git a/scripts/mkstatus b/scripts/mkstatus
index f27d571..21d5e71 100755
--- a/scripts/mkstatus
+++ b/scripts/mkstatus
@@ -71,9 +71,6 @@ local names = {
   { scriptdir,    "mkstatus",                       },
   { scriptdir,    "mktests",                        },
 
-  --- merge script
-  { loaderdir,    "luaotfload-package.lua",         },
-
   --- fontloader runtimes
   { rtdir,        "fontloader-basics-gen.lua",      },
   { rtdir,        "fontloader-reference.lua",       },
diff --git a/src/fontloader/luaotfload-package.lua b/src/fontloader/luaotfload-package.lua
deleted file mode 100644
index 524dbcb..0000000
--- a/src/fontloader/luaotfload-package.lua
+++ /dev/null
@@ -1,99 +0,0 @@
---
------------------------------------------------------------------------
---         FILE:  luaotfload-package.lua
---  DESCRIPTION:  Luatex fontloader packaging
--- REQUIREMENTS:  luatex, mtx-package
---       AUTHOR:  Philipp Gesang
---      LICENSE:  GNU GPL v2.0
------------------------------------------------------------------------
---
-
---- The original initialization sequence by Hans Hagen, see the file
---- luatex-fonts.lua for details:
----
----   [01] l-lua.lua
----   [02] l-lpeg.lua
----   [03] l-function.lua
----   [04] l-string.lua
----   [05] l-table.lua
----   [06] l-io.lua
----   [07] l-file.lua
----   [08] l-boolean.lua
----   [09] l-math.lua
----   [10] util-str.lua
----   [11] luatex-basics-gen.lua
----   [12] data-con.lua
----   [13] luatex-basics-nod.lua
----   [14] font-ini.lua
----   [15] font-con.lua
----   [16] luatex-fonts-enc.lua
----   [17] font-cid.lua
----   [18] font-map.lua
----   [19] luatex-fonts-syn.lua
----   [20] font-tfm.lua
----   [21] font-afm.lua
----   [22] font-afk.lua
----   [23] luatex-fonts-tfm.lua
----   [24] font-oti.lua
----   [25] font-otf.lua
----   [26] font-otb.lua
----   [27] luatex-fonts-inj.lua
----   [28] luatex-fonts-ota.lua
----   [29] luatex-fonts-otn.lua
----   [30] font-otp.lua
----   [31] luatex-fonts-lua.lua
----   [32] font-def.lua
----   [33] luatex-fonts-def.lua
----   [34] luatex-fonts-ext.lua
----   [35] luatex-fonts-cbk.lua
----
---- Of these, nos. 01--10 are provided by the Lualibs. Keeping them
---- around in the Luaotfload fontloader is therefore unnecessary.
---- Packaging needs to account for this difference.
-
-loadmodule "l-lua.lua"
-loadmodule "l-lpeg.lua"
-loadmodule "l-function.lua"
-loadmodule "l-string.lua"
-loadmodule "l-table.lua"
-loadmodule "l-io.lua"
-loadmodule "l-file.lua"
-loadmodule "l-boolean.lua"
-loadmodule "l-math.lua"
-loadmodule "util-str.lua"
-
---- Another file containing auxiliary definitions must be present
---- prior to initialization of the configuration.
-
-loadmodule "luatex-basics-gen.lua"
-
---- The files below constitute the “fontloader proper”. Some of the
---- functionality like file resolvers is overloaded later by
---- Luaotfload. Consequently, the resulting package is pretty
---- bare-bones and not usable independently.
-
-loadmodule("data-con.lua")
-loadmodule("luatex-basics-nod.lua")
-loadmodule("font-ini.lua")
-loadmodule("font-con.lua")
-loadmodule("luatex-fonts-enc.lua")
-loadmodule("font-cid.lua")
-loadmodule("font-map.lua")
-loadmodule("luatex-fonts-syn.lua")
-loadmodule("font-tfm.lua")
-loadmodule("font-afm.lua")
-loadmodule("font-afk.lua")
-loadmodule("luatex-fonts-tfm.lua")
-loadmodule("font-oti.lua")
-loadmodule("font-otf.lua")
-loadmodule("font-otb.lua")
-loadmodule("luatex-fonts-inj.lua")
-loadmodule("luatex-fonts-ota.lua")
-loadmodule("luatex-fonts-otn.lua")
-loadmodule("font-otp.lua")
-loadmodule("luatex-fonts-lua.lua")
-loadmodule("font-def.lua")
-loadmodule("luatex-fonts-def.lua")
-loadmodule("luatex-fonts-ext.lua")
-loadmodule("luatex-fonts-cbk.lua")
-