Merge pull request #330 from phi-gamma/master

fontloader update

3 files changed, 220 insertions, 100 deletions

diff --git a/scripts/mkcharacters b/scripts/mkcharacters
index 59582f2..a31c19c 100755
--- a/scripts/mkcharacters
+++ b/scripts/mkcharacters
@@ -4,16 +4,17 @@
 --        USAGE:  ./mkcharacters.lua 
 --  DESCRIPTION:  import parts of char-def.lua
 -- REQUIREMENTS:  lua, ConTeXt, the lualibs package
---       AUTHOR:  Philipp Gesang (Phg), <phg42.2a@gmail.com>
+--       AUTHOR:  Philipp Gesang (Phg), <phg@phi-gamma.net>
 -----------------------------------------------------------------------
--- we create a stripped-down version of char-def.lua
+-- We create a stripped-down version of char-def.lua, suitable for use
+-- with the generic font loader.
 -----------------------------------------------------------------------
 
 -----------------------------------------------------------------------
 --                              config
 -----------------------------------------------------------------------
-local charfile      = "./build/luaotfload-characters.lua"
-local chardef       = arg[1]
+local mkivpath      = arg[1]
+local charfile      = arg[2] or "./build/luaotfload-characters.lua"
 
 ---  for every code point char-def.lua provides a set of fields. they
 ---  are:
@@ -54,29 +55,36 @@ local import = {
 
 kpse.set_program_name"luatex"
 
-for _, lib in next, { "lualibs-lua.lua",
-                      "lualibs-lpeg.lua",
-                      "lualibs-table.lua", } do
-  local found = assert(kpse.find_file(lib, "lua"),
-                       "Could not locate " .. lib .. ".\n"
-                       .. "Please install the lualibs package.")
-  require(found)
-end
+require "lualibs"
+
+local chardef
+local charini
 
-if not chardef then
-  chardef = kpse.expand_path("~/context/tex/texmf-context/tex/context/base/")
-          .. "/char-def.lua"
+if not mkivpath then
+  mkivpath = assert (kpse.expand_path
+                      "~/context/tex/texmf-context/tex/context/base/mkiv/",
+                     "Failed to locate ConTeXt.")
 end
 
+chardef = mkivpath .. "/char-def.lua"
+charini = mkivpath .. "/char-ini.lua"
+
+--- we could grab the files from contextgarden but as Context is part
+--- of TL it’s not worth bothering
 if not (chardef and lfs.isfile(chardef)) then
-  --- we could grab the file from contextgarden but as Context is part
-  --- of TL it’s not worth bothering 
   chardef = assert(kpse.find_file("char-def.lua", "lua"),
-                   "Could not find ConTeXt.")
+                   "Failed to locate file char-def.lua from ConTeXt.")
+end
+
+if not (charini and lfs.isfile(charini)) then
+  charini = assert(kpse.find_file("char-ini.lua", "lua"),
+                   "Failed to locate file char-ini.lua from ConTeXt.")
 end
 
 io.write(string.format("extracting data from char-def.lua at %s\n",
                        chardef))
+io.write(string.format("loading code from char-ini.lua at %s\n",
+                       charini))
 
 -----------------------------------------------------------------------
 --                           functionality
@@ -127,6 +135,60 @@ local extract_fields = function (data)
   return extract_fields_indeed(data, {}, nil)
 end
 
+--[[ extract_classifiers : from luatex-basics-prepare.tex ]]
+
+local extract_classifiers = function (chardata)
+  dofile (charini)
+  local s_init = 1    local s_rphf =  7
+  local s_medi = 2    local s_half =  8
+  local s_fina = 3    local s_pref =  9
+  local s_isol = 4    local s_blwf = 10
+  local s_mark = 5    local s_pstf = 11
+  local s_rest = 6
+
+  local mappers = {
+    l = s_init,  -- left
+    d = s_medi,  -- double
+    c = s_medi,  -- joiner
+    r = s_fina,  -- right
+    u = s_isol,  -- nonjoiner
+  }
+
+  local first_arabic,  last_arabic  = characters.blockrange("arabic")
+  local first_syriac,  last_syriac  = characters.blockrange("syriac")
+  local first_mandiac, last_mandiac = characters.blockrange("mandiac")
+  local first_nko,     last_nko     = characters.blockrange("nko")
+
+  local classifiers = { }
+
+  for k, c in next, chardata do
+    if k > 0 then
+      local c = chardata[k]
+      if c then
+        local arabic = c.arabic
+        if arabic then
+          classifiers[k] = mappers[arabic]
+          elseif k >= first_arabic  and k <= last_arabic  or k >= first_syriac  and k <= last_syriac  or
+            k >= first_mandiac and k <= last_mandiac or k >= first_nko     and k <= last_nko     then
+            if c.category == "mn" then
+              classifiers[k] = s_mark
+            else
+              classifiers[k] = s_rest
+            end
+          end
+        end
+      end
+    end
+    return classifiers
+  end
+
+local amend_table_fields = function (data, classifiers)
+  --- installed by luatex-basics-prepare.tex
+  data.characters  = { }
+  data.classifiers = classifiers
+  return data
+end
+
 local writedata = function (data)
   local outchan = io.open(charfile, "w")
   if not outchan then
@@ -140,8 +202,10 @@ end
 
 do
   local chardata    = get_characters()
+  local classifiers = extract_classifiers(chardata)
   local stripped    = extract_fields(chardata)
-  local serialized  = table.serialize(stripped, true, {
+  local amended     = amend_table_fields(stripped, classifiers)
+  local serialized  = table.serialize(amended, true, {
     compact   = true,
     noquotes  = true,
     hexify    = true, --- for consistency with char-def
diff --git a/scripts/mkimport b/scripts/mkimport
index 08537d7..25d7560 100755
--- a/scripts/mkimport
+++ b/scripts/mkimport
@@ -63,7 +63,7 @@ local luatex_fonts_prefix = "luatex"
 local fontloader_subdir   = "src/fontloader"
 
 local origin_paths = {
-  context    = "tex/context/base",
+  context    = { "tex/context/base/mkiv", "tex/context/base", },
   fontloader = "tex/generic/context/luatex",
 }
 
@@ -105,12 +105,22 @@ local loader_target_dir  = "/build/"
 -------------------------------------------------------------------------------
 
 local die = function (...)
-  io.stderr:write "[\x1b[1;30;41mfatal error\x1b[0m]: "
+  io.stderr:write "[\x1b[1;30;41mfatal error\x1b[0m] "
   io.stderr:write (stringformat (...))
   io.stderr:write "\naborting.\n"
   os.exit (1)
 end
 
+local uncertain = 0
+
+local hmm = function (...)
+  uncertain = uncertain + 1
+  --[[ sorta like an error but non-fatal ]]
+  io.stderr:write "[\x1b[1;31merror\x1b[0m] "
+  io.stderr:write (stringformat (...))
+  io.stderr:write "; continuing nontheless.\n"
+end
+
 local emphasis = function (txt)
   return stringformat("\x1b[1m%s\x1b[0m", txt)
 end
@@ -191,18 +201,13 @@ local imports = {
     { name = "basics-gen"        , ours = nil          , kind = kind_essential },
     { name = "basics-nod"        , ours = nil          , kind = kind_merged    },
     { name = "basics"            , ours = nil          , kind = kind_tex       },
-    { name = "fonts-cbk"         , ours = nil          , kind = kind_merged    },
-    { name = "fonts-def"         , ours = nil          , kind = kind_merged    },
     { name = "fonts-demo-vf-1"   , ours = nil          , kind = kind_ignored   },
     { name = "fonts-enc"         , ours = nil          , kind = kind_merged    },
     { name = "fonts-ext"         , ours = nil          , kind = kind_merged    },
-    { name = "fonts-lua"         , ours = nil          , kind = kind_merged    },
     { name = "fonts-merged"      , ours = "reference"  , kind = kind_essential },
-    { name = "fonts-ota"         , ours = nil          , kind = kind_merged    },
     { name = "fonts"             , ours = nil          , kind = kind_merged    },
     { name = "fonts"             , ours = nil          , kind = kind_tex       },
     { name = "fonts-syn"         , ours = nil          , kind = kind_ignored   },
-    { name = "fonts-tfm"         , ours = nil          , kind = kind_merged    },
     { name = "languages"         , ours = nil          , kind = kind_ignored   },
     { name = "languages"         , ours = nil          , kind = kind_tex       },
     { name = "math"              , ours = nil          , kind = kind_ignored   },
@@ -224,18 +229,28 @@ local imports = {
     { name = "data-con"          , ours = "data-con"          , kind = kind_merged    },
     { name = "font-afk"          , ours = "font-afk"          , kind = kind_merged    },
     { name = "font-afm"          , ours = "font-afm"          , kind = kind_merged    },
+    { name = "font-cff"          , ours = "font-cff"          , kind = kind_merged    },
     { name = "font-cid"          , ours = "font-cid"          , kind = kind_merged    },
     { name = "font-con"          , ours = "font-con"          , kind = kind_merged    },
     { name = "font-def"          , ours = "font-def"          , kind = kind_merged    },
+    { name = "font-dsp"          , ours = "font-dsp"          , kind = kind_merged    },
+    { name = "font-gbn"          , ours = "font-gbn"          , kind = kind_merged    },
     { name = "font-ini"          , ours = "font-ini"          , kind = kind_merged    },
-    { name = "font-inj"          , ours = "font-inj"          , kind = kind_merged    },
+    { name = "font-lua"          , ours = "font-lua"          , kind = kind_merged    },
     { name = "font-map"          , ours = "font-map"          , kind = kind_merged    },
-    { name = "font-otb"          , ours = "font-otb"          , kind = kind_merged    },
-    { name = "font-otf"          , ours = "font-otf"          , kind = kind_merged    },
+    { name = "font-osd"          , ours = "font-osd"          , kind = kind_merged    },
+    { name = "font-ota"          , ours = "font-ota"          , kind = kind_merged    },
+    { name = "font-otd"          , ours = "font-otd"          , kind = kind_merged    },
     { name = "font-oti"          , ours = "font-oti"          , kind = kind_merged    },
-    { name = "font-otn"          , ours = "font-otn"          , kind = kind_merged    },
-    { name = "font-otp"          , ours = "font-otp"          , kind = kind_merged    },
+    { name = "font-otj"          , ours = "font-otj"          , kind = kind_merged    },
+    { name = "font-otl"          , ours = "font-otl"          , kind = kind_merged    },
+    { name = "font-oto"          , ours = "font-oto"          , kind = kind_merged    },
+    { name = "font-otr"          , ours = "font-otr"          , kind = kind_merged    },
+    { name = "font-ots"          , ours = "font-ots"          , kind = kind_merged    },
+    { name = "font-oup"          , ours = "font-oup"          , kind = kind_merged    },
     { name = "font-tfm"          , ours = "font-tfm"          , kind = kind_merged    },
+    { name = "font-ttf"          , ours = "font-ttf"          , kind = kind_merged    },
+
     { name = "l-boolean"         , ours = "l-boolean"         , kind = kind_lualibs   },
     { name = "l-file"            , ours = "l-file"            , kind = kind_lualibs   },
     { name = "l-function"        , ours = "l-function"        , kind = kind_lualibs   },
@@ -246,6 +261,7 @@ local imports = {
     { name = "l-string"          , ours = "l-string"          , kind = kind_lualibs   },
     { name = "l-table"           , ours = "l-table"           , kind = kind_lualibs   },
     { name = "util-str"          , ours = "util-str"          , kind = kind_lualibs   },
+    { name = "util-fil"          , ours = "util-fil"          , kind = kind_lualibs   },
   }, --[[ [context] ]]
 } --[[ [imports] ]]
 
@@ -266,33 +282,39 @@ local package = {
 ---   [08] l-boolean.lua
 ---   [09] l-math.lua
 ---   [10] util-str.lua
----   [11] luatex-basics-gen.lua
----   [12] data-con.lua
----   [13] luatex-basics-nod.lua
----   [14] font-ini.lua
----   [15] font-con.lua
----   [16] luatex-fonts-enc.lua
----   [17] font-cid.lua
----   [18] font-map.lua
----   [19] luatex-fonts-syn.lua
----   [20] font-tfm.lua
----   [21] font-afm.lua
----   [22] font-afk.lua
----   [23] luatex-fonts-tfm.lua
----   [24] font-oti.lua
----   [25] font-otf.lua
----   [26] font-otb.lua
----   [27] font-inj.lua
----   [28] luatex-fonts-ota.lua
----   [30] font-otn.lua
----   [30] font-otp.lua
----   [31] luatex-fonts-lua.lua
----   [32] font-def.lua
----   [33] luatex-fonts-def.lua
----   [34] luatex-fonts-ext.lua
----   [35] luatex-fonts-cbk.lua
+---   [11] util-fil.lua
+---   [12] luatex-basics-gen.lua
+---   [13] data-con.lua
+---   [14] luatex-basics-nod.lua
+---   [15] luatex-basics-chr.lua
+---   [16] font-ini.lua
+---   [17] font-con.lua
+---   [18] luatex-fonts-enc.lua
+---   [19] font-cid.lua
+---   [20] font-map.lua
+---   [21] luatex-fonts-syn.lua
+---   [22] font-tfm.lua
+---   [23] font-afm.lua
+---   [24] font-afk.lua
+---   [25] font-oti.lua
+---   [26] font-otr.lua
+---   [27] font-cff.lua
+---   [28] font-ttf.lua
+---   [29] font-dsp.lua
+---   [30] font-oup.lua
+---   [31] font-otl.lua
+---   [32] font-oto.lua
+---   [33] font-otj.lua
+---   [34] font-ota.lua
+---   [35] font-ots.lua
+---   [36] font-osd.lua
+---   [37] font-lua.lua
+---   [38] font-def.lua
+---   [39] font-xtx.lua
+---   [40] luatex-fonts-ext.lua
+---   [41] font-gbn.lua
 ---
---- Of these, nos. 01--10 are provided by the Lualibs. Keeping them
+--- Of these, nos. 01--11 are provided by the Lualibs. Keeping them
 --- around in the Luaotfload fontloader is therefore unnecessary.
 --- Packaging needs to account for this difference.
 
@@ -306,12 +328,18 @@ local package = {
     "l-boolean",
     "l-math",
     "util-str",
+    "util-fil",
 
 --- Another file containing auxiliary definitions must be present
 --- prior to initialization of the configuration.
 
     "luatex-basics-gen",
 
+--- We have a custom script for autogenerating data so we don’t use the
+--- definitions from upstream.
+
+    "basics-chr",
+
   }, --[[ [package.optional] ]]
 
 --- The files below constitute the “fontloader proper”. Some of the
@@ -332,19 +360,22 @@ local package = {
     "font-tfm",
     "font-afm",
     "font-afk",
-    "fonts-tfm",
     "font-oti",
-    "font-otf",
-    "font-otb",
-    "font-inj",
-    "fonts-ota",
-    "font-otn",
-    "font-otp",
-    "fonts-lua",
+    "font-otr",
+    "font-cff",
+    "font-ttf",
+    "font-dsp",
+    "font-oup",
+    "font-otl",
+    "font-oto",
+    "font-otj",
+    "font-ota",
+    "font-ots",
+    "font-osd",
+    "font-lua",
     "font-def",
-    "fonts-def",
     "fonts-ext",
-    "fonts-cbk",
+    "font-gbn",
 
   }, --[[ [package.required] ]]
 
@@ -352,7 +383,8 @@ local package = {
 
 local hash_file = function (fname)
   if not lfsisfile (fname) then
-    die ("cannot find %s.", fname)
+    hmm ("cannot find %s", fname)
+    return nil
   end
   local raw = ioloaddata (fname)
   if not raw then
@@ -361,12 +393,27 @@ local hash_file = function (fname)
   return md5sumhexa (raw)
 end
 
+local first_existing_subpath = function (pfx, subs)
+  if not subs then return nil end
+  local t_subs = type (subs)
+  if t_subs == "table" then
+    for i = 1, #subs do
+      local sub = subs[i]
+      local pth = file.join (pfx, sub)
+      if lfsisdir (pth) then return pth end
+    end
+  elseif t_subs == "string" then
+    local pth = file.join (pfx, subs)
+    if lfsisdir (pth) then return pth end
+  end
+  return nil
+end
+
 local derive_category_path = function (cat)
-  local subpath  = origin_paths[cat] or die ("category " .. cat .. " unknown")
-  local location = file.join (parms.context_root, subpath)
-  if not lfsisdir (location) then
-    die ("invalid base path defined for category "
-         .. cat .. " at " .. location)
+  local location = first_existing_subpath (parms.context_root,
+                                           origin_paths[cat])
+  if not location then
+    die ("invalid base path defined for category " .. cat)
   end
   return location
 end
@@ -476,20 +523,21 @@ local news = function ()
         end
         if is_readable (ourpath) then imported = true end
         local src_hash = hash_file (fullpath)
-        local dst_hash = imported and hash_file (ourpath)
-        local same     = src_hash == dst_hash -- same!
-
-        if same then
-          good ("file %s unchanged", emphasis (ourname))
-          status.good[#status.good + 1] = ourname
-        elseif not dst_hash then
-          attention ("new file %s requires import from %s",
-                    emphasis (ourname),
-                    emphasis (fullpath))
-          status.create[#status.create + 1] = ourname
-        else --- src and dst exist but differ
-          attention ("file %s requires import", emphasis (ourname))
-          status.import[#status.import + 1] = ourname
+        if src_hash then
+          local dst_hash = imported and hash_file (ourpath)
+          local same     = src_hash == dst_hash -- same!
+          if same then
+            good ("file %s unchanged", emphasis (ourname))
+            status.good[#status.good + 1] = ourname
+          elseif not dst_hash then
+            attention ("new file %s requires import from %s",
+                      emphasis (ourname),
+                      emphasis (fullpath))
+            status.create[#status.create + 1] = ourname
+          else --- src and dst exist but differ
+            attention ("file %s requires import", emphasis (ourname))
+            status.import[#status.import + 1] = ourname
+          end
         end
       end
 
@@ -541,7 +589,7 @@ local summarize_status = function (counters)
   local sum = imported + skipped + created + failed
   if sum < 1 then die ("garbage total of imported files: %s", sum) end
   separator ()
-  status (" RESULT: %d files processed", sum)
+  status (" RESULT: %d files processed, %d errors", sum, uncertain)
   separator ()
   if created  > 0 then status ("created:  %d (%d %%)", created , created  * 100 / sum) end
   if imported > 0 then status ("imported: %d (%d %%)", imported, imported * 100 / sum) end
@@ -593,7 +641,7 @@ local import = function (arg)
       local def  = defs[i]
       local stat = import_file (def.name, def.kind, def, cat)
       if stat == import_failed then
-        die (stringformat ("import failed at file %d of %d (%s)",
+        hmm (stringformat ("import failed at file %d of %d (%s)",
                            i, ndefs, def.name))
       end
       statcount[stat] = statcount[stat] or 0
@@ -601,7 +649,7 @@ local import = function (arg)
     end
   end
   summarize_status (statcount)
-  return 0
+  return uncertain == 0 and 0 or -42
 end --[[ [local import = function (arg)] ]]
 
 local find_in_path = function (root, subdir, target)
diff --git a/scripts/mkstatus b/scripts/mkstatus
index c5ded0d..5e3be9a 100755
--- a/scripts/mkstatus
+++ b/scripts/mkstatus
@@ -92,23 +92,30 @@ local names = {
   { miscdir,      "fontloader-font-def.lua",        },
   { miscdir,      "fontloader-font-ini.lua",        },
   { miscdir,      "fontloader-font-map.lua",        },
-  { miscdir,      "fontloader-font-otb.lua",        },
-  { miscdir,      "fontloader-font-otf.lua",        },
   { miscdir,      "fontloader-font-oti.lua",        },
-  { miscdir,      "fontloader-font-otp.lua",        },
-  { miscdir,      "fontloader-fonts-cbk.lua",       },
-  { miscdir,      "fontloader-fonts-def.lua",       },
+  { miscdir,      "fontloader-font-gbn.lua",        },
+  { miscdir,      "fontloader-font-def.lua",        },
   { miscdir,      "fontloader-fonts-demo-vf-1.lua", },
   { miscdir,      "fontloader-fonts-enc.lua",       },
   { miscdir,      "fontloader-fonts-ext.lua",       },
-  { miscdir,      "fontloader-font-inj.lua",        },
   { miscdir,      "fontloader-fonts.lua",           },
-  { miscdir,      "fontloader-fonts-lua.lua",       },
-  { miscdir,      "fontloader-fonts-ota.lua",       },
-  { miscdir,      "fontloader-font-otn.lua",        },
+  { miscdir,      "fontloader-font-lua.lua",        },
   { miscdir,      "fontloader-fonts-syn.lua",       },
-  { miscdir,      "fontloader-fonts-tfm.lua",       },
   { miscdir,      "fontloader-font-tfm.lua",        },
+  { miscdir,      "fontloader-font-tfm.lua",        },
+  { miscdir,      "fontloader-font-otr.lua",        },
+  { miscdir,      "fontloader-font-cff.lua",        },
+  { miscdir,      "fontloader-font-ttf.lua",        },
+  { miscdir,      "fontloader-font-dsp.lua",        },
+  { miscdir,      "fontloader-font-oup.lua",        },
+  { miscdir,      "fontloader-font-otl.lua",        },
+  { miscdir,      "fontloader-font-oto.lua",        },
+  { miscdir,      "fontloader-font-otj.lua",        },
+  { miscdir,      "fontloader-font-ota.lua",        },
+  { miscdir,      "fontloader-font-ots.lua",        },
+  { miscdir,      "fontloader-font-osd.lua",        },
+
+  --- lua libraries
   { miscdir,      "fontloader-languages.lua",       },
   { miscdir,      "fontloader-l-boolean.lua",       },
   { miscdir,      "fontloader-l-file.lua",          },
@@ -125,6 +132,7 @@ local names = {
   { miscdir,      "fontloader-swiglib.lua",         },
   { miscdir,      "fontloader-swiglib-test.lua",    },
   { miscdir,      "fontloader-util-str.lua",        },
+  { miscdir,      "fontloader-util-fil.lua",        },
 
 } --[[local names]]