Merge pull request #324 from phi-gamma/master

lookups, combos, fallbacks

8 files changed, 325 insertions, 61 deletions

diff --git a/doc/luaotfload.conf.rst b/doc/luaotfload.conf.rst
index d624e00..a77c644 100644
--- a/doc/luaotfload.conf.rst
+++ b/doc/luaotfload.conf.rst
@@ -6,7 +6,7 @@
                      Luaotfload configuration file
 -----------------------------------------------------------------------
 
-:Date:                  2015-12-09
+:Date:                  2016-02-04
 :Copyright:             GPL v2.0
 :Version:               2.6
 :Manual section:        5
@@ -155,10 +155,10 @@ The list of ``formats`` must be a comma separated sequence of strings
 containing one or more of these elements:
 
 * ``otf``               (OpenType format),
-* ``ttf`` and ``ttc``   (TrueType format),
+* ``ttf`` and ``ttc``       (TrueType format),
 * ``dfont``             (Macintosh TrueType format),
 * ``afm``               (Adobe Font Metrics),
-* ``pfb`` and ``pfa``   (PostScript format).
+* ``pfb`` and ``pfa``       (PostScript format).
 
 It corresponds loosely to the ``--formats`` option to
 **luaotfload-tool**. Invalid or duplicate members are ignored; if the
@@ -293,7 +293,7 @@ restore the previous behavior if necessary.
 The ``definer`` allows for switching the ``define_font`` callback.
 Apart from the default ``patch`` one may also choose the ``generic``
 one that comes with the vanilla fontloader. Beware that this might
-break tools like Fontspect that rely on the ``patch_font`` callback
+break tools like Fontspec that rely on the ``patch_font`` callback
 provided by Luaotfload to perform important corrections on font data.
 
 The fontloader backend can be selected by setting the value of
diff --git a/src/fontloader/misc/fontloader-font-otn.lua b/src/fontloader/misc/fontloader-font-otn.lua
index 081630a..9cdb79b 100644
--- a/src/fontloader/misc/fontloader-font-otn.lua
+++ b/src/fontloader/misc/fontloader-font-otn.lua
@@ -3897,9 +3897,9 @@ local function split(replacement,original)
 end
 
 local valid = { -- does contextpos work?
-    coverage        = { chainsub = true, chainpos = true, contextsub = true, contextpos = true },
+    coverage        = { chainsub = true, chainpos = true, contextsub = true, contextpos = false },
     reversecoverage = { reversesub = true },
-    glyphs          = { chainsub = true, chainpos = true, contextsub = true, contextpos = true },
+    glyphs          = { chainsub = true, chainpos = true, contextsub = true, contextpos = false },
 }
 
 local function prepare_contextchains(tfmdata)
diff --git a/src/luaotfload-database.lua b/src/luaotfload-database.lua
index 367723b..1d5dfd8 100644
--- a/src/luaotfload-database.lua
+++ b/src/luaotfload-database.lua
@@ -583,8 +583,6 @@ local style_category = {
 
 local type1_metrics = { "tfm", "ofm", }
 
-local dummy_findfile = resolvers.findfile -- from basics-gen
-
 local lookup_filename = function (filename)
     if not name_index then name_index = load_names () end
     local files    = name_index.files
@@ -630,6 +628,8 @@ end
 
 --doc]]--
 
+local dummy_findfile = resolvers.findfile -- from basics-gen
+
 --- string -> string * string * bool
 local lookup_font_file
 lookup_font_file = function (filename)
diff --git a/src/luaotfload-features.lua b/src/luaotfload-features.lua
index fd43e58..4bcfbc3 100644
--- a/src/luaotfload-features.lua
+++ b/src/luaotfload-features.lua
@@ -19,6 +19,7 @@ local C                 = lpeg.C
 
 local table             = table
 local tabletohash       = table.tohash
+local tablesort         = table.sort
 local setmetatableindex = table.setmetatableindex
 local insert            = table.insert
 
@@ -28,6 +29,7 @@ local insert            = table.insert
 local fonts             = fonts
 local definers          = fonts.definers
 local handlers          = fonts.handlers
+local fontidentifiers   = fonts.hashes and fonts.hashes.identifiers
 
 local as_script, normalize
 
@@ -65,6 +67,135 @@ local stringformat     = string.format
 local stringis_empty   = string.is_empty
 local mathceil         = math.ceil
 
+local cmp_by_idx = function (a, b) return a.idx < b.idx end
+
+local defined_combos = 0
+
+local handle_combination = function (combo, spec)
+    defined_combos = defined_combos + 1
+    if not combo [1] then
+        report ("both", 0, "load",
+                "combo %d: Empty font combination requested.",
+                defined_combos)
+        return false
+    end
+
+    if not fontidentifiers then
+        fontidentifiers = fonts.hashes and fonts.hashes.identifiers
+    end
+
+    local chain   = { }
+    local fontids = { }
+    local n       = #combo
+
+    tablesort (combo, cmp_by_idx)
+
+    --- pass 1: skim combo and resolve fonts
+    report ("both", 0, "load", "combo %d: combining %d fonts.",
+            defined_combos, n)
+    for i = 1, n do
+        local cur = combo [i]
+        local id  = cur.id
+        local idx = cur.idx
+        local fnt = fontidentifiers [id]
+        if fnt then
+            local chars = cur.chars
+            if chars == true then
+                report ("both", 0, "load",
+                        " *> %.2d: fallback font %d at rank %d.",
+                        i, id, idx)
+            else
+                report ("both", 0, "load",
+                        " *> %.2d: include font %d at rank %d (%d items).",
+                        i, id, idx, (chars and #chars or 0))
+            end
+            chain   [#chain + 1]   = { fnt, chars, idx = idx }
+            fontids [#fontids + 1] = { id = id }
+        else
+            report ("both", 0, "load",
+                    " *> %.2d: font %d at rank %d unknown, skipping.",
+                    n, id, idx)
+            --- TODO might instead attempt to define the font at this point
+            ---      but that’d require some modifications to the syntax
+        end
+    end
+
+    local nc = #chain
+    if nc == 0 then
+        report ("both", 0, "load",
+                " *> no valid font (of %d) in combination.", n)
+        return false
+    end
+
+    local basefnt = chain [1] [1]
+    if nc == 1 then
+        report ("both", 0, "load",
+                " *> combination boils down to a single font (%s) \z
+                 of %d initially specified; not pursuing this any \z
+                 further.", basefnt.fullname, n)
+        return basefnt
+    end
+
+    local basechar       = basefnt.characters
+    local baseprop       = basefnt.properties
+    baseprop.name        = spec.name
+    baseprop.virtualized = true
+    basefnt.fonts        = fontids
+
+    for i = 2, nc do
+        local cur = chain [i]
+        local fnt = cur [1]
+        local def = cur [2]
+        local src = fnt.characters
+        local cnt = 0
+
+        local pickchr = function (uc, unavailable)
+            local chr = src [uc]
+            if unavailable == true and basechar [uc] then
+                --- fallback mode: already known
+                return
+            end
+            if chr then
+                chr.commands = { { "slot", i, uc } }
+                basechar [uc] = chr
+                cnt = cnt + 1
+            end
+        end
+
+        if def == true then --> fallback; grab all currently unavailable
+            for uc, _chr in next, src do pickchr (uc, true) end
+        else --> grab only defined range
+            for j = 1, #def do
+                local this = def [j]
+                if type (this) == "number" then
+                    report ("both", 0, "load",
+                            " *> [%d][%d]: import codepoint U+%.4X",
+                            i, j, this)
+                    pickchr (this)
+                elseif type (this) == "table" then
+                    local lo, hi = unpack (this)
+                    report ("both", 0, "load",
+                            " *> [%d][%d]: import codepoint range U+%.4X--U+%.4X",
+                            i, j, lo, hi)
+                    for uc = lo, hi do pickchr (uc) end
+                else
+                    report ("both", 0, "load",
+                            " *> item no. %d of combination definition \z
+                             %d not processable.", j, i)
+                end
+            end
+        end
+        report ("both", 0, "load",
+                " *> font %d / %d: imported %d glyphs into combo.",
+                i, nc, cnt)
+    end
+    spec.file       = basefnt.filename
+    spec.name       = stringformat ("luaotfload<%d>", defined_combos)
+    spec.features   = { normal = { spec.specification } }
+    spec.forced     = "evl"
+    spec.eval       = function () return basefnt end
+    return spec
+end
 
 ---[[ begin excerpt from font-ott.lua ]]
 
@@ -993,7 +1124,10 @@ local import_values = {
     { "mode",   true },
 }
 
-local lookup_types = { "anon", "file", "kpse", "my", "name", "path" }
+local lookup_types = { "anon"  , "file", "kpse"
+                     , "my"    , "name", "path"
+                     , "combo"
+                     }
 
 local select_lookup = function (request)
     for i=1, #lookup_types do
@@ -1049,6 +1183,7 @@ end
 local handle_request = function (specification)
     local request = lpegmatch(luaotfload.parsers.font_request,
                               specification.specification)
+----inspect(request)
     if not request then
         --- happens when called with an absolute path
         --- in an anonymous lookup;
@@ -1077,8 +1212,13 @@ local handle_request = function (specification)
         specification.lookup = "path"
         return specification
     end
-    local lookup, name  = select_lookup(request)
-    request.features    = apply_default_features(request.features)
+
+    local lookup, name = select_lookup (request)
+    if lookup == "combo" then
+        return handle_combination (request.combo, specification)
+    end
+
+    request.features = apply_default_features(request.features)
 
     if name then
         specification.name    = name
@@ -1116,8 +1256,8 @@ if as_script == true then --- skip the remainder of the file
     return
 else
     local registersplit = definers.registersplit
-    registersplit (":", handle_request, "cryptic")
-    registersplit ("",  handle_request, "more cryptic") -- catches \font\text=[names]
+    registersplit (":", handle_request, "common")
+    registersplit ("",  handle_request, "xetex path style") -- catches \font\text=[names]
 end
 
 ---[[ end included font-ltx.lua ]]
@@ -1749,4 +1889,4 @@ return {
     end
 }
 
--- vim:tw=71:sw=4:ts=4:expandtab
+-- vim:tw=79:sw=4:ts=4:expandtab
diff --git a/src/luaotfload-loaders.lua b/src/luaotfload-loaders.lua
index 0f22f46..fb01821 100644
--- a/src/luaotfload-loaders.lua
+++ b/src/luaotfload-loaders.lua
@@ -14,13 +14,42 @@ if not luaotfload then error "this module requires Luaotfload" end
 
 local logreport = luaotfload.log and luaotfload.log.report or print
 
+local lua_reader = function (specification)
+  local fullname = specification.filename or ""
+  if fullname == "" then
+    local forced = specification.forced or ""
+    if forced ~= "" then
+      fullname = specification.name .. "." .. forced
+    else
+      fullname = specification.name
+    end
+  end
+  local fullname = resolvers.findfile (fullname) or ""
+  if fullname ~= "" then
+    local loader = loadfile (fullname)
+    loader = loader and loader ()
+    return loader and loader (specification)
+  end
+end
+
+local eval_reader = function (specification)
+  local eval = specification.eval
+  if not eval or type (eval) ~= "function" then return nil end
+  logreport ("both", 0, "loaders",
+             "eval: found tfmdata for “%s”, injecting.",
+             specification.name)
+  return eval ()
+end
+
 local install_formats = function ()
   local fonts = fonts
   if not fonts then return false end
 
-  local readers  = fonts.readers
-  local handlers = fonts.handlers
-  local formats  = fonts.formats
+  local readers   = fonts.readers
+  local sequence  = readers.sequence
+  local seqset    = table.tohash (sequence)
+  local handlers  = fonts.handlers
+  local formats   = fonts.formats
   if not readers or not handlers or not formats then return false end
 
   local aux = function (which, reader)
@@ -32,10 +61,18 @@ local install_formats = function ()
     formats  [which] = "type1"
     readers  [which] = reader
     handlers [which] = { }
+    if not seqset [which] then
+      logreport ("both", 0, "loaders",
+                 "Extending reader sequence for “%s”.", which)
+      sequence [#sequence + 1] = which
+      seqset   [which]         = true
+    end
     return true
   end
 
-  return aux ("pfa", function (spec) return readers.opentype (spec, "pfa", "type1") end)
+  return aux ("evl", eval_reader)
+     and aux ("lua", lua_reader)
+     and aux ("pfa", function (spec) return readers.opentype (spec, "pfa", "type1") end)
      and aux ("pfb", function (spec) return readers.opentype (spec, "pfb", "type1") end)
      and aux ("ofm", readers.tfm)
 end
@@ -86,6 +123,7 @@ do
       logreport ("both", 0, "loaders", "   > name %q",     result.name     or "<nil>")
       logreport ("both", 0, "loaders", "   > fontname %q", result.fontname or "<nil>")
       logreport ("both", 0, "loaders", "   > fullname %q", result.fullname or "<nil>")
+      logreport ("both", 0, "loaders", "   > type %s",     result.type     or "<nil>")
       return result
     end
   end
diff --git a/src/luaotfload-main.lua b/src/luaotfload-main.lua
index c843f26..d5fdb9e 100644
--- a/src/luaotfload-main.lua
+++ b/src/luaotfload-main.lua
@@ -13,7 +13,7 @@ local luaotfload                  = luaotfload
 luaotfload.log                    = luaotfload.log or { }
 luaotfload.version                = "2.6"
 luaotfload.loaders                = { }
-luaotfload.min_luatex_version     = 80             --- i. e. 0.79
+luaotfload.min_luatex_version     = 80             --- i. e. 0.80
 luaotfload.fontloader_package     = "reference"    --- default: from current Context
 
 local authors = "\z
diff --git a/src/luaotfload-parsers.lua b/src/luaotfload-parsers.lua
index 0349cdc..b190c2c 100644
--- a/src/luaotfload-parsers.lua
+++ b/src/luaotfload-parsers.lua
@@ -45,6 +45,7 @@ local lfsisdir          = lfs.isdir
 ---                         COMMON PATTERNS
 -------------------------------------------------------------------------------
 
+local asterisk          = P"*"
 local dot               = P"."
 local colon             = P":"
 local semicolon         = P";"
@@ -72,6 +73,8 @@ local digit             = R"09"
 local alpha             = R("az", "AZ")
 local anum              = alpha + digit
 local decimal           = digit^1 * (dot * digit^0)^-1
+local hexdigit          = R("09", "af", "AF")
+local hex               = P"0x" * hexdigit^1
 
 -------------------------------------------------------------------------------
 ---                                FONTCONFIG
@@ -560,9 +563,49 @@ local modifier          = slash * (other_modifier      --> ignore
                                  + garbage_modifier)   --> warn
 local modifier_list     = Cg(Ct(modifier^0), "modifiers")
 
+--- combining ---------------------------------------------------------
+---
+--- \font \three = combo: 1->42; 2->99,0xdeadbeef; 3->(1,U+2a-U+1337*42)
+---                       v  +~  v  +~ +~~~~~~~~~  v   v +~~~ +~~~~~ +~
+---                       |  |   |  |  |           |   | |    |      |
+---                       |  |   |  |  |           |   | |    |      |
+---        idx   :     ---+--÷---+--÷--÷-----------+   | +----+      |
+---                          |      |  |               | |           |
+---        id    :     ------+------+--÷---------------+ +-----------+
+---                                    |                 |
+---        chars :     ----------------+-----------------+
+---
+local comborowsep       = ws * semicolon * ws
+local combomapsep       = ws * P"->"     * ws
+local combodefsep       = ws * comma     * ws
+local comborangesep     = ws * asterisk  * ws
+
+local combohex          = hex     / tonumber
+local combouint         = digit^1 / tonumber
+local tonumber16        = function (s) return tonumber (s, 16) end
+local combouni          = P"U+" * (digit^1 / tonumber16)
+local combonum          = combohex + combouni + combouint
+local comborange        = Ct(combonum * dash * combonum) + combonum
+local combochars        = comborange * (comborangesep * comborange)^0
+local parenthesized     = function (p) return P"(" * ws * p * ws * P")" end
+
+local comboidxpat       = Cg(combouint, "idx")
+local comboidpat        = Cg(combouint, "id" )
+local combocharspat     = Cg(P"fallback" * Cc(true) + Ct(combochars^1), "chars")
+local comboidcharspat   = comboidpat * combodefsep * combocharspat
+
+local comboidx          = parenthesized (comboidxpat    ) + comboidxpat
+local comboid           = parenthesized (comboidpat     ) + comboidpat
+local comboidchars      = parenthesized (comboidcharspat) + comboidcharspat
+
+local combodef1         = Ct(comboidx * combomapsep * comboid) --> no chars
+local combodef          = Ct(comboidx * combomapsep * comboidchars)
+local combolist         = Ct(combodef1 * (comborowsep * combodef)^1)
+
 --- lookups -----------------------------------------------------------
 local fontname          = C((1-S":(/")^1)  --- like luatex-fonts
 local unsupported       = Cmt((1-S":(")^1, check_garbage)
+local combo             = P"combo:" * ws * Cg(combolist, "combo")
 local prefixed          = P"name:" * ws * Cg(fontname, "name")
 --- initially we intended file: to emulate the behavior of
 --- luatex-fonts, i.e. no paths allowed. after all, we do have XeTeX
@@ -626,6 +669,7 @@ local specification     = (prefixed + unprefixed)
                         * subfont^-1
                         * modifier_list^-1
 local font_request      = Ct(path_lookup   * (colon^-1 * features)^-1
+                           + combo --> TODO: feature list needed?
                            + specification * (colon    * features)^-1)
 
 --  lpeg.print(font_request)
diff --git a/src/luaotfload-resolvers.lua b/src/luaotfload-resolvers.lua
index 3d7f6b0..05a0656 100644
--- a/src/luaotfload-resolvers.lua
+++ b/src/luaotfload-resolvers.lua
@@ -62,14 +62,18 @@ local logreport           = luaotfload.log.report
 
 local resolve_file
 resolve_file = function (specification)
-    local name   = fonts.names.lookup_font_file (specification.name)
+    local name, _format, success = fonts.names.lookup_font_file (specification.name)
     local suffix = filesuffix (name)
     if fonts.formats[suffix] then
         specification.forced      = stringlower (suffix)
-        specification.forcedname  = fileremovesuffix(name)
+        specification.forcedname  = fileremovesuffix (name)
     else
         specification.name = name
     end
+    if success ~= true then
+        logreport ("log", 1, "resolve", "file lookup of %q unsuccessful", name)
+    end
+    return success
 end
 
 --[[doc--
@@ -85,22 +89,22 @@ end
 local resolve_path
 resolve_path = function (specification)
     local name       = specification.name
-    local exists, _  = lfsisfile(name)
+    local exists, _  = lfsisfile (name)
     if not exists then -- resort to file: lookup
-        logreport ("log", 0, "load",
+        logreport ("log", 1, "resolve",
                    "path lookup of %q unsuccessful, falling back to file:",
                    name)
-        resolve_file (specification)
+        return resolve_file (specification)
+    end
+    local suffix = filesuffix (name)
+    if fonts.formats [suffix] then
+        specification.forced      = stringlower (suffix)
+        specification.name        = fileremovesuffix (name)
+        specification.forcedname  = name
     else
-        local suffix = filesuffix (name)
-        if fonts.formats[suffix] then
-            specification.forced      = stringlower (suffix)
-            specification.name        = fileremovesuffix(name)
-            specification.forcedname  = name
-        else
-            specification.name = name
-        end
+        specification.name = name
     end
+    return true
 end
 
 --[[doc--
@@ -120,18 +124,17 @@ resolve_name = function (specification)
     end
     local resolved, subfont = resolver (specification)
     if resolved then
-        logreport ("log", 0, "load", "Lookup/name: %q -> \"%s%s\"",
-                   specification.name,
-                   resolved,
+        logreport ("log", 1, "resolve", "name lookup %q -> \"%s%s\"",
+                   specification.name, resolved,
                    subfont and stringformat ("(%d)", subfont) or "")
         specification.resolved   = resolved
         specification.sub        = subfont
         specification.forced     = stringlower (filesuffix (resolved) or "")
         specification.forcedname = resolved
         specification.name       = fileremovesuffix (resolved)
-    else
-        resolve_file (specification)
+        return true
     end
+    return resolve_file (specification)
 end
 
 --[[doc--
@@ -158,36 +161,74 @@ end
 
 --doc]]--
 
-local type1_formats = { "tfm", "ofm", "TFM", "OFM", }
+local tex_formats = { "tfm", "ofm", "TFM", "OFM", }
 
-local resolve_anon
-resolve_anon = function (specification)
+local resolve_tex_format = function (specification)
     local name = specification.name
-    for i=1, #type1_formats do
-        local format = type1_formats[i]
+    for i=1, #tex_formats do
+        local format = tex_formats [i]
         local suffix = filesuffix (name)
-        if resolvers.findfile(name, format) then
+        if resolvers.findfile (name, format) then
             local usename = suffix == format and fileremovesuffix (name) or name
             specification.forcedname = file.addsuffix (usename, format)
             specification.forced     = format
-            return
+            return true
         end
     end
-    --- under some weird circumstances absolute paths get
-    --- passed to the definer; we have to catch them
-    --- before the name: resolver misinterprets them.
-    name = specification.specification
-    local exists, _ = lfsisfile(name)
-    if exists then --- garbage; we do this because we are nice,
-                   --- not because it is correct
-        logreport ("log", 1, "load", "file %q exists", name)
-        logreport ("log", 1, "load",
-                   "... overriding borked anon: lookup with path: lookup")
-        specification.name = name
-        resolve_path (specification)
-        return
+    return false
+end
+
+local resolve_path_if_exists = function (specification)
+    local spec = specification.specification
+    local exists, _void = lfsisfile (spec)
+    if exists then
+        --- If this path is taken a file matching the specification
+        --- literally was found. In this situation, Luaotfload is
+        --- expected to load that file directly, even though we provide
+        --- explicit “file” and “path” lookups to address exactly this
+        --- situation.
+        logreport ("log", 1, "resolve",
+                   "file %q exists, performing path lookup", spec)
+        specification.name = spec
+        return resolve_path (specification)
+    end
+    return false
+end
+
+local resolve_methods = {
+    tex  = resolve_tex_format,
+    path = resolve_path_if_exists,
+    name = resolve_name,
+}
+
+local resolve_sequence = function (seq, specification)
+    for i = 1, #seq do
+        local id  = seq [i]
+        local mth = resolve_methods [id]
+        logreport ("both", 0, "resolve", "step %d: apply method %q (%s)", i, id, mth)
+        if mth (specification) == true then
+            logreport ("both", 0, "resolve",
+                       "%d: method %q indicated lookup success", i, id)
+            logreport ("both", 0, "resolve",
+                       "method %q resolved %q -> %s (%s)",
+                       id, specification.specification,
+                       specification.name,
+                       specification.forcedname)
+            return true
+        end
     end
-    resolve_name (specification)
+    logreport ("both", 0, "resolve",
+               "sequence of %d lookups yielded nothing appropriate", #seq)
+    return false
+end
+
+local default_anon_sequence = {
+    "tex", "path", "name",
+}
+
+local resolve_anon
+resolve_anon = function (specification)
+    return resolve_sequence (default_anon_sequence, specification)
 end
 
 --[[doc--
@@ -207,16 +248,17 @@ resolve_kpse = function (specification)
         if resolvers.findfile (name, suffix) then
             specification.forced       = stringlower (suffix)
             specification.forcedname   = name
-            return
+            return true
         end
     end
     for t, format in next, fonts.formats do --- brute force
         if kpsefind_file (name, format) then
             specification.forced = t
             specification.name   = name
-            return
+            return true
         end
     end
+    return false
 end
 
 --[[doc--
@@ -250,5 +292,5 @@ return {
     end, --- [.init]
 }
 
---- vim:ft=lua:ts=8:sw=4:et
+--- vim:ft=lua:ts=8:sw=4:et:tw=79