[fontloader] sync with Context as of 2016-10-17

6 files changed, 269 insertions, 48 deletions

diff --git a/src/fontloader/misc/fontloader-font-con.lua b/src/fontloader/misc/fontloader-font-con.lua
index b3e506b..18f2217 100644
--- a/src/fontloader/misc/fontloader-font-con.lua
+++ b/src/fontloader/misc/fontloader-font-con.lua
@@ -447,12 +447,16 @@ function constructors.scale(tfmdata,specification)
     local haskerns         = properties.haskerns     or properties.mode == "base" -- we can have afm in node mode
     local hasligatures     = properties.hasligatures or properties.mode == "base" -- we can have afm in node mode
     local realdimensions   = properties.realdimensions
+    local writingmode      = properties.writingmode or "horizontal"
+    local identity         = properties.identity or "horizontal"
     --
     if changed and not next(changed) then
         changed = false
     end
     --
-    target.type = isvirtual and "virtual" or "real"
+    target.type        = isvirtual and "virtual" or "real"
+    target.writingmode = writingmode == "vertical" and "vertical" or "horizontal"
+    target.identity    = identity == "vertical" and "vertical" or "horizontal"
     --
     target.postprocessors = tfmdata.postprocessors
     --
@@ -894,6 +898,8 @@ function constructors.finalize(tfmdata)
             cidinfo       = tfmdata.cidinfo       or nil,
             format        = tfmdata.format        or "type1",
             direction     = tfmdata.direction     or 0,
+            writingmode   = tfmdata.writingmode   or "horizontal",
+            identity      = tfmdata.identity      or "horizontal",
         }
     end
     if not tfmdata.resources then
@@ -1091,7 +1097,11 @@ end)
 
 do
 
-    local function setindeed(mode,target,group,name,action,position)
+    local function setindeed(mode,source,target,group,name,position)
+        local action = source[mode]
+        if not action then
+            return
+        end
         local t = target[mode]
         if not t then
             report_defining("fatal error in setting feature %a, group %a, mode %a",name,group,mode)
@@ -1122,15 +1132,10 @@ do
             report_defining("fatal source error in setting feature %a, group %a",name,group)
             os.exit()
         end
-        local node     = source.node
-        local base     = source.base
         local position = source.position
-        if node then
-            setindeed("node",target,group,name,node,position)
-        end
-        if base then
-            setindeed("base",target,group,name,base,position)
-        end
+        setindeed("node",source,target,group,name,position)
+        setindeed("base",source,target,group,name,position)
+        setindeed("plug",source,target,group,name,position)
     end
 
     local function register(where,specification)
@@ -1197,9 +1202,9 @@ do
                 defaults     = { },
                 descriptions = tables and tables.features or { },
                 used         = statistics and statistics.usedfeatures or { },
-                initializers = { base = { }, node = { } },
-                processors   = { base = { }, node = { } },
-                manipulators = { base = { }, node = { } },
+                initializers = { base = { }, node = { }, plug = { } },
+                processors   = { base = { }, node = { }, plug = { } },
+                manipulators = { base = { }, node = { }, plug = { } },
             }
             features.register = function(specification) return register(features,specification) end
             handler.features = features -- will also become hidden
diff --git a/src/fontloader/misc/fontloader-font-oti.lua b/src/fontloader/misc/fontloader-font-oti.lua
index 5e812bb..e10a261 100644
--- a/src/fontloader/misc/fontloader-font-oti.lua
+++ b/src/fontloader/misc/fontloader-font-oti.lua
@@ -72,6 +72,7 @@ registerotffeature {
     initializers = {
         base = setmode,
         node = setmode,
+        plug = setmode,
     }
 }
 
@@ -81,6 +82,7 @@ registerotffeature {
     initializers = {
         base = setlanguage,
         node = setlanguage,
+        plug = setlanguage,
     }
 }
 
@@ -90,6 +92,7 @@ registerotffeature {
     initializers = {
         base = setscript,
         node = setscript,
+        plug = setscript,
     }
 }
 
@@ -157,4 +160,3 @@ function otffeatures.checkeddefaultlanguage(featuretype,autolanguage,languages)
         end
     end
 end
-
diff --git a/src/fontloader/misc/fontloader-font-otl.lua b/src/fontloader/misc/fontloader-font-otl.lua
index cf6603f..d4eaed7 100644
--- a/src/fontloader/misc/fontloader-font-otl.lua
+++ b/src/fontloader/misc/fontloader-font-otl.lua
@@ -675,7 +675,7 @@ local function getgsub(tfmdata,k,kind,value)
             local properties = tfmdata.properties
             local validlookups, lookuplist = otf.collectlookups(rawdata,kind,properties.script,properties.language)
             if validlookups then
-                local choice = tonumber(value) or 1 -- no random here (yet)
+             -- local choice = tonumber(value) or 1 -- no random here (yet)
                 for i=1,#lookuplist do
                     local lookup   = lookuplist[i]
                     local steps    = lookup.steps
diff --git a/src/fontloader/misc/fontloader-font-otr.lua b/src/fontloader/misc/fontloader-font-otr.lua
index 9cdbc3d..2b18c1e 100644
--- a/src/fontloader/misc/fontloader-font-otr.lua
+++ b/src/fontloader/misc/fontloader-font-otr.lua
@@ -975,7 +975,7 @@ readers.head = function(f,fontdata)
 end
 
 -- This table is a rather simple one. No treatment of values is needed here. Most
--- variables are not used but nofhmetrics is quite important.
+-- variables are not used but nofmetrics is quite important.
 
 readers.hhea = function(f,fontdata,specification)
     if specification.details then
@@ -983,7 +983,7 @@ readers.hhea = function(f,fontdata,specification)
         if datatable then
             setposition(f,datatable.offset)
             fontdata.horizontalheader = {
-                version             = readfixed(f),
+                version             = readfixed(f), -- two ushorts: major minor
                 ascender            = readfword(f),
                 descender           = readfword(f),
                 linegap             = readfword(f),
@@ -999,11 +999,45 @@ readers.hhea = function(f,fontdata,specification)
                 reserved_3          = readshort(f),
                 reserved_4          = readshort(f),
                 metricdataformat    = readshort(f),
-                nofhmetrics         = readushort(f),
+                nofmetrics          = readushort(f),
             }
         else
             fontdata.horizontalheader = {
-                nofhmetrics = 0,
+                nofmetrics = 0,
+            }
+        end
+    end
+end
+
+readers.vhea = function(f,fontdata,specification)
+    if specification.details then
+        local datatable = fontdata.tables.vhea
+        if datatable then
+            setposition(f,datatable.offset)
+            local version = readfixed(f)
+            fontdata.verticalheader = {
+                version              = version,
+                ascender             = readfword(f),
+                descender            = readfword(f),
+                linegap              = readfword(f),
+                maxadvanceheight     = readufword(f),
+                mintopsidebearing    = readfword(f),
+                minbottomsidebearing = readfword(f),
+                maxextent            = readfword(f),
+                caretsloperise       = readshort(f),
+                caretsloperun        = readshort(f),
+                caretoffset          = readshort(f),
+                reserved_1           = readshort(f),
+                reserved_2           = readshort(f),
+                reserved_3           = readshort(f),
+                reserved_4           = readshort(f),
+                metricdataformat     = readshort(f),
+                nofmetrics           = readushort(f),
+            }
+--             inspect(fontdata.verticalheader)
+        else
+            fontdata.verticalheader = {
+                nofmetrics = 0,
             }
         end
     end
@@ -1064,11 +1098,12 @@ readers.hmtx = function(f,fontdata,specification)
         local datatable = fontdata.tables.hmtx
         if datatable then
             setposition(f,datatable.offset)
-            local nofmetrics      = fontdata.horizontalheader.nofhmetrics
-            local glyphs          = fontdata.glyphs
-            local nofglyphs       = fontdata.nofglyphs
-            local width           = 0 -- advance
-            local leftsidebearing = 0
+            local horizontalheader = fontdata.horizontalheader
+            local nofmetrics       = horizontalheader.nofmetrics
+            local glyphs           = fontdata.glyphs
+            local nofglyphs        = fontdata.nofglyphs
+            local width            = 0 -- advance
+            local leftsidebearing  = 0
             for i=0,nofmetrics-1 do
                 local glyph     = glyphs[i]
                 width           = readshort(f)
@@ -1095,6 +1130,53 @@ readers.hmtx = function(f,fontdata,specification)
     end
 end
 
+readers.vmtx = function(f,fontdata,specification)
+    if specification.glyphs then
+        local datatable = fontdata.tables.vmtx
+        if datatable then
+            setposition(f,datatable.offset)
+            local verticalheader = fontdata.verticalheader
+            local nofmetrics     = verticalheader.nofmetrics
+            local glyphs         = fontdata.glyphs
+            local nofglyphs      = fontdata.nofglyphs
+            local vheight        = 0
+            local vdefault       = verticalheader.ascender + verticalheader.descender
+            local topsidebearing = 0
+            for i=0,nofmetrics-1 do
+                local glyph     = glyphs[i]
+                vheight         = readshort(f)
+                topsidebearing  = readshort(f)
+                if vheight ~= 0 and vheight ~= vdefault then
+                    glyph.vheight = vheight
+                end
+             -- if topsidebearing ~= 0 then
+             --     glyph.tsb = topsidebearing
+             -- end
+            end
+            -- The next can happen in for instance a monospace font or in a cjk font
+            -- with fixed heights.
+            for i=nofmetrics,nofglyphs-1 do
+                local glyph = glyphs[i]
+                if vheight ~= 0 and vheight ~= vdefault then
+                    glyph.vheight = vheight
+                end
+             -- if topsidebearing ~= 0 then
+             --     glyph.tsb = topsidebearing
+             -- end
+            end
+        end
+    end
+end
+
+readers.vorg = function(f,fontdata,specification)
+    if specification.glyphs then
+        local datatable = fontdata.tables.vorg
+        if datatable then
+            report("todo: %s","vorg")
+        end
+    end
+end
+
 -- The post table relates to postscript (printing) but has some relevant properties for other
 -- usage as well. We just use the names from the microsoft specification. The version 2.0
 -- description is somewhat fuzzy but it is a hybrid with overloads.
@@ -1751,7 +1833,7 @@ end
 -- some properties in order to read following tables. When details is true we also
 -- initialize the glyphs data.
 
-local function getinfo(maindata,sub,platformnames,rawfamilynames)
+local function getinfo(maindata,sub,platformnames,rawfamilynames,metricstoo)
     local fontdata = sub and maindata.subfonts and maindata.subfonts[sub] or maindata
     local names    = fontdata.names
     local info     = nil
@@ -1808,6 +1890,29 @@ local function getinfo(maindata,sub,platformnames,rawfamilynames)
             descender      = metrics.typodescender,
             platformnames  = platformnames and fontdata.platformnames or nil,
         }
+        if metricstoo then
+            local keys = {
+                "version",
+                "ascender", "descender", "linegap",
+             -- "caretoffset", "caretsloperise", "caretsloperun",
+                "maxadvancewidth", "maxadvanceheight", "maxextent",
+             -- "metricdataformat",
+                "minbottomsidebearing", "mintopsidebearing",
+            }
+            local h = fontdata.horizontalheader or { }
+            local v = fontdata.verticalheader   or { }
+            if h then
+                local th = { }
+                local tv = { }
+                for i=1,#keys do
+                    local key = keys[i]
+                    th[key] = h[key] or 0
+                    tv[key] = v[key] or 0
+                end
+                info.horizontalmetrics = th
+                info.verticalmetrics   = tv
+            end
+        end
     elseif n then
         info = {
             filename = fontdata.filename,
@@ -1900,7 +2005,10 @@ local function readdata(f,offset,specification)
     readers["head"](f,fontdata,specification)
     readers["maxp"](f,fontdata,specification)
     readers["hhea"](f,fontdata,specification)
+    readers["vhea"](f,fontdata,specification)
     readers["hmtx"](f,fontdata,specification)
+    readers["vmtx"](f,fontdata,specification)
+    readers["vorg"](f,fontdata,specification)
     readers["post"](f,fontdata,specification)
     readers["cff" ](f,fontdata,specification)
     readers["cmap"](f,fontdata,specification)
@@ -2084,7 +2192,7 @@ function readers.loadfont(filename,n)
             descriptions  = fontdata.descriptions,
             format        = fontdata.format,
             goodies       = { },
-            metadata      = getinfo(fontdata,n), -- no platformnames here !
+            metadata      = getinfo(fontdata,n,false,false,true), -- no platformnames here !
             properties    = {
                 hasitalics    = fontdata.hasitalics or false,
                 maxcolorclass = fontdata.maxcolorclass,
diff --git a/src/fontloader/misc/fontloader-font-ots.lua b/src/fontloader/misc/fontloader-font-ots.lua
index 1c64720..17e1a3c 100644
--- a/src/fontloader/misc/fontloader-font-ots.lua
+++ b/src/fontloader/misc/fontloader-font-ots.lua
@@ -128,6 +128,7 @@ local trace_details      = false  registertracker("otf.details",      function(v
 local trace_steps        = false  registertracker("otf.steps",        function(v) trace_steps        = v end)
 local trace_skips        = false  registertracker("otf.skips",        function(v) trace_skips        = v end)
 local trace_directions   = false  registertracker("otf.directions",   function(v) trace_directions   = v end)
+local trace_plugins      = false  registertracker("otf.plugins",      function(v) trace_plugins      = v end)
 
 local trace_kernruns     = false  registertracker("otf.kernruns",     function(v) trace_kernruns     = v end)
 local trace_discruns     = false  registertracker("otf.discruns",     function(v) trace_discruns     = v end)
@@ -136,6 +137,7 @@ local trace_testruns     = false  registertracker("otf.testruns",     function(v
 
 ----- zwnjruns           = true   registerdirective("otf.zwnjruns",     function(v) zwnjruns = v end)
 local optimizekerns      = true
+local alwaysdisc         = true   registerdirective("otf.alwaysdisc", function(v) alwaysdisc = v end)
 
 local report_direct   = logs.reporter("fonts","otf direct")
 local report_subchain = logs.reporter("fonts","otf subchain")
@@ -599,7 +601,7 @@ local function toligature(head,start,stop,char,dataset,sequence,markflag,discfou
                 setlink(discfound,next)
                 setboth(base)
                 setfield(base,"components",copied)
-                setdisc(discfound,pre,post,base,discretionary_code)
+                setdisc(discfound,pre,post,base) -- was discretionary_code
                 base = prev -- restart
             end
         end
@@ -3047,7 +3049,11 @@ end
 -- -- local a = getattr(start,0)
 -- -- if not a or (a == attr) then
 --
--- and even that one is probably not needed.
+-- and even that one is probably not needed. However, we can handle interesting
+-- cases now:
+--
+--  1{2{\oldstyle\discretionary{3}{4}{5}}6}7\par
+--  1{2\discretionary{3{\oldstyle3}}{{\oldstyle4}4}{5{\oldstyle5}5}6}7\par
 
 local nesting = 0
 
@@ -3077,6 +3083,7 @@ local function c_run_single(head,font,attr,lookupcache,step,dataset,sequence,rlm
                     start = getnext(start)
                 end
             else
+                -- go on can be a mixed one
                 start = getnext(start)
             end
         elseif char == false then
@@ -3122,6 +3129,8 @@ local function t_run_single(start,stop,font,attr,lookupcache)
                         return true, d > 1
                     end
                 end
+            else
+                -- go on can be a mixed one
             end
             start = getnext(start)
         else
@@ -3207,6 +3216,7 @@ local function c_run_multiple(head,font,attr,steps,nofsteps,dataset,sequence,rlm
                     start = getnext(start)
                 end
             else
+                -- go on can be a mixed one
                 start = getnext(start)
             end
         elseif char == false then
@@ -3261,6 +3271,8 @@ local function t_run_multiple(start,stop,font,attr,steps,nofsteps)
                         report_missing_coverage(dataset,sequence)
                     end
                 end
+            else
+                -- go on can be a mixed one
             end
             start = getnext(start)
         else
@@ -3324,7 +3336,7 @@ local function k_run_multiple(sub,injection,last,font,attr,steps,nofsteps,datase
     end
 end
 
--- to be checkedL nowadays we probably can assume properly matched directions
+-- to be checked, nowadays we probably can assume properly matched directions
 -- so maybe we no longer need a stack
 
 local function txtdirstate(start,stack,top,rlparmode)
@@ -3365,6 +3377,10 @@ local function pardirstate(start)
     return getnext(start), new, new
 end
 
+otf.helpers             = otf.helpers or { }
+otf.helpers.txtdirstate = txtdirstate
+otf.helpers.pardirstate = pardirstate
+
 local function featuresprocessor(head,font,attr)
 
     local sequences = sequencelists[font] -- temp hack
@@ -3413,6 +3429,8 @@ local function featuresprocessor(head,font,attr)
     local done      = false
     local datasets  = otf.dataset(tfmdata,font,attr)
 
+    local forcedisc = alwaysdisc or not attr
+
     local dirstack  = { } -- could move outside function but we can have local runs
 
     sweephead       = { }
@@ -3425,7 +3443,7 @@ local function featuresprocessor(head,font,attr)
     -- Keeping track of the headnode is needed for devanagari (I generalized it a bit
     -- so that multiple cases are also covered.)
 
-    -- We don't goto the next node of a disc node is created so that we can then treat
+    -- We don't goto the next node when a disc node is created so that we can then treat
     -- the pre, post and replace. It's a bit of a hack but works out ok for most cases.
 
     for s=1,#datasets do
@@ -3527,16 +3545,26 @@ local function featuresprocessor(head,font,attr)
                            -- whatever glyph
                            start = getnext(start)
                         elseif id == disc_code then
-                            local ok
-                            if gpossing then
-                                start, ok = kernrun(start,k_run_single,             font,attr,lookupcache,step,dataset,sequence,rlmode,handler)
-                            elseif typ == "gsub_ligature" then
-                                start, ok = testrun(start,t_run_single,c_run_single,font,attr,lookupcache,step,dataset,sequence,rlmode,handler)
+                            -- ctx: we could assume the same attr as the surrounding glyphs but then we loose
+                            -- the option to have interesting disc nodes (we gain upto 10% on extreme tests);
+                            -- if a disc would have a font field we could even be more strict (see oldstyle test
+                            -- case)
+                            local a = forcedisc or getsubtype(start) == discretionary_code or getattr(start,0) == attr
+                            if a then
+                                --     local attr = false
+                                local ok
+                                if gpossing then
+                                    start, ok = kernrun(start,k_run_single,             font,attr,lookupcache,step,dataset,sequence,rlmode,handler)
+                                elseif typ == "gsub_ligature" then
+                                    start, ok = testrun(start,t_run_single,c_run_single,font,attr,lookupcache,step,dataset,sequence,rlmode,handler)
+                                else
+                                    start, ok = comprun(start,c_run_single,             font,attr,lookupcache,step,dataset,sequence,rlmode,handler)
+                                end
+                                if ok then
+                                    success = true
+                                end
                             else
-                                start, ok = comprun(start,c_run_single,             font,attr,lookupcache,step,dataset,sequence,rlmode,handler)
-                            end
-                            if ok then
-                                success = true
+                                start = getnext(start)
                             end
                         elseif id == math_code then
                             start = getnext(end_of_math(start))
@@ -3596,16 +3624,22 @@ local function featuresprocessor(head,font,attr)
                     elseif char == false then
                         start = getnext(start)
                     elseif id == disc_code then
-                        local ok
-                        if gpossing then
-                            start, ok = kernrun(start,k_run_multiple,               font,attr,steps,nofsteps,dataset,sequence,rlmode,handler)
-                        elseif typ == "gsub_ligature" then
-                            start, ok = testrun(start,t_run_multiple,c_run_multiple,font,attr,steps,nofsteps,dataset,sequence,rlmode,handler)
+                        -- see comment above
+                        local a = forcedisc or getsubtype(start) == discretionary_code or getattr(start,0) == attr
+                        if a then
+                            local ok
+                            if gpossing then
+                                start, ok = kernrun(start,k_run_multiple,               font,attr,steps,nofsteps,dataset,sequence,rlmode,handler)
+                            elseif typ == "gsub_ligature" then
+                                start, ok = testrun(start,t_run_multiple,c_run_multiple,font,attr,steps,nofsteps,dataset,sequence,rlmode,handler)
+                            else
+                                start, ok = comprun(start,c_run_multiple,               font,attr,steps,nofsteps,dataset,sequence,rlmode,handler)
+                            end
+                            if ok then
+                                success = true
+                            end
                         else
-                            start, ok = comprun(start,c_run_multiple,               font,attr,steps,nofsteps,dataset,sequence,rlmode,handler)
-                        end
-                        if ok then
-                            success = true
+                            start = getnext(start)
                         end
                     elseif id == math_code then
                         start = getnext(end_of_math(start))
@@ -3637,6 +3671,34 @@ end
 
 -- so far
 
+local plugins = { }
+otf.plugins   = plugins
+
+function otf.registerplugin(name,f)
+    if type(name) == "string" and type(f) == "function" then
+        plugins[name] = { name, f }
+    end
+end
+
+local function plugininitializer(tfmdata,value)
+    if type(value) == "string" then
+        tfmdata.shared.plugin = plugins[value]
+    end
+end
+
+local function pluginprocessor(head,font)
+    local s = fontdata[font].shared
+    local p = s and s.plugin
+    if p then
+        if trace_plugins then
+            report_process("applying plugin %a",p[1])
+        end
+        return p[2](head,font)
+    else
+        return head, false
+    end
+end
+
 local function featuresinitializer(tfmdata,value)
     -- nothing done here any more
 end
@@ -3648,9 +3710,11 @@ registerotffeature {
     initializers = {
         position = 1,
         node     = featuresinitializer,
+        plug     = plugininitializer,
     },
     processors   = {
         node     = featuresprocessor,
+        plug     = pluginprocessor,
     }
 }
 
diff --git a/src/fontloader/misc/fontloader-util-fil.lua b/src/fontloader/misc/fontloader-util-fil.lua
index eeb6856..eb054a5 100644
--- a/src/fontloader/misc/fontloader-util-fil.lua
+++ b/src/fontloader/misc/fontloader-util-fil.lua
@@ -109,6 +109,10 @@ function files.readcardinal2(f)
     local a, b = byte(f:read(2),1,2)
     return 0x100 * a + b
 end
+function files.readcardinal2le(f)
+    local b, a = byte(f:read(2),1,2)
+    return 0x100 * a + b
+end
 
 function files.readinteger2(f)
     local a, b = byte(f:read(2),1,2)
@@ -120,11 +124,25 @@ function files.readinteger2(f)
         return n
     end
 end
+function files.readinteger2le(f)
+    local b, a = byte(f:read(2),1,2)
+    local n = 0x100 * a + b
+    if n >= 0x8000 then
+     -- return n - 0xFFFF - 1
+        return n - 0x10000
+    else
+        return n
+    end
+end
 
 function files.readcardinal3(f)
     local a, b, c = byte(f:read(3),1,3)
     return 0x10000 * a + 0x100 * b + c
 end
+function files.readcardinal3le(f)
+    local c, b, a = byte(f:read(3),1,3)
+    return 0x10000 * a + 0x100 * b + c
+end
 
 function files.readinteger3(f)
     local a, b, c = byte(f:read(3),1,3)
@@ -136,11 +154,25 @@ function files.readinteger3(f)
         return n
     end
 end
+function files.readinteger3le(f)
+    local c, b, a = byte(f:read(3),1,3)
+    local n = 0x10000 * a + 0x100 * b + c
+    if n >= 0x80000 then
+     -- return n - 0xFFFFFF - 1
+        return n - 0x1000000
+    else
+        return n
+    end
+end
 
 function files.readcardinal4(f)
     local a, b, c, d = byte(f:read(4),1,4)
     return 0x1000000 * a + 0x10000 * b + 0x100 * c + d
 end
+function files.readcardinal4le(f)
+    local d, c, b, a = byte(f:read(4),1,4)
+    return 0x1000000 * a + 0x10000 * b + 0x100 * c + d
+end
 
 function files.readinteger4(f)
     local a, b, c, d = byte(f:read(4),1,4)
@@ -152,6 +184,16 @@ function files.readinteger4(f)
         return n
     end
 end
+function files.readinteger4le(f)
+    local d, c, b, a = byte(f:read(4),1,4)
+    local n = 0x1000000 * a + 0x10000 * b + 0x100 * c + d
+    if n >= 0x8000000 then
+     -- return n - 0xFFFFFFFF - 1
+        return n - 0x100000000
+    else
+        return n
+    end
+end
 
 function files.readfixed4(f)
     local a, b, c, d = byte(f:read(4),1,4)