2016-01-07 13:40:00

11 files changed, 343 insertions, 170 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 9bfe09d64..bef916274 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2016.01.06 17:29}
+\newcontextversion{2016.01.07 13:38}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 3cc58dc20..5f938ae29 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 45b39c573..2a44f12ea 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -39,7 +39,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2016.01.06 17:29}
+\edef\contextversion{2016.01.07 13:38}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/font-dsp.lua b/tex/context/base/font-dsp.lua
index 93c961536..77ddea12b 100644
--- a/tex/context/base/font-dsp.lua
+++ b/tex/context/base/font-dsp.lua
@@ -1494,7 +1494,7 @@ do
                 -- this is expected in th efont handler (faster checking)
                 if flags[1] then flags[1] = "mark" end
                 if flags[2] then flags[2] = "ligature" end
-                if flags[3] then flags[2] = "base" end
+                if flags[3] then flags[3] = "base" end
                 --
                 local markclass    = lookup.markclass
              -- local chain        = lookup.chain
diff --git a/tex/context/base/font-otc.lua b/tex/context/base/font-otc.lua
index 21cf7fc7c..393dbbe87 100644
--- a/tex/context/base/font-otc.lua
+++ b/tex/context/base/font-otc.lua
@@ -22,26 +22,30 @@ local registerotffeature  = otf.features.register
 local setmetatableindex   = table.setmetatableindex
 
 local normalized = {
-    substitution = "substitution",
-    single       = "substitution",
-    ligature     = "ligature",
-    alternate    = "alternate",
-    multiple     = "multiple",
-    kern         = "kern",
+    substitution      = "substitution",
+    single            = "substitution",
+    ligature          = "ligature",
+    alternate         = "alternate",
+    multiple          = "multiple",
+    kern              = "kern",
+    chainsubstitution = "chainsubstitution",
+    chainposition     = "chainposition",
 }
 
 local types = {
-    substitution = "gsub_single",
-    ligature     = "gsub_ligature",
-    alternate    = "gsub_alternate",
-    multiple     = "gsub_multiple",
-    kern         = "gpos_pair",
+    substitution      = "gsub_single",
+    ligature          = "gsub_ligature",
+    alternate         = "gsub_alternate",
+    multiple          = "gsub_multiple",
+    kern              = "gpos_pair",
+    chainsubstitution = "gsub_contextchain",
+    chainposition     = "gpos_contextchain",
 }
 
 setmetatableindex(types, function(t,k) t[k] = k return k end) -- "key"
 
 local everywhere = { ["*"] = { ["*"] = true } } -- or: { ["*"] = { "*" } }
-local noflags    = { }
+local noflags    = { false, false, false, false }
 
 -- beware: shared, maybe we should copy the sequence
 
@@ -57,6 +61,9 @@ local function addfeature(data,feature,specifications)
     if gsubfeatures and gsubfeatures[feature] then
         return -- already present
     end
+
+    -- todo alse gpos
+
     local fontfeatures = resources.features or everywhere
     local unicodes     = resources.unicodes
     local splitter     = lpeg.splitter(" ",unicodes)
@@ -82,6 +89,246 @@ local function addfeature(data,feature,specifications)
     local stepkey      = coverup.stepkey
     local register     = coverup.register
 
+    local function prepare_substitution(list,featuretype)
+        local coverage = { }
+        local cover    = coveractions[featuretype]
+        for code, replacement in next, list do
+            local unicode     = tounicode(code)
+            local description = descriptions[unicode]
+            if description then
+                if type(replacement) == "table" then
+                    replacement = replacement[1]
+                end
+                replacement = tounicode(replacement)
+                if replacement and descriptions[replacement] then
+                    cover(coverage,unicode,replacement)
+                    done = done + 1
+                else
+                    skip = skip + 1
+                end
+            else
+                skip = skip + 1
+            end
+        end
+        return coverage
+    end
+
+    local function prepare_alternate(list,featuretype)
+        local coverage = { }
+        local cover    = coveractions[featuretype]
+        for code, replacement in next, list do
+            local unicode     = tounicode(code)
+            local description = descriptions[unicode]
+            if not description then
+                skip = skip + 1
+            elseif type(replacement) == "table" then
+                local r = { }
+                for i=1,#replacement do
+                    local u = tounicode(replacement[i])
+                    r[i] = descriptions[u] and u or unicode
+                end
+                cover(coverage,unicode,r)
+                done = done + 1
+            else
+                local u = tounicode(replacement)
+                if u then
+                    cover(coverage,unicode,{ u })
+                    done = done + 1
+                else
+                    skip = skip + 1
+                end
+            end
+        end
+        return coverage
+    end
+
+    local function prepare_multiple(list,featuretype)
+        local coverage = { }
+        local cover    = coveractions[featuretype]
+        for code, replacement in next, list do
+            local unicode     = tounicode(code)
+            local description = descriptions[unicode]
+            if not description then
+                skip = skip + 1
+            elseif type(replacement) == "table" then
+                local r, n = { }, 0
+                for i=1,#replacement do
+                    local u = tounicode(replacement[i])
+                    if descriptions[u] then
+                        n = n + 1
+                        r[n] = u
+                    end
+                end
+                if n > 0 then
+                    cover(coverage,unicode,r)
+                    done = done + 1
+                else
+                    skip = skip + 1
+                end
+            else
+                local u = tounicode(replacement)
+                if u then
+                    cover(coverage,unicode,{ u })
+                    done = done + 1
+                else
+                    skip = skip + 1
+                end
+            end
+        end
+        return coverage
+    end
+
+    local function prepare_ligature(list,featuretype)
+        local coverage = { }
+        local cover    = coveractions[featuretype]
+        for code, ligature in next, list do
+            local unicode     = tounicode(code)
+            local description = descriptions[unicode]
+            if description then
+                if type(ligature) == "string" then
+                    ligature = { lpegmatch(splitter,ligature) }
+                end
+                local present = true
+                for i=1,#ligature do
+                    local l = ligature[i]
+                    local u = tounicode(l)
+                    if descriptions[u] then
+                        ligature[i] = u
+                    else
+                        present = false
+                        break
+                    end
+                end
+                if present then
+                    cover(coverage,unicode,ligature)
+                    done = done + 1
+                else
+                    skip = skip + 1
+                end
+            else
+                skip = skip + 1
+            end
+        end
+        return coverage
+    end
+
+    local function prepare_kern(list,featuretype)
+        local coverage = { }
+        local cover    = coveractions[featuretype]
+        for code, replacement in next, list do
+            local unicode     = tounicode(code)
+            local description = descriptions[unicode]
+            if description and type(replacement) == "table" then
+                local r = { }
+                for k, v in next, replacement do
+                    local u = tounicode(k)
+                    if u then
+                        r[u] = v
+                    end
+                end
+                if next(r) then
+                    cover(coverage,unicode,r)
+                    done = done + 1
+                else
+                    skip = skip + 1
+                end
+            else
+                skip = skip + 1
+            end
+        end
+        return coverage, "kern"
+    end
+
+    local function prepare_chain(list,featuretype,sublookups)
+        -- todo: coveractions
+        local rules    = list.rules
+        local coverage = { }
+        if rules then
+            local rulehash     = { }
+            local rulesize     = 0
+            local sequence     = { }
+            local nofsequences = 0
+            local lookuptype   = types[featuretype]
+            for nofrules=1,#rules do
+                local rule         = rules[nofrules]
+                local current      = rule.current
+                local before       = rule.before
+                local after        = rule.after
+                local replacements = rule.replacements or false
+                local sequence     = { }
+                local nofsequences = 0
+                if before then
+                    for n=1,#before do
+                        nofsequences = nofsequences + 1
+                        sequence[nofsequences] = before[n]
+                    end
+                end
+                local start = nofsequences + 1
+                for n=1,#current do
+                    nofsequences = nofsequences + 1
+                    sequence[nofsequences] = current[n]
+                end
+                local stop = nofsequences
+                if after then
+                    for n=1,#after do
+                        nofsequences = nofsequences + 1
+                        sequence[nofsequences] = after[n]
+                    end
+                end
+                local lookups = rule.lookups or false
+                local subtype = nil
+                if lookups and sublookups then
+                    for k, v in next, lookups do
+                        local lookup = sublookups[v]
+                        if lookup then
+                            lookups[k] = lookup
+                            if not subtype then
+                                subtype = lookup.type
+                            end
+                        else
+                            -- already expanded
+                        end
+                    end
+                end
+                if nofsequences > 0 then -- we merge coverage into one
+                    -- we copy as we can have different fonts
+                    local hashed = { }
+                    for i=1,nofsequences do
+                        local t = { }
+                        local s = sequence[i]
+                        for i=1,#s do
+                            local u = tounicode(s[i])
+                            if u then
+                                t[u] = true
+                            end
+                        end
+                        hashed[i] = t
+                    end
+                    sequence = hashed
+                    -- now we create the rule
+                    rulesize = rulesize + 1
+                    rulehash[rulesize] = {
+                        nofrules,     -- 1
+                        lookuptype,   -- 2
+                        sequence,     -- 3
+                        start,        -- 4
+                        stop,         -- 5
+                        lookups,      -- 6 (6/7 also signal of what to do)
+                        replacements, -- 7
+                        subtype,      -- 8
+                    }
+                    for unic in next, sequence[start] do
+                        local cu = coverage[unic]
+                        if not cu then
+                            coverage[unic] = rulehash -- can now be done cleaner i think
+                        end
+                    end
+                end
+            end
+        end
+        return coverage
+    end
+
     for s=1,#specifications do
         local specification = specifications[s]
         local valid         = specification.valid
@@ -92,162 +339,85 @@ local function addfeature(data,feature,specifications)
                 specification.initialize = initialize(specification,data) and initialize or nil
             end
             local askedfeatures = specification.features or everywhere
-            local askedsteps    = specifications.steps or specification.subtables or { specification.data } or { }
+            local askedsteps    = specification.steps or specification.subtables or { specification.data } or { }
             local featuretype   = normalized[specification.type or "substitution"] or "substitution"
             local featureflags  = specification.flags or noflags
             local featureorder  = specification.order or { feature }
-            local added         = false
+            local featurechain  = (featuretype == "chainsubstitution" or featuretype == "chainposition") and 1 or 0
             local nofsteps      = 0
             local steps         = { }
+            local sublookups    = specification.lookups
+            if sublookups then
+                local s = { }
+                for i=1,#sublookups do
+                    local specification = sublookups[i]
+                    local askedsteps    = specification.steps or specification.subtables or { specification.data } or { }
+                    local featuretype   = normalized[specification.type or "substitution"] or "substitution"
+                    local featureflags  = specification.flags or noflags
+                    local nofsteps      = 0
+                    local steps         = { }
+                    for i=1,#askedsteps do
+                        local list     = askedsteps[i]
+                        local coverage = nil
+                        local format   = nil
+                        if featuretype == "substitution" then
+                            coverage, format = prepare_substitution(list,featuretype)
+                        elseif featuretype == "ligature" then
+                            coverage, format = prepare_ligature(list,featuretype)
+                        elseif featuretype == "alternate" then
+                            coverage, format = prepare_alternate(list,featuretype)
+                        elseif featuretype == "multiple" then
+                            coverage, format = prepare_multiple(list,featuretype)
+                        elseif featuretype == "kern" then
+                            coverage, format = prepare_kern(list,featuretype)
+                        end
+                        if coverage and next(coverage) then
+                            nofsteps = nofsteps + 1
+                            steps[nofsteps] = register(coverage,featuretype,format,feature,nofsteps,descriptions,resources)
+                        end
+                    end
+                    s[i] = {
+                        [stepkey] = steps,
+                        nofsteps  = nofsteps,
+                        type      = types[featuretype],
+                    }
+                end
+                sublookups = s
+            end
             for i=1,#askedsteps do
                 local list     = askedsteps[i]
-                local coverage = { }
-                local cover    = coveractions[featuretype]
+                local coverage = nil
                 local format   = nil
-                if not cover then
-                    -- unknown
-                elseif featuretype == "substitution" then
-                    for code, replacement in next, list do
-                        local unicode     = tounicode(code)
-                        local description = descriptions[unicode]
-                        if description then
-                            if type(replacement) == "table" then
-                                replacement = replacement[1]
-                            end
-                            replacement = tounicode(replacement)
-                            if replacement and descriptions[replacement] then
-                                cover(coverage,unicode,replacement)
-                                done = done + 1
-                            else
-                                skip = skip + 1
-                            end
-                        else
-                            skip = skip + 1
-                        end
-                    end
+                if featuretype == "substitution" then
+                    coverage, format = prepare_substitution(list,featuretype)
                 elseif featuretype == "ligature" then
-                    for code, ligature in next, list do
-                        local unicode     = tounicode(code)
-                        local description = descriptions[unicode]
-                        if description then
-                            if type(ligature) == "string" then
-                                ligature = { lpegmatch(splitter,ligature) }
-                            end
-                            local present = true
-                            for i=1,#ligature do
-                                local l = ligature[i]
-                                local u = tounicode(l)
-                                if descriptions[u] then
-                                    ligature[i] = u
-                                else
-                                    present = false
-                                    break
-                                end
-                            end
-                            if present then
-                                cover(coverage,unicode,ligature)
-                                done = done + 1
-                            else
-                                skip = skip + 1
-                            end
-                        else
-                            skip = skip + 1
-                        end
-                    end
+                    coverage, format = prepare_ligature(list,featuretype)
                 elseif featuretype == "alternate" then
-                    for code, replacement in next, list do
-                        local unicode     = tounicode(code)
-                        local description = descriptions[unicode]
-                        if not description then
-                            skip = skip + 1
-                        elseif type(replacement) == "table" then
-                            local r = { }
-                            for i=1,#replacement do
-                                local u = tounicode(replacement[i])
-                                r[i] = descriptions[u] and u or unicode
-                            end
-                            cover(coverage,unicode,r)
-                            done = done + 1
-                        else
-                            local u = tounicode(replacement)
-                            if u then
-                                cover(coverage,unicode,{ u })
-                                done = done + 1
-                            else
-                                skip = skip + 1
-                            end
-                        end
-                    end
-                elseif featuretype == "multiple" then -- todo: unicode can be table
-                    for code, replacement in next, list do
-                        local unicode     = tounicode(code)
-                        local description = descriptions[unicode]
-                        if not description then
-                            skip = skip + 1
-                        elseif type(replacement) == "table" then
-                            local r, n = { }, 0
-                            for i=1,#replacement do
-                                local u = tounicode(replacement[i])
-                                if descriptions[u] then
-                                    n = n + 1
-                                    r[n] = u
-                                end
-                            end
-                            if n > 0 then
-                                cover(coverage,unicode,r)
-                                done = done + 1
-                            else
-                                skip = skip + 1
-                            end
-                        else
-                            local u = tounicode(replacement)
-                            if u then
-                                cover(coverage,unicode,{ u })
-                                done = done + 1
-                            else
-                                skip = skip + 1
-                            end
-                        end
-                    end
+                    coverage, format = prepare_alternate(list,featuretype)
+                elseif featuretype == "multiple" then
+                    coverage, format = prepare_multiple(list,featuretype)
                 elseif featuretype == "kern" then
-                    for code, replacement in next, list do
-                        local unicode     = tounicode(code)
-                        local description = descriptions[unicode]
-                        if description and type(replacement) == "table" then
-                            local r = { }
-                            for k, v in next, replacement do
-                                local u = tounicode(k)
-                                if u then
-                                    r[u] = v
-                                end
-                            end
-                            if next(r) then
-                                cover(coverage,unicode,r)
-                                done = done + 1
-                            else
-                                skip = skip + 1
-                            end
-                        else
-                            skip = skip + 1
-                        end
-                    end
-                    format = "kern"
+                    coverage, format = prepare_kern(list,featuretype)
+                elseif featuretype == "chainsubstitution" or featuretype == "chainposition" then
+                    coverage, format = prepare_chain(list,featuretype,sublookups)
                 end
-                if next(coverage) then
-                    added = true
+                if coverage and next(coverage) then
                     nofsteps = nofsteps + 1
                     steps[nofsteps] = register(coverage,featuretype,format,feature,nofsteps,descriptions,resources)
                 end
             end
-            if added then
+            if nofsteps > 0 then
                 -- script = { lang1, lang2, lang3 } or script = { lang1 = true, ... }
                 for k, v in next, askedfeatures do
                     if v[1] then
                         askedfeatures[k] = tohash(v)
                     end
                 end
+                if featureflags[1] then featureflags[1] = "mark" end
+                if featureflags[2] then featureflags[2] = "ligature" end
+                if featureflags[3] then featureflags[3] = "base" end
                 local sequence = {
-                    chain     = 0,
+                    chain     = featurechain,
                     features  = { [feature] = askedfeatures },
                     flags     = featureflags,
                     name      = feature, -- not needed
diff --git a/tex/context/base/font-otf.lua b/tex/context/base/font-otf.lua
index 4967558eb..e90ec738f 100644
--- a/tex/context/base/font-otf.lua
+++ b/tex/context/base/font-otf.lua
@@ -2950,11 +2950,13 @@ end
 otf.coverup = {
     stepkey = "subtables",
     actions = {
-        substitution = justset,
-        alternate    = justset,
-        multiple     = justset,
-        ligature     = justset,
-        kern         = justset,
+        substitution      = justset,
+        alternate         = justset,
+        multiple          = justset,
+        ligature          = justset,
+        kern              = justset,
+        chainsubstitution = justset,
+        chainposition     = justset,
     },
     register = function(coverage,lookuptype,format,feature,n,descriptions,resources)
         local name = formatters["ctx_%s_%s_%s"](feature,lookuptype,n) -- we can have a mix of types
diff --git a/tex/context/base/font-otl.lua b/tex/context/base/font-otl.lua
index 30a351973..cc6befa9d 100644
--- a/tex/context/base/font-otl.lua
+++ b/tex/context/base/font-otl.lua
@@ -803,12 +803,14 @@ end
 otf.coverup = {
     stepkey = "steps",
     actions = {
-        substitution = justset,
-        alternate    = justset,
-        multiple     = justset,
-        kern         = justset,
-        pair         = justset,
-        ligature     = function(coverage,unicode,ligature)
+        chainsubstitution = justset,
+        chainposition     = justset,
+        substitution      = justset,
+        alternate         = justset,
+        multiple          = justset,
+        kern              = justset,
+        pair              = justset,
+        ligature          = function(coverage,unicode,ligature)
             local first = ligature[1]
             local tree  = coverage[first]
             if not tree then
diff --git a/tex/context/base/font-ots.lua b/tex/context/base/font-ots.lua
index 0f457a834..207d14650 100644
--- a/tex/context/base/font-ots.lua
+++ b/tex/context/base/font-ots.lua
@@ -1797,7 +1797,7 @@ end
 -- elseif char == zwnj and sequence[n][32] then -- brrr
 
 local function show_skip(dataset,sequence,char,ck,class)
-    logwarning("%s: skipping char %s, class %a, rule %a, lookuptype %a",cref(dataset,sequence),gref(char),class,ck[1],ck[8])
+    logwarning("%s: skipping char %s, class %a, rule %a, lookuptype %a",cref(dataset,sequence),gref(char),class,ck[1],ck[8] or ck[2])
 end
 
 -- A previous version had disc collapsing code in the (single sub) handler plus some
@@ -2169,12 +2169,14 @@ end
 --     end
 -- end
 
+local noflags = { false, false, false, false }
+
 local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
     local sweepnode    = sweepnode
     local sweeptype    = sweeptype
     local diskseen     = false
     local checkdisc    = getprev(head)
-    local flags        = sequence.flags
+    local flags        = sequence.flags or noflags
     local done         = false
     local skipmark     = flags[1]
     local skipligature = flags[2]
@@ -2581,7 +2583,7 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
             local diskchain = diskseen or sweepnode
             if trace_contexts then
                 local rule       = ck[1]
-                local lookuptype = ck[8]
+                local lookuptype = ck[8] or ck[2]
                 local first      = ck[4]
                 local last       = ck[5]
                 local char       = getchar(start)
diff --git a/tex/context/base/font-oup.lua b/tex/context/base/font-oup.lua
index 64d658d6a..004e487c5 100644
--- a/tex/context/base/font-oup.lua
+++ b/tex/context/base/font-oup.lua
@@ -1904,7 +1904,7 @@ function readers.expand(data)
             -- we also need to do sublookups
             for i=1,#sequences do
                 local sequence = sequences[i]
-                local steps = sequence.steps
+                local steps    = sequence.steps
                 if steps then
                     local kind = sequence.type
                     local markclass = sequence.markclass
@@ -1922,8 +1922,7 @@ function readers.expand(data)
                         if baseclasses then
                             local coverage = step.coverage
                             for k, v in next, coverage do
-    --                             v[1] = baseclasses[v[2]] -- slot 1 is a placeholder
-                                v[1] = baseclasses[v[1]]
+                                v[1] = baseclasses[v[1]] -- slot 1 is a placeholder
                             end
                         elseif kind == "gpos_cursive" then
                             local coverage = step.coverage
@@ -1943,7 +1942,7 @@ function readers.expand(data)
                                 local current      = rule.current
                                 local before       = rule.before
                                 local after        = rule.after
-                                local replacements = rule.replacements
+                                local replacements = rule.replacements or false
                                 local sequence     = { }
                                 local nofsequences = 0
                                 if before then
@@ -1964,7 +1963,7 @@ function readers.expand(data)
                                         sequence[nofsequences] = after[n]
                                     end
                                 end
-                                local lookups = rule.lookups
+                                local lookups = rule.lookups or false
                                 local subtype = nil
                                 if lookups then
                                     for k, v in next, lookups do
@@ -1987,7 +1986,7 @@ function readers.expand(data)
                                         sequence,     -- 3
                                         start,        -- 4
                                         stop,         -- 5
-                                        rule.lookups, -- 6
+                                        lookups,      -- 6 (6/7 also signal of what to do)
                                         replacements, -- 7
                                         subtype,      -- 8
                                     }
@@ -1995,8 +1994,6 @@ function readers.expand(data)
                                         local cu = coverage[unic]
                                         if not cu then
                                             coverage[unic] = rulehash -- can now be done cleaner i think
--- else
---     sequence[start] = nil
                                         end
                                     end
                                 end
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index d08186367..4bab1ac7f 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index b28c97183..871897ac1 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf