From f9551155d56f6fb9975360d6d15ec06b08129358 Mon Sep 17 00:00:00 2001
From: Marius <mariausol@gmail.com>
Date: Fri, 5 Oct 2012 22:00:13 +0300
Subject: beta 2012.10.05 19:52

---
 tex/context/base/cont-new.mkii                     |   2 +-
 tex/context/base/cont-new.mkiv                     |   2 +-
 tex/context/base/context-version.pdf               | Bin 4143 -> 4142 bytes
 tex/context/base/context-version.png               | Bin 104331 -> 105863 bytes
 tex/context/base/context.mkii                      |   2 +-
 tex/context/base/context.mkiv                      |   2 +-
 tex/context/base/font-ctx.lua                      |  88 +++++++
 tex/context/base/font-ota.lua                      |  58 ++---
 tex/context/base/font-otd.lua                      |  22 +-
 tex/context/base/font-otn.lua                      | 206 +++++++++++----
 tex/context/base/font-ott.lua                      |   8 +-
 tex/context/base/font-pre.mkiv                     |  12 +-
 tex/context/base/font-tra.mkiv                     |   6 +-
 tex/context/base/l-table.lua                       |   3 +-
 tex/context/base/math-arr.mkiv                     |   8 +
 tex/context/base/node-tra.lua                      |  52 ++--
 tex/context/base/pack-com.mkiv                     |   4 +-
 tex/context/base/status-files.pdf                  | Bin 24528 -> 24573 bytes
 tex/context/base/status-lua.pdf                    | Bin 195100 -> 195300 bytes
 tex/context/base/util-sql-imp-library.lua          |   1 +
 tex/context/base/util-sql-imp-swiglib.lua          |   7 +
 tex/generic/context/luatex/luatex-fonts-ext.lua    |  16 +-
 tex/generic/context/luatex/luatex-fonts-merged.lua | 285 +++++++++++++--------
 23 files changed, 540 insertions(+), 244 deletions(-)

(limited to 'tex')

diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index d398d6cc5..cd49e7efc 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.10.02 22:03}
+\newcontextversion{2012.10.05 19:52}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index c5e0682b0..d8adcf388 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.10.02 22:03}
+\newcontextversion{2012.10.05 19:52}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index b04ab3399..e9bc07c32 100644
Binary files a/tex/context/base/context-version.pdf and b/tex/context/base/context-version.pdf differ
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index 7d1880385..8f66709c2 100644
Binary files a/tex/context/base/context-version.png and b/tex/context/base/context-version.png differ
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 02d5ae27c..a69d67d62 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.10.02 22:03}
+\edef\contextversion{2012.10.05 19:52}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index d491e43e7..7cfee6b89 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.10.02 22:03}
+\edef\contextversion{2012.10.05 19:52}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/font-ctx.lua b/tex/context/base/font-ctx.lua
index 7d4fa35db..1b3384833 100644
--- a/tex/context/base/font-ctx.lua
+++ b/tex/context/base/font-ctx.lua
@@ -1623,3 +1623,91 @@ nodes.injections.installnewkern(function(k)
 end)
 
 directives.register("nodes.injections.fontkern", function(v) kern.subtype = v and 0 or 1 end)
+
+-- here
+
+local trace_analyzing    = false  trackers.register("otf.analyzing", function(v) trace_analyzing = v end)
+
+local otffeatures        = fonts.constructors.newfeatures("otf")
+local registerotffeature = otffeatures.register
+
+local analyzers          = fonts.analyzers
+local methods            = analyzers.methods
+
+local get_attribute      = node.has_attribute
+local set_attribute      = node.set_attribute
+local unset_attribute    = node.unset_attribute
+local traverse_by_id     = node.traverse_id
+
+local a_color            = attributes.private('color')
+local a_colormodel       = attributes.private('colormodel')
+local a_state            = attributes.private('state')
+local m_color            = attributes.list[a_color] or { }
+
+local glyph_code         = nodes.nodecodes.glyph
+
+local names = {
+    "font:1", "font:2", "font:3", "font:3",                     -- arabic
+    "font:4", "font:5", "font:6", "font:7", "font:8", "font:9", -- devanagary
+}
+
+local function markstates(head)
+    if head then
+        local model = get_attribute(head,a_colormodel) or 1
+        for glyph in traverse_by_id(glyph_code,head) do
+            local a = get_attribute(glyph,a_state)
+            if a then
+                local name = names[a]
+                if name then
+                    local color = m_color[name]
+                    if color then
+                        set_attribute(glyph,a_colormodel,model)
+                        set_attribute(glyph,a_color,color)
+                    end
+                end
+            end
+        end
+    end
+end
+
+local function analyzeprocessor(head,font,attr)
+    local tfmdata = fontdata[font]
+    local script, language = otf.scriptandlanguage(tfmdata,attr)
+    local action = methods[script]
+    if not action then
+        return head, false
+    end
+    if type(action) == "function" then
+        local head, done = action(head,font,attr)
+        if done and trace_analyzing then
+            markstates(head)
+        end
+        return head, done
+    end
+    action = action[language]
+    if action then
+        local head, done = action(head,font,attr)
+        if done and trace_analyzing then
+            markstates(head)
+        end
+        return head, done
+    else
+        return head, false
+    end
+end
+
+registerotffeature { -- adapts
+    name         = "analyze",
+    processors = {
+        node     = analyzeprocessor,
+    }
+}
+
+function methods.nocolor(head,font,attr)
+    for n in traverse_by_id(glyph_code,head) do
+        if not font or n.font == font then
+            unset_attribute(n,a_color)
+        end
+    end
+    return head, true
+end
diff --git a/tex/context/base/font-ota.lua b/tex/context/base/font-ota.lua
index edf5996b3..962f5d4d3 100644
--- a/tex/context/base/font-ota.lua
+++ b/tex/context/base/font-ota.lua
@@ -40,11 +40,6 @@ local fontdata            = fonts.hashes.identifiers
 local state               = attributes.private('state')
 local categories          = characters and characters.categories or { } -- sorry, only in context
 
-local tracers             = nodes.tracers
-local colortracers        = tracers and tracers.colors
-local setnodecolor        = colortracers and colortracers.set   or function() end
-local resetnodecolor      = colortracers and colortracers.reset or function() end
-
 local otffeatures         = fonts.constructors.newfeatures("otf")
 local registerotffeature  = otffeatures.register
 
@@ -117,14 +112,14 @@ end
 local function analyzeinitializer(tfmdata,value) -- attr
     local script, language = otf.scriptandlanguage(tfmdata) -- attr
     local action = initializers[script]
-    if action then
-        if type(action) == "function" then
+    if not action then
+        -- skip
+    elseif type(action) == "function" then
+        return action(tfmdata,value)
+    else
+        local action = action[language]
+        if action then
             return action(tfmdata,value)
-        else
-            local action = action[language]
-            if action then
-                return action(tfmdata,value)
-            end
         end
     end
 end
@@ -133,14 +128,14 @@ local function analyzeprocessor(head,font,attr)
     local tfmdata = fontdata[font]
     local script, language = otf.scriptandlanguage(tfmdata,attr)
     local action = methods[script]
-    if action then
-        if type(action) == "function" then
+    if not action then
+        -- skip
+    elseif type(action) == "function" then
+        return action(head,font,attr)
+    else
+        action = action[language]
+        if action then
             return action(head,font,attr)
-        else
-            action = action[language]
-            if action then
-                return action(head,font,attr)
-            end
         end
     end
     return head, false
@@ -252,7 +247,6 @@ local isol_fina_medi_init = {
 
 local arab_warned = { }
 
-
 -- todo: gref
 
 local function warning(current,what)
@@ -263,37 +257,24 @@ local function warning(current,what)
     end
 end
 
-function methods.nocolor(head,font,attr)
-    for n in traverse_id(glyph_code,head) do
-        if not font or n.font == font then
-            resetnodecolor(n)
-        end
-    end
-    return head, true
-end
-
 local function finish(first,last)
     if last then
         if first == last then
             local fc = first.char
             if isol_fina_medi_init[fc] or isol_fina[fc] then
                 set_attribute(first,state,4) -- isol
-                if trace_analyzing then setnodecolor(first,"font:isol") end
             else
                 warning(first,"isol")
                 set_attribute(first,state,0) -- error
-                if trace_analyzing then resetnodecolor(first) end
             end
         else
             local lc = last.char
             if isol_fina_medi_init[lc] or isol_fina[lc] then -- why isol here ?
             -- if laststate == 1 or laststate == 2 or laststate == 4 then
                 set_attribute(last,state,3) -- fina
-                if trace_analyzing then setnodecolor(last,"font:fina") end
             else
                 warning(last,"fina")
                 set_attribute(last,state,0) -- error
-                if trace_analyzing then resetnodecolor(last) end
             end
         end
         first, last = nil, nil
@@ -302,11 +283,9 @@ local function finish(first,last)
         local fc = first.char
         if isol_fina_medi_init[fc] or isol_fina[fc] then
             set_attribute(first,state,4) -- isol
-            if trace_analyzing then setnodecolor(first,"font:isol") end
         else
             warning(first,"isol")
             set_attribute(first,state,0) -- error
-            if trace_analyzing then resetnodecolor(first) end
         end
         first = nil
     end
@@ -324,20 +303,16 @@ function methods.arab(head,font,attr) -- maybe make a special version with no tr
             local char = current.char
             if marks[char] or (useunicodemarks and categories[char] == "mn") then
                 set_attribute(current,state,5) -- mark
-                if trace_analyzing then setnodecolor(current,"font:mark") end
             elseif isol[char] then -- can be zwj or zwnj too
                 first, last = finish(first,last)
                 set_attribute(current,state,4) -- isol
-                if trace_analyzing then setnodecolor(current,"font:isol") end
                 first, last = nil, nil
             elseif not first then
                 if isol_fina_medi_init[char] then
                     set_attribute(current,state,1) -- init
-                    if trace_analyzing then setnodecolor(current,"font:init") end
                     first, last = first or current, current
                 elseif isol_fina[char] then
                     set_attribute(current,state,4) -- isol
-                    if trace_analyzing then setnodecolor(current,"font:isol") end
                     first, last = nil, nil
                 else -- no arab
                     first, last = finish(first,last)
@@ -345,18 +320,15 @@ function methods.arab(head,font,attr) -- maybe make a special version with no tr
             elseif isol_fina_medi_init[char] then
                 first, last = first or current, current
                 set_attribute(current,state,2) -- medi
-                if trace_analyzing then setnodecolor(current,"font:medi") end
             elseif isol_fina[char] then
                 if not has_attribute(last,state,1) then
                     -- tricky, we need to check what last may be !
                     set_attribute(last,state,2) -- medi
-                    if trace_analyzing then setnodecolor(last,"font:medi") end
                 end
                 set_attribute(current,state,3) -- fina
-                if trace_analyzing then setnodecolor(current,"font:fina") end
                 first, last = nil, nil
             elseif char >= 0x0600 and char <= 0x06FF then
-                if trace_analyzing then setnodecolor(current,"font:rest") end
+                set_attribute(current,state,6) -- rest
                 first, last = finish(first,last)
             else --no
                 first, last = finish(first,last)
diff --git a/tex/context/base/font-otd.lua b/tex/context/base/font-otd.lua
index ebd487959..1d8c4ad43 100644
--- a/tex/context/base/font-otd.lua
+++ b/tex/context/base/font-otd.lua
@@ -6,6 +6,7 @@ if not modules then modules = { } end modules ['font-otd'] = {
     license   = "see context related readme files"
 }
 
+local type = type
 local match = string.match
 local sequenced = table.sequenced
 
@@ -120,6 +121,12 @@ local special_attributes = {
     medi = 2,
     fina = 3,
     isol = 4
+ -- devanagari
+    rphf = 5,
+    half = 6,
+    pref = 7,
+    blwf = 8,
+    pstf = 9,
 }
 
 local resolved = { } -- we only resolve a font,script,language,attribute pair once
@@ -180,7 +187,7 @@ end
 --     return v
 -- end)
 
-function otf.dataset(tfmdata,sequences,font,attr) -- attr only when explicit (as in special parbuilder)
+function otf.dataset(tfmdata,font,attr) -- attr only when explicit (as in special parbuilder)
 
     local script, language, s_enabled, a_enabled, dynamic
 
@@ -220,12 +227,17 @@ function otf.dataset(tfmdata,sequences,font,attr) -- attr only when explicit (as
     end
     local ra = rl[attr]
     if ra == nil then -- attr can be false
-        ra = { }
+        ra = {
+            -- indexed but we can also add specific data by key in:
+        }
         rl[attr] = ra
+        local sequences = tfmdata.resources.sequences
         setmetatableindex(ra, function(t,k)
-            local v = initialize(sequences[k],script,language,s_enabled,a_enabled,font,attr,dynamic)
-            t[k] = v or false
-            return v
+            if type(k) == "number" then
+                local v = initialize(sequences[k],script,language,s_enabled,a_enabled,font,attr,dynamic)
+                t[k] = v or false
+                return v
+            end
         end)
     end
 
diff --git a/tex/context/base/font-otn.lua b/tex/context/base/font-otn.lua
index ef842b099..ba9b0ed94 100644
--- a/tex/context/base/font-otn.lua
+++ b/tex/context/base/font-otn.lua
@@ -116,6 +116,8 @@ results in different tables.</p>
 -- we now use only one hash. If needed we can have multiple again but in that
 -- case I will probably prefix (i.e. rename) the lookups in the cached font file.
 
+-- Todo: make plugin feature that operates on char/glyphnode arrays
+
 local concat, insert, remove = table.concat, table.insert, table.remove
 local format, gmatch, gsub, find, match, lower, strip = string.format, string.gmatch, string.gsub, string.find, string.match, string.lower, string.strip
 local type, next, tonumber, tostring = type, next, tonumber, tostring
@@ -151,6 +153,7 @@ local report_subchain = logs.reporter("fonts","otf subchain")
 local report_chain    = logs.reporter("fonts","otf chain")
 local report_process  = logs.reporter("fonts","otf process")
 local report_prepare  = logs.reporter("fonts","otf prepare")
+local report_warning  = logs.reporter("fonts","otf warning")
 
 registertracker("otf.verbose_chain", function(v) otf.setcontextchain(v and "verbose") end)
 registertracker("otf.normal_chain",  function(v) otf.setcontextchain(v and "normal")  end)
@@ -185,7 +188,6 @@ local glyph_code         = nodecodes.glyph
 local glue_code          = nodecodes.glue
 local disc_code          = nodecodes.disc
 local whatsit_code       = nodecodes.whatsit
-local user_code          = nodecodes.user
 
 local dir_code           = whatcodes.dir
 local localpar_code      = whatcodes.localpar
@@ -311,35 +313,110 @@ local function pref(kind,lookupname)
     return format("feature %s, lookup %s",kind,lookupname)
 end
 
--- we can assume that languages that use marks are not hyphenated
--- we can also assume that at most one discretionary is present
+-- We can assume that languages that use marks are not hyphenated. We can also assume
+-- that at most one discretionary is present.
+
+-- We do need components in funny kerning mode but maybe I can better reconstruct then
+-- as we do have the font components info available; removing components makes the
+-- previous code much simpler. Also, later on copying and freeing becomes easier.
+-- However, for arabic we need to keep them around for the sake of mark placement
+-- and indices.
+
+-- local function collapsecomponents(start)
+--     local c = start
+--     while c do
+--         local cp = c.components
+--         if cp then
+--             flush_node_list(cp)
+--             c.components = nil
+--         end
+--         c = c.next
+--     end
+--     return start
+-- end
 
-local function markstoligature(kind,lookupname,start,stop,char)
-    local n = copy_node(start)
+local function collapsecomponents(start)
+    if not start.next then
+        report_warning("suspicious ligature components")
+        -- actually an error
+        return components
+    else
+        local head = nil
+        local tail = nil
+        while start do
+            local components = start.components
+            if components then
+                if head then
+                    tail.next = components
+                    components.prev = tail
+                else
+                    head = components
+                end
+                tail = find_node_tail(components)
+                start.components = nil
+            else
+                if head then
+                    tail.next = start
+                    start.prev = tail
+                    tail = start
+                else
+                    head = start
+                    tail = start
+                end
+            end
+            start = start.next
+        end
+        return head
+    end
+end
+
+local function markstoligature(kind,lookupname,start,stop,char) -- some trickery to keep head as is
+    -- [start]..[stop]
     local keep = start
-    local current
-    current, start = insert_node_after(start,start,n)
+    local current, start = insert_node_after(start,start,copy_node(start))
+    -- [current][start]..[stop]
     local snext = stop.next
     current.next = snext
     if snext then
         snext.prev = current
     end
-    start.prev, stop.next = nil, nil
-    current.char, current.subtype, current.components = char, ligature_code, start
+    start.prev = nil
+    stop.next = nil
+    current.char = char
+    current.subtype = ligature_code
+    current.components = collapsecomponents(start)
     return keep
 end
 
+-- The next code is somewhat complicated by the fact that some fonts can have ligatures made
+-- from ligatures that themselves have marks. This was identified by Kai in for instance
+-- arabtype:  KAF LAM SHADDA ALEF FATHA (0x0643 0x0644 0x0651 0x0627 0x064E). This becomes
+-- KAF LAM-ALEF with a SHADDA on the first and a FATHA op de second component. In a next
+-- iteration this becomes a KAF-LAM-ALEF with a SHADDA on the second and a FATHA on the
+-- third component.
+
+local function getcomponentindex(start) -- so we cannot remove components !
+	local i = 0
+	if start.subtype == ligature_code then
+		local comp = start.components
+		while comp do
+			i = i + getcomponentindex(comp)
+			comp = comp.next
+		end
+		return i
+	elseif not marks[start.char] then
+        return 1
+    else
+        return 0
+	end
+end
+
 local function toligature(kind,lookupname,start,stop,char,markflag,discfound) -- brr head
     if start == stop then
         start.char = char
         return start
     elseif discfound then
      -- print("start->stop",nodes.tosequence(start,stop))
-        local components = start.components
-        if components then
-            flush_node_list(components)
-            start.components = nil
-        end
         local lignode = copy_node(start)
         lignode.font = start.font
         lignode.char = char
@@ -348,7 +425,7 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound) --
         local prev = start.prev
         stop.next = nil
         start.prev = nil
-        lignode.components = start
+        lignode.components = collapsecomponents(start)
      -- print("lignode",nodes.tosequence(lignode))
      -- print("components",nodes.tosequence(lignode.components))
         prev.next = lignode
@@ -362,29 +439,32 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound) --
     else
         -- start is the ligature
         local deletemarks = markflag ~= "mark"
-        local n = copy_node(start)
-        local current
-        current, start = insert_node_after(start,start,n)
-        local snext = stop.next
-        current.next = snext
-        if snext then
-            snext.prev = current
+        local prev = start.prev
+        local next = stop.next
+        local current, start = insert_node_after(start,start,copy_node(start))
+        -- [start->current][copyofstart->start]...[stop]
+        current.next = next
+        if next then
+            next.prev = current
         end
         start.prev = nil
         stop.next = nil
         current.char = char
         current.subtype = ligature_code
-        current.components = start
+        current.components = collapsecomponents(start)
         local head = current
         -- this is messy ... we should get rid of the components eventually
-        local i = 0 -- is index of base
+        local baseindex = 0
+        local componentindex = 0
         while start do
-            if not marks[start.char] then
-                i = i + 1
+            local char = start.char
+            if not marks[char] then
+				baseindex = baseindex + componentindex
+				componentindex = getcomponentindex(start)
             elseif not deletemarks then -- quite fishy
-                set_attribute(start,ligacomp,i)
+                set_attribute(start,ligacomp,baseindex + (has_attribute(start,ligacomp) or componentindex))
                 if trace_marks then
-                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(start.char),i)
+                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(char),has_attribute(start,ligacomp))
                 end
                 head, current = insert_node_after(head,current,copy_node(start))
             end
@@ -392,22 +472,17 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound) --
         end
         start = current.next
         while start and start.id == glyph_code do
-            if marks[start.char] then
-                set_attribute(start,ligacomp,i)
+            local char = start.char
+            if marks[char] then
+                set_attribute(start,ligacomp,baseindex + (has_attribute(start,ligacomp) or componentindex))
                 if trace_marks then
-                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(start.char),i)
+                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(char),has_attribute(start,ligacomp))
                 end
             else
                 break
             end
             start = start.next
         end
-        --
-        -- we do need components in funny kerning mode but maybe I can better reconstruct then
-        -- as we do have the font components info available; removing components makes the
-        -- previous code much simpler
-        --
-        -- flush_node_list(head.components)
         return head
     end
 end
@@ -489,12 +564,12 @@ function handlers.gsub_alternate(start,kind,lookupname,alternative,sequence)
     local choice = get_alternative_glyph(start,alternative,value)
     if choice then
         if trace_alternatives then
-            logprocess("%s: replacing %s by alternative %s (%s)",pref(kind,lookupname),gref(start.char),gref(choice),choice)
+            logprocess("%s: replacing %s by alternative %s (%s)",pref(kind,lookupname),gref(char),gref(choice),choice)
         end
         start.char = choice
     else
         if trace_alternatives then
-            logwarning("%s: no variant %s for %s",pref(kind,lookupname),tostring(value),gref(start.char))
+            logwarning("%s: no variant %s for %s",pref(kind,lookupname),tostring(value),gref(char))
         end
     end
     return start, true
@@ -577,6 +652,7 @@ function handlers.gsub_ligature(start,kind,lookupname,ligature,sequence)
                     local stopchar = stop.char
                     start = toligature(kind,lookupname,start,stop,lig,skipmark,discfound)
                     logprocess("%s: replacing %s upto %s by ligature %s",pref(kind,lookupname),gref(startchar),gref(stopchar),gref(start.char))
+-- print("start",nodes.listtoutf(start.components))
                 else
                     start = toligature(kind,lookupname,start,stop,lig,skipmark,discfound)
                 end
@@ -988,6 +1064,10 @@ local function delete_till_stop(start,stop,ignoremarks) -- keeps start
         repeat -- start x x m x x stop => start m
             local next = start.next
             if not marks[next.char] then
+local components = next.components
+if components then
+    node.flush_list(components)
+end
                 delete_node(start,next)
             end
             n = n + 1
@@ -995,6 +1075,10 @@ local function delete_till_stop(start,stop,ignoremarks) -- keeps start
     else -- start x x x stop => start
         repeat
             local next = start.next
+local components = next.components
+if components then
+    node.flush_list(components)
+end
             delete_node(start,next)
             n = n + 1
         until next == stop
@@ -1967,7 +2051,13 @@ local special_attributes = {
     init = 1,
     medi = 2,
     fina = 3,
-    isol = 4
+    isol = 4,
+ -- devanagari
+    rphf = 5,
+    half = 6,
+    pref = 7,
+    blwf = 8,
+    pstf = 9,
 }
 
 local function initialize(sequence,script,language,enabled)
@@ -1986,7 +2076,7 @@ local function initialize(sequence,script,language,enabled)
     return false
 end
 
-function otf.dataset(tfmdata,sequences,font) -- generic variant, overloaded in context
+function otf.dataset(tfmdata,font) -- generic variant, overloaded in context
     local shared     = tfmdata.shared
     local properties = tfmdata.properties
     local language   = properties.language or "dflt"
@@ -2004,12 +2094,17 @@ function otf.dataset(tfmdata,sequences,font) -- generic variant, overloaded in c
     end
     local rl = rs[language]
     if not rl then
-        rl = { }
+        rl = {
+            -- indexed but we can also add specific data by key
+        }
         rs[language] = rl
+        local sequences = tfmdata.resources.sequences
         setmetatableindex(rl, function(t,k)
-            local v = enabled and initialize(sequences[k],script,language,enabled)
-            t[k] = v
-            return v
+            if type(k) == "number" then
+                local v = enabled and initialize(sequences[k],script,language,enabled)
+                t[k] = v
+                return v
+            end
         end)
     end
     return rl
@@ -2060,7 +2155,7 @@ local function featuresprocessor(head,font,attr)
 
     local sequences        = resources.sequences
     local done             = false
-    local datasets         = otf.dataset(tfmdata,sequences,font,attr)
+    local datasets         = otf.dataset(tfmdata,font,attr)
 
     local dirstack = { } -- could move outside function
 
@@ -2069,6 +2164,9 @@ local function featuresprocessor(head,font,attr)
     -- to keep track of directions anyway. Also at some point I want to play with
     -- font interactions and then we do need the full sweeps.
 
+    -- Keeping track of the headnode is needed for devanagari (I generalized it a bit
+    -- so that multiple cases are also covered.
+
     for s=1,#sequences do
         local dataset = datasets[s]
         if dataset then
@@ -2104,8 +2202,12 @@ local function featuresprocessor(head,font,attr)
                                         if lookupcache then
                                             local lookupmatch = lookupcache[start.char]
                                             if lookupmatch then
+                                                local headnode = start == head
                                                 start, success = handler(start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i)
                                                 if success then
+                                                    if headnode then
+                                                        head = start
+                                                    end
                                                     break
                                                 end
                                             end
@@ -2149,10 +2251,14 @@ local function featuresprocessor(head,font,attr)
                                             local lookupmatch = lookupcache[start.char]
                                             if lookupmatch then
                                                 -- sequence kan weg
+                                                local headnode = start == head
                                                 local ok
                                                 start, ok = handler(start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,1)
                                                 if ok then
                                                     success = true
+                                                    if headnode then
+                                                        head = start
+                                                    end
                                                 end
                                             end
                                             if start then start = start.next end
@@ -2222,10 +2328,14 @@ local function featuresprocessor(head,font,attr)
                                                 local lookupmatch = lookupcache[start.char]
                                                 if lookupmatch then
                                                     -- we could move all code inline but that makes things even more unreadable
+                                                    local headnode = start == head
                                                     local ok
                                                     start, ok = handler(start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i)
                                                     if ok then
                                                         success = true
+                                                        if headnode then
+                                                            head = start
+                                                        end
                                                         break
                                                     end
                                                 end
@@ -2551,3 +2661,7 @@ registerotffeature {
         node     = featuresprocessor,
     }
 }
+
+-- this will change but is needed for an experiment:
+
+otf.handlers = handlers
diff --git a/tex/context/base/font-ott.lua b/tex/context/base/font-ott.lua
index 1aeda273b..c0a6c4d27 100644
--- a/tex/context/base/font-ott.lua
+++ b/tex/context/base/font-ott.lua
@@ -626,9 +626,11 @@ local features = allocate {
     ['trep'] = 'traditional tex replacements',
     ['tlig'] = 'traditional tex ligatures',
 
-    ['ss..']   = 'stylistic set ..',
-    ['cv..']   = 'character variant ..',
-    ['js..']   = 'justification ..',
+    ['ss..'] = 'stylistic set ..',
+    ['cv..'] = 'character variant ..',
+    ['js..'] = 'justification ..',
+
+    ["dv.."] = "devanagari ..",
 }
 
 local baselines = allocate {
diff --git a/tex/context/base/font-pre.mkiv b/tex/context/base/font-pre.mkiv
index 8288ef412..8d3edb9ec 100644
--- a/tex/context/base/font-pre.mkiv
+++ b/tex/context/base/font-pre.mkiv
@@ -169,7 +169,7 @@
 \definecolor[font:fina][b=.75]
 \definecolor[font:isol][r=.75,g=.75] % [y=.75]
 \definecolor[font:mark][r=.75,b=.75] % [m=.75]
-\definecolor[font:rest][g=.75,b=.75] % [c=.75]
+\definecolor[font:rest][b=.75,g=.75] % [c=.75]
 
 \definecolor[trace:r][r=.75,t=.5,a=1]
 \definecolor[trace:g][g=.75,t=.5,a=1]
@@ -189,6 +189,16 @@
 \definecolor[trace:ds][s=.75,t=.75,a=1]
 \definecolor[trace:do][r=1,g=.6,b=.1,t=.75,a=1]
 
+\definecolor[font:1] [r=.75]
+\definecolor[font:2] [g=.75]
+\definecolor[font:3] [b=.75]
+\definecolor[font:4] [r=.75,g=.75]
+\definecolor[font:5] [r=.75,b=.75]
+\definecolor[font:6] [b=.75,g=.75]
+\definecolor[font:7] [r=.75]
+\definecolor[font:8] [g=.75]
+\definecolor[font:9] [b=.75]
+
 %D Now we're up to some definitions.
 
 \definebodyfontenvironment
diff --git a/tex/context/base/font-tra.mkiv b/tex/context/base/font-tra.mkiv
index abe034d8a..e838d4938 100644
--- a/tex/context/base/font-tra.mkiv
+++ b/tex/context/base/font-tra.mkiv
@@ -116,9 +116,9 @@
 
 \unexpanded\def\otfstepcharcommand#1#2#3% font char class
   {\removeunwantedspaces
-   \hskip.5em plus .125em\relax
+   \hskip.5\emwidth \s!plus .125\emwidth\relax
    \doif{#3}{mark}{\underbar}{U+\hexnumber{#2}}:\ruledhbox{\ctxlua{nodes.tracers.fontchar(#1,#2)}}%
-   \hskip.5em plus .125em\relax}
+   \hskip.5\emwidth \s!plus .125\emwidth\relax}
 
 \unexpanded\def\otfstepmessagecommand#1#2%
   {\begingroup
@@ -126,7 +126,7 @@
    \veryraggedright
    \forgetparindent
    \forgeteverypar
-   \hangindent1em
+   \hangindent\emwidth
    \hangafter\plusone
    \dontleavehmode\hbox{\detokenize{#1}}\removeunwantedspaces
    \doifsomething{#2}{\break\detokenize{#2}}\endgraf
diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua
index 2b3319e45..80f28c2cd 100644
--- a/tex/context/base/l-table.lua
+++ b/tex/context/base/l-table.lua
@@ -415,7 +415,8 @@ local function do_serialize(root,name,depth,level,indexed)
         if compact then
             last = #root
             for k=1,last do
-                if not root[k] then
+--                 if not root[k] then
+                if root[k] == nil then
                     last = k - 1
                     break
                 end
diff --git a/tex/context/base/math-arr.mkiv b/tex/context/base/math-arr.mkiv
index 2d57a9337..5b50303d0 100644
--- a/tex/context/base/math-arr.mkiv
+++ b/tex/context/base/math-arr.mkiv
@@ -336,6 +336,14 @@
 \unexpanded\def\doublebond{{\xequal}}
 \unexpanded\def\triplebond{{\xtriplerel}}
 
+%D A bit or arrow juggling:
+%D
+%D \startbuffer
+%D \hbox to \hsize{\rightoverleftarrowfill}
+%D \stopbuffer
+%D
+%D \typebuffer \blank \getbuffer \blank
+
 \unexpanded\def\rightoverleftarrowfill
    {\specrightoverleftarrowfill}
 
diff --git a/tex/context/base/node-tra.lua b/tex/context/base/node-tra.lua
index 387dd99a6..6b5f49964 100644
--- a/tex/context/base/node-tra.lua
+++ b/tex/context/base/node-tra.lua
@@ -46,6 +46,7 @@ local copy_node_list  = node.copy_list
 local hpack_node_list = node.hpack
 local free_node_list  = node.flush_list
 local traverse_nodes  = node.traverse
+local traverse_by_id  = node.traverse_id
 
 local nodecodes       = nodes.nodecodes
 local whatcodes       = nodes.whatcodes
@@ -352,7 +353,7 @@ end
 
 function nodes.handlers.checkglyphs(head,message)
     local t = { }
-    for g in traverse_id(glyph_code,head) do
+    for g in traverse_by_id(glyph_code,head) do
         t[#t+1] = format("U+%04X:%s",g.char,g.subtype)
     end
     if #t > 0 then
@@ -714,72 +715,73 @@ local get_attribute   = node.has_attribute
 local set_attribute   = node.set_attribute
 local unset_attribute = node.unset_attribute
 
-local attribute  = attributes.private('color')
-local colormodel = attributes.private('colormodel')
-local mapping    = attributes.list[attribute] or { }
+local a_color         = attributes.private('color')
+local a_colormodel    = attributes.private('colormodel')
+local a_state         = attributes.private('state')
+local m_color         = attributes.list[a_color] or { }
 
 function colors.set(n,c,s)
-    local mc = mapping[c]
+    local mc = m_color[c]
     if not mc then
-        unset_attribute(n,attribute)
+        unset_attribute(n,a_color)
     else
-        if not get_attribute(n,colormodel) then
-            set_attribute(n,colormodel,s or 1)
+        if not get_attribute(n,a_colormodel) then
+            set_attribute(n,a_colormodel,s or 1)
         end
-        set_attribute(n,attribute,mc)
+        set_attribute(n,a_color,mc)
     end
 end
 
 function colors.setlist(n,c,s)
     while n do
-        local mc = mapping[c]
+        local mc = m_color[c]
         if not mc then
-            unset_attribute(n,attribute)
+            unset_attribute(n,a_color)
         else
-            if not get_attribute(n,colormodel) then
-                set_attribute(n,colormodel,s or 1)
+            if not get_attribute(n,a_colormodel) then
+                set_attribute(n,a_colormodel,s or 1)
             end
-            set_attribute(n,attribute,mc)
+            set_attribute(n,a_color,mc)
         end
         n = n.next
     end
 end
 
 function colors.reset(n)
-    unset_attribute(n,attribute)
+    unset_attribute(n,a_color)
 end
 
 -- maybe
 
-local transparencies = { }
+local transparencies   = { }
 tracers.transparencies = transparencies
 
-local attribute = attributes.private('transparency')
-local mapping   = attributes.list[attribute] or { }
+local a_transparency   = attributes.private('transparency')
+local m_transparency   = attributes.list[a_transparency] or { }
 
 function transparencies.set(n,t)
-    local mt = mapping[t]
+    local mt = m_transparency[t]
     if not mt then
-        unset_attribute(n,attribute)
+        unset_attribute(n,a_transparency)
     else
-        set_attribute(n,attribute,mt)
+        set_attribute(n,a_transparency,mt)
     end
 end
 
 function transparencies.setlist(n,c,s)
     while n do
-        local mt = mapping[c]
+        local mt = m_transparency[c]
         if not mt then
-            unset_attribute(n,attribute)
+            unset_attribute(n,a_transparency)
         else
-            set_attribute(n,attribute,mt)
+            set_attribute(n,a_transparency,mt)
         end
         n = n.next
     end
 end
 
 function transparencies.reset(n)
-    unset_attribute(n,attribute)
+    unset_attribute(n,a_transparency)
 end
 
 -- for the moment here
diff --git a/tex/context/base/pack-com.mkiv b/tex/context/base/pack-com.mkiv
index 8b4d5c0c6..9320411ce 100644
--- a/tex/context/base/pack-com.mkiv
+++ b/tex/context/base/pack-com.mkiv
@@ -663,8 +663,8 @@
 \setvalue{\??pairedboxalign\v!middle}% 4
   {\let\pack_pairedboxes_align_l\hss
    \let\pack_pairedboxes_align_r\hss
-   \let\pack_pairedboxes_align_t\hss
-   \let\pack_pairedboxes_align_b\hss}
+   \let\pack_pairedboxes_align_t\vss
+   \let\pack_pairedboxes_align_b\vss}
 
 \setvalue{\??pairedboxalign\v!bottom}{\getvalue{\??pairedboxalign\v!low }}
 \setvalue{\??pairedboxalign   \v!top}{\getvalue{\??pairedboxalign\v!high}}
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 7cda58a77..e4b35c632 100644
Binary files a/tex/context/base/status-files.pdf and b/tex/context/base/status-files.pdf differ
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 02aab58d1..cedb02ae4 100644
Binary files a/tex/context/base/status-lua.pdf and b/tex/context/base/status-lua.pdf differ
diff --git a/tex/context/base/util-sql-imp-library.lua b/tex/context/base/util-sql-imp-library.lua
index e04c4ac44..f16739085 100644
--- a/tex/context/base/util-sql-imp-library.lua
+++ b/tex/context/base/util-sql-imp-library.lua
@@ -276,6 +276,7 @@ local celltemplate = "cells[%s]"
 methods.library = {
     runner       = function() end, -- never called
     execute      = execute,
+    initialize   = initialize,     -- returns session
     usesfiles    = false,
     wraptemplate = wraptemplate,
     celltemplate = celltemplate,
diff --git a/tex/context/base/util-sql-imp-swiglib.lua b/tex/context/base/util-sql-imp-swiglib.lua
index 4b9cda896..25723ee84 100644
--- a/tex/context/base/util-sql-imp-swiglib.lua
+++ b/tex/context/base/util-sql-imp-swiglib.lua
@@ -153,6 +153,7 @@ end
 -- end
 
 local util_mysql_fetch_fields_from_current_row = mysql.util_mysql_fetch_fields_from_current_row
+local util_mysql_fetch_all_rows                = mysql.util_mysql_fetch_all_rows
 
 local function list(t)
     return util_mysql_fetch_fields_from_current_row(t._result_)
@@ -168,11 +169,16 @@ local function hash(t)
     return data
 end
 
+local function wholelist(t)
+    return util_mysql_fetch_all_rows(t._result_)
+end
+
 local mt = { __index = {
         -- regular
         finish      = finish,
         list        = list,
         hash        = hash,
+        wholelist   = wholelist,
         -- compatibility
         numrows     = numrows,
         getcolnames = getcolnames,
@@ -413,6 +419,7 @@ local celltemplate = "cells[%s]"
 methods.swiglib = {
     runner       = function() end, -- never called
     execute      = execute,
+    initialize   = initialize, -- returns session
     usesfiles    = false,
     wraptemplate = wraptemplate,
     celltemplate = celltemplate,
diff --git a/tex/generic/context/luatex/luatex-fonts-ext.lua b/tex/generic/context/luatex/luatex-fonts-ext.lua
index d8884ccc7..b60d04512 100644
--- a/tex/generic/context/luatex/luatex-fonts-ext.lua
+++ b/tex/generic/context/luatex/luatex-fonts-ext.lua
@@ -18,18 +18,14 @@ local otffeatures = fonts.constructors.newfeatures("otf")
 
 local function initializeitlc(tfmdata,value)
     if value then
-        -- the magic 40 and it formula come from Dohyun Kim
-        local parameters  = tfmdata.parameters
+        -- the magic 40 and it formula come from Dohyun Kim but we might need another guess
+        local parameters = tfmdata.parameters
         local italicangle = parameters.italicangle
         if italicangle and italicangle ~= 0 then
-            local uwidth = (parameters.uwidth or 40)/2
-            for unicode, d in next, tfmdata.descriptions do
-                local it = d.boundingbox[3] - d.width + uwidth
-                if it ~= 0 then
-                    d.italic = it
-                end
-            end
-            tfmdata.properties.hasitalics = true
+            local properties = tfmdata.properties
+            local factor = tonumber(value) or 1
+            properties.hasitalics = true
+            properties.autoitalicamount = factor * (parameters.uwidth or 40)/2
         end
     end
 end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 1a375d009..920b43eff 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 10/02/12 22:03:14
+-- merge date  : 10/05/12 19:52:22
 
 do -- begin closure to overcome local limits and interference
 
@@ -561,7 +561,8 @@ local function do_serialize(root,name,depth,level,indexed)
         if compact then
             last = #root
             for k=1,last do
-                if not root[k] then
+--                 if not root[k] then
+                if root[k] == nil then
                     last = k - 1
                     break
                 end
@@ -8934,6 +8935,8 @@ results in different tables.</p>
 -- we now use only one hash. If needed we can have multiple again but in that
 -- case I will probably prefix (i.e. rename) the lookups in the cached font file.
 
+-- Todo: make plugin feature that operates on char/glyphnode arrays
+
 local concat, insert, remove = table.concat, table.insert, table.remove
 local format, gmatch, gsub, find, match, lower, strip = string.format, string.gmatch, string.gsub, string.find, string.match, string.lower, string.strip
 local type, next, tonumber, tostring = type, next, tonumber, tostring
@@ -8969,6 +8972,7 @@ local report_subchain = logs.reporter("fonts","otf subchain")
 local report_chain    = logs.reporter("fonts","otf chain")
 local report_process  = logs.reporter("fonts","otf process")
 local report_prepare  = logs.reporter("fonts","otf prepare")
+local report_warning  = logs.reporter("fonts","otf warning")
 
 registertracker("otf.verbose_chain", function(v) otf.setcontextchain(v and "verbose") end)
 registertracker("otf.normal_chain",  function(v) otf.setcontextchain(v and "normal")  end)
@@ -9003,7 +9007,6 @@ local glyph_code         = nodecodes.glyph
 local glue_code          = nodecodes.glue
 local disc_code          = nodecodes.disc
 local whatsit_code       = nodecodes.whatsit
-local user_code          = nodecodes.user
 
 local dir_code           = whatcodes.dir
 local localpar_code      = whatcodes.localpar
@@ -9129,35 +9132,110 @@ local function pref(kind,lookupname)
     return format("feature %s, lookup %s",kind,lookupname)
 end
 
--- we can assume that languages that use marks are not hyphenated
--- we can also assume that at most one discretionary is present
+-- We can assume that languages that use marks are not hyphenated. We can also assume
+-- that at most one discretionary is present.
+
+-- We do need components in funny kerning mode but maybe I can better reconstruct then
+-- as we do have the font components info available; removing components makes the
+-- previous code much simpler. Also, later on copying and freeing becomes easier.
+-- However, for arabic we need to keep them around for the sake of mark placement
+-- and indices.
+
+-- local function collapsecomponents(start)
+--     local c = start
+--     while c do
+--         local cp = c.components
+--         if cp then
+--             flush_node_list(cp)
+--             c.components = nil
+--         end
+--         c = c.next
+--     end
+--     return start
+-- end
+
+local function collapsecomponents(start)
+    if not start.next then
+        report_warning("suspicious ligature components")
+        -- actually an error
+        return components
+    else
+        local head = nil
+        local tail = nil
+        while start do
+            local components = start.components
+            if components then
+                if head then
+                    tail.next = components
+                    components.prev = tail
+                else
+                    head = components
+                end
+                tail = find_node_tail(components)
+                start.components = nil
+            else
+                if head then
+                    tail.next = start
+                    start.prev = tail
+                    tail = start
+                else
+                    head = start
+                    tail = start
+                end
+            end
+            start = start.next
+        end
+        return head
+    end
+end
 
-local function markstoligature(kind,lookupname,start,stop,char)
-    local n = copy_node(start)
+local function markstoligature(kind,lookupname,start,stop,char) -- some trickery to keep head as is
+    -- [start]..[stop]
     local keep = start
-    local current
-    current, start = insert_node_after(start,start,n)
+    local current, start = insert_node_after(start,start,copy_node(start))
+    -- [current][start]..[stop]
     local snext = stop.next
     current.next = snext
     if snext then
         snext.prev = current
     end
-    start.prev, stop.next = nil, nil
-    current.char, current.subtype, current.components = char, ligature_code, start
+    start.prev = nil
+    stop.next = nil
+    current.char = char
+    current.subtype = ligature_code
+    current.components = collapsecomponents(start)
     return keep
 end
 
+-- The next code is somewhat complicated by the fact that some fonts can have ligatures made
+-- from ligatures that themselves have marks. This was identified by Kai in for instance
+-- arabtype:  KAF LAM SHADDA ALEF FATHA (0x0643 0x0644 0x0651 0x0627 0x064E). This becomes
+-- KAF LAM-ALEF with a SHADDA on the first and a FATHA op de second component. In a next
+-- iteration this becomes a KAF-LAM-ALEF with a SHADDA on the second and a FATHA on the
+-- third component.
+
+local function getcomponentindex(start) -- so we cannot remove components !
+	local i = 0
+	if start.subtype == ligature_code then
+		local comp = start.components
+		while comp do
+			i = i + getcomponentindex(comp)
+			comp = comp.next
+		end
+		return i
+	elseif not marks[start.char] then
+        return 1
+    else
+        return 0
+	end
+end
+
 local function toligature(kind,lookupname,start,stop,char,markflag,discfound) -- brr head
     if start == stop then
         start.char = char
         return start
     elseif discfound then
      -- print("start->stop",nodes.tosequence(start,stop))
-        local components = start.components
-        if components then
-            flush_node_list(components)
-            start.components = nil
-        end
         local lignode = copy_node(start)
         lignode.font = start.font
         lignode.char = char
@@ -9166,7 +9244,7 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound) --
         local prev = start.prev
         stop.next = nil
         start.prev = nil
-        lignode.components = start
+        lignode.components = collapsecomponents(start)
      -- print("lignode",nodes.tosequence(lignode))
      -- print("components",nodes.tosequence(lignode.components))
         prev.next = lignode
@@ -9180,29 +9258,32 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound) --
     else
         -- start is the ligature
         local deletemarks = markflag ~= "mark"
-        local n = copy_node(start)
-        local current
-        current, start = insert_node_after(start,start,n)
-        local snext = stop.next
-        current.next = snext
-        if snext then
-            snext.prev = current
+        local prev = start.prev
+        local next = stop.next
+        local current, start = insert_node_after(start,start,copy_node(start))
+        -- [start->current][copyofstart->start]...[stop]
+        current.next = next
+        if next then
+            next.prev = current
         end
         start.prev = nil
         stop.next = nil
         current.char = char
         current.subtype = ligature_code
-        current.components = start
+        current.components = collapsecomponents(start)
         local head = current
         -- this is messy ... we should get rid of the components eventually
-        local i = 0 -- is index of base
+        local baseindex = 0
+        local componentindex = 0
         while start do
-            if not marks[start.char] then
-                i = i + 1
+            local char = start.char
+            if not marks[char] then
+				baseindex = baseindex + componentindex
+				componentindex = getcomponentindex(start)
             elseif not deletemarks then -- quite fishy
-                set_attribute(start,ligacomp,i)
+                set_attribute(start,ligacomp,baseindex + (has_attribute(start,ligacomp) or componentindex))
                 if trace_marks then
-                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(start.char),i)
+                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(char),has_attribute(start,ligacomp))
                 end
                 head, current = insert_node_after(head,current,copy_node(start))
             end
@@ -9210,22 +9291,17 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound) --
         end
         start = current.next
         while start and start.id == glyph_code do
-            if marks[start.char] then
-                set_attribute(start,ligacomp,i)
+            local char = start.char
+            if marks[char] then
+                set_attribute(start,ligacomp,baseindex + (has_attribute(start,ligacomp) or componentindex))
                 if trace_marks then
-                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(start.char),i)
+                    logwarning("%s: keep mark %s, gets index %s",pref(kind,lookupname),gref(char),has_attribute(start,ligacomp))
                 end
             else
                 break
             end
             start = start.next
         end
-        --
-        -- we do need components in funny kerning mode but maybe I can better reconstruct then
-        -- as we do have the font components info available; removing components makes the
-        -- previous code much simpler
-        --
-        -- flush_node_list(head.components)
         return head
     end
 end
@@ -9307,12 +9383,12 @@ function handlers.gsub_alternate(start,kind,lookupname,alternative,sequence)
     local choice = get_alternative_glyph(start,alternative,value)
     if choice then
         if trace_alternatives then
-            logprocess("%s: replacing %s by alternative %s (%s)",pref(kind,lookupname),gref(start.char),gref(choice),choice)
+            logprocess("%s: replacing %s by alternative %s (%s)",pref(kind,lookupname),gref(char),gref(choice),choice)
         end
         start.char = choice
     else
         if trace_alternatives then
-            logwarning("%s: no variant %s for %s",pref(kind,lookupname),tostring(value),gref(start.char))
+            logwarning("%s: no variant %s for %s",pref(kind,lookupname),tostring(value),gref(char))
         end
     end
     return start, true
@@ -9395,6 +9471,7 @@ function handlers.gsub_ligature(start,kind,lookupname,ligature,sequence)
                     local stopchar = stop.char
                     start = toligature(kind,lookupname,start,stop,lig,skipmark,discfound)
                     logprocess("%s: replacing %s upto %s by ligature %s",pref(kind,lookupname),gref(startchar),gref(stopchar),gref(start.char))
+-- print("start",nodes.listtoutf(start.components))
                 else
                     start = toligature(kind,lookupname,start,stop,lig,skipmark,discfound)
                 end
@@ -9806,6 +9883,10 @@ local function delete_till_stop(start,stop,ignoremarks) -- keeps start
         repeat -- start x x m x x stop => start m
             local next = start.next
             if not marks[next.char] then
+local components = next.components
+if components then
+    node.flush_list(components)
+end
                 delete_node(start,next)
             end
             n = n + 1
@@ -9813,6 +9894,10 @@ local function delete_till_stop(start,stop,ignoremarks) -- keeps start
     else -- start x x x stop => start
         repeat
             local next = start.next
+local components = next.components
+if components then
+    node.flush_list(components)
+end
             delete_node(start,next)
             n = n + 1
         until next == stop
@@ -10785,7 +10870,13 @@ local special_attributes = {
     init = 1,
     medi = 2,
     fina = 3,
-    isol = 4
+    isol = 4,
+ -- devanagari
+    rphf = 5,
+    half = 6,
+    pref = 7,
+    blwf = 8,
+    pstf = 9,
 }
 
 local function initialize(sequence,script,language,enabled)
@@ -10804,7 +10895,7 @@ local function initialize(sequence,script,language,enabled)
     return false
 end
 
-function otf.dataset(tfmdata,sequences,font) -- generic variant, overloaded in context
+function otf.dataset(tfmdata,font) -- generic variant, overloaded in context
     local shared     = tfmdata.shared
     local properties = tfmdata.properties
     local language   = properties.language or "dflt"
@@ -10822,12 +10913,17 @@ function otf.dataset(tfmdata,sequences,font) -- generic variant, overloaded in c
     end
     local rl = rs[language]
     if not rl then
-        rl = { }
+        rl = {
+            -- indexed but we can also add specific data by key
+        }
         rs[language] = rl
+        local sequences = tfmdata.resources.sequences
         setmetatableindex(rl, function(t,k)
-            local v = enabled and initialize(sequences[k],script,language,enabled)
-            t[k] = v
-            return v
+            if type(k) == "number" then
+                local v = enabled and initialize(sequences[k],script,language,enabled)
+                t[k] = v
+                return v
+            end
         end)
     end
     return rl
@@ -10878,7 +10974,7 @@ local function featuresprocessor(head,font,attr)
 
     local sequences        = resources.sequences
     local done             = false
-    local datasets         = otf.dataset(tfmdata,sequences,font,attr)
+    local datasets         = otf.dataset(tfmdata,font,attr)
 
     local dirstack = { } -- could move outside function
 
@@ -10887,6 +10983,9 @@ local function featuresprocessor(head,font,attr)
     -- to keep track of directions anyway. Also at some point I want to play with
     -- font interactions and then we do need the full sweeps.
 
+    -- Keeping track of the headnode is needed for devanagari (I generalized it a bit
+    -- so that multiple cases are also covered.
+
     for s=1,#sequences do
         local dataset = datasets[s]
         if dataset then
@@ -10922,8 +11021,12 @@ local function featuresprocessor(head,font,attr)
                                         if lookupcache then
                                             local lookupmatch = lookupcache[start.char]
                                             if lookupmatch then
+                                                local headnode = start == head
                                                 start, success = handler(start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i)
                                                 if success then
+                                                    if headnode then
+                                                        head = start
+                                                    end
                                                     break
                                                 end
                                             end
@@ -10967,10 +11070,14 @@ local function featuresprocessor(head,font,attr)
                                             local lookupmatch = lookupcache[start.char]
                                             if lookupmatch then
                                                 -- sequence kan weg
+                                                local headnode = start == head
                                                 local ok
                                                 start, ok = handler(start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,1)
                                                 if ok then
                                                     success = true
+                                                    if headnode then
+                                                        head = start
+                                                    end
                                                 end
                                             end
                                             if start then start = start.next end
@@ -11040,10 +11147,14 @@ local function featuresprocessor(head,font,attr)
                                                 local lookupmatch = lookupcache[start.char]
                                                 if lookupmatch then
                                                     -- we could move all code inline but that makes things even more unreadable
+                                                    local headnode = start == head
                                                     local ok
                                                     start, ok = handler(start,dataset[4],lookupname,lookupmatch,sequence,lookuphash,i)
                                                     if ok then
                                                         success = true
+                                                        if headnode then
+                                                            head = start
+                                                        end
                                                         break
                                                     end
                                                 end
@@ -11370,6 +11481,10 @@ registerotffeature {
     }
 }
 
+-- this will change but is needed for an experiment:
+
+otf.handlers = handlers
+
 end -- closure
 
 do -- begin closure to overcome local limits and interference
@@ -12039,11 +12154,6 @@ local fontdata            = fonts.hashes.identifiers
 local state               = attributes.private('state')
 local categories          = characters and characters.categories or { } -- sorry, only in context
 
-local tracers             = nodes.tracers
-local colortracers        = tracers and tracers.colors
-local setnodecolor        = colortracers and colortracers.set   or function() end
-local resetnodecolor      = colortracers and colortracers.reset or function() end
-
 local otffeatures         = fonts.constructors.newfeatures("otf")
 local registerotffeature  = otffeatures.register
 
@@ -12116,14 +12226,14 @@ end
 local function analyzeinitializer(tfmdata,value) -- attr
     local script, language = otf.scriptandlanguage(tfmdata) -- attr
     local action = initializers[script]
-    if action then
-        if type(action) == "function" then
+    if not action then
+        -- skip
+    elseif type(action) == "function" then
+        return action(tfmdata,value)
+    else
+        local action = action[language]
+        if action then
             return action(tfmdata,value)
-        else
-            local action = action[language]
-            if action then
-                return action(tfmdata,value)
-            end
         end
     end
 end
@@ -12132,14 +12242,14 @@ local function analyzeprocessor(head,font,attr)
     local tfmdata = fontdata[font]
     local script, language = otf.scriptandlanguage(tfmdata,attr)
     local action = methods[script]
-    if action then
-        if type(action) == "function" then
+    if not action then
+        -- skip
+    elseif type(action) == "function" then
+        return action(head,font,attr)
+    else
+        action = action[language]
+        if action then
             return action(head,font,attr)
-        else
-            action = action[language]
-            if action then
-                return action(head,font,attr)
-            end
         end
     end
     return head, false
@@ -12251,7 +12361,6 @@ local isol_fina_medi_init = {
 
 local arab_warned = { }
 
-
 -- todo: gref
 
 local function warning(current,what)
@@ -12262,37 +12371,24 @@ local function warning(current,what)
     end
 end
 
-function methods.nocolor(head,font,attr)
-    for n in traverse_id(glyph_code,head) do
-        if not font or n.font == font then
-            resetnodecolor(n)
-        end
-    end
-    return head, true
-end
-
 local function finish(first,last)
     if last then
         if first == last then
             local fc = first.char
             if isol_fina_medi_init[fc] or isol_fina[fc] then
                 set_attribute(first,state,4) -- isol
-                if trace_analyzing then setnodecolor(first,"font:isol") end
             else
                 warning(first,"isol")
                 set_attribute(first,state,0) -- error
-                if trace_analyzing then resetnodecolor(first) end
             end
         else
             local lc = last.char
             if isol_fina_medi_init[lc] or isol_fina[lc] then -- why isol here ?
             -- if laststate == 1 or laststate == 2 or laststate == 4 then
                 set_attribute(last,state,3) -- fina
-                if trace_analyzing then setnodecolor(last,"font:fina") end
             else
                 warning(last,"fina")
                 set_attribute(last,state,0) -- error
-                if trace_analyzing then resetnodecolor(last) end
             end
         end
         first, last = nil, nil
@@ -12301,11 +12397,9 @@ local function finish(first,last)
         local fc = first.char
         if isol_fina_medi_init[fc] or isol_fina[fc] then
             set_attribute(first,state,4) -- isol
-            if trace_analyzing then setnodecolor(first,"font:isol") end
         else
             warning(first,"isol")
             set_attribute(first,state,0) -- error
-            if trace_analyzing then resetnodecolor(first) end
         end
         first = nil
     end
@@ -12323,20 +12417,16 @@ function methods.arab(head,font,attr) -- maybe make a special version with no tr
             local char = current.char
             if marks[char] or (useunicodemarks and categories[char] == "mn") then
                 set_attribute(current,state,5) -- mark
-                if trace_analyzing then setnodecolor(current,"font:mark") end
             elseif isol[char] then -- can be zwj or zwnj too
                 first, last = finish(first,last)
                 set_attribute(current,state,4) -- isol
-                if trace_analyzing then setnodecolor(current,"font:isol") end
                 first, last = nil, nil
             elseif not first then
                 if isol_fina_medi_init[char] then
                     set_attribute(current,state,1) -- init
-                    if trace_analyzing then setnodecolor(current,"font:init") end
                     first, last = first or current, current
                 elseif isol_fina[char] then
                     set_attribute(current,state,4) -- isol
-                    if trace_analyzing then setnodecolor(current,"font:isol") end
                     first, last = nil, nil
                 else -- no arab
                     first, last = finish(first,last)
@@ -12344,18 +12434,15 @@ function methods.arab(head,font,attr) -- maybe make a special version with no tr
             elseif isol_fina_medi_init[char] then
                 first, last = first or current, current
                 set_attribute(current,state,2) -- medi
-                if trace_analyzing then setnodecolor(current,"font:medi") end
             elseif isol_fina[char] then
                 if not has_attribute(last,state,1) then
                     -- tricky, we need to check what last may be !
                     set_attribute(last,state,2) -- medi
-                    if trace_analyzing then setnodecolor(last,"font:medi") end
                 end
                 set_attribute(current,state,3) -- fina
-                if trace_analyzing then setnodecolor(current,"font:fina") end
                 first, last = nil, nil
             elseif char >= 0x0600 and char <= 0x06FF then
-                if trace_analyzing then setnodecolor(current,"font:rest") end
+                set_attribute(current,state,6) -- rest
                 first, last = finish(first,last)
             else --no
                 first, last = finish(first,last)
@@ -12989,18 +13076,14 @@ local otffeatures = fonts.constructors.newfeatures("otf")
 
 local function initializeitlc(tfmdata,value)
     if value then
-        -- the magic 40 and it formula come from Dohyun Kim
-        local parameters  = tfmdata.parameters
+        -- the magic 40 and it formula come from Dohyun Kim but we might need another guess
+        local parameters = tfmdata.parameters
         local italicangle = parameters.italicangle
         if italicangle and italicangle ~= 0 then
-            local uwidth = (parameters.uwidth or 40)/2
-            for unicode, d in next, tfmdata.descriptions do
-                local it = d.boundingbox[3] - d.width + uwidth
-                if it ~= 0 then
-                    d.italic = it
-                end
-            end
-            tfmdata.properties.hasitalics = true
+            local properties = tfmdata.properties
+            local factor = tonumber(value) or 1
+            properties.hasitalics = true
+            properties.autoitalicamount = factor * (parameters.uwidth or 40)/2
         end
     end
 end
-- 
cgit v1.2.3