Updating to latest ConTeXt stable (2010.05.08)

13 files changed, 442 insertions, 269 deletions

diff --git a/otfl-font-def.lua b/otfl-font-def.lua
index 0cbff06..f64bae5 100644
--- a/otfl-font-def.lua
+++ b/otfl-font-def.lua
@@ -534,8 +534,10 @@ function define.register(fontdata,id)
             if trace_defining then
                 logs.report("define font","loading at 2 id %s, hash: %s",id or "?",hash or "?")
             end
-            fonts.ids[id] = fontdata
-            fonts.chr[id] = fontdata.characters
+            fonts.identifiers[id] = fontdata
+            fonts.characters [id] = fontdata.characters
+            fonts.quads      [id] = fontdata.parameters.quad
+            -- todo: extra functions, e.g. setdigitwidth etc in list
             tfm.internalized[hash] = id
         end
     end
diff --git a/otfl-font-dum.lua b/otfl-font-dum.lua
index 1d79ab1..5224ae1 100644
--- a/otfl-font-dum.lua
+++ b/otfl-font-dum.lua
@@ -346,3 +346,13 @@ fonts.protrusions.setups['default'] = {
     [0x06D4] = { 0, 1    }, -- arabic full stop       ۔
 
 }
+
+-- normalizer
+
+fonts.otf.meanings = fonts.otf.meanings or { }
+
+fonts.otf.meanings.normalize = fonts.otf.meanings.normalize or function(t)
+    if t.rand then
+        t.rand = "random"
+    end
+end
diff --git a/otfl-font-ini.lua b/otfl-font-ini.lua
index bcf46ad..b998868 100644
--- a/otfl-font-ini.lua
+++ b/otfl-font-ini.lua
@@ -22,8 +22,11 @@ fontloader.totable = fontloader.to_table
 -- fix comes last
 
 fonts     = fonts     or { }
-fonts.ids = fonts.ids or { } -- aka fontdata
-fonts.chr = fonts.chr or { } -- aka chardata
+
+fonts.ids = fonts.ids or { } fonts.identifiers = fonts.ids -- aka fontdata
+fonts.chr = fonts.chr or { } fonts.characters  = fonts.chr -- aka chardata
+fonts.qua = fonts.qua or { } fonts.quads       = fonts.qua -- aka quaddata
+
 fonts.tfm = fonts.tfm or { }
 
 fonts.mode    = 'base'
diff --git a/otfl-font-map.lua b/otfl-font-map.lua
index 9dd2bd0..208ad50 100644
--- a/otfl-font-map.lua
+++ b/otfl-font-map.lua
@@ -22,116 +22,10 @@ of obsolete. Some code may move to runtime or auxiliary modules.</p>
 <p>The name to unciode related code will stay of course.</p>
 --ldx]]--
 
-fonts               = fonts               or { }
-fonts.map           = fonts.map           or { }
-fonts.map.data      = fonts.map.data      or { }
-fonts.map.encodings = fonts.map.encodings or { }
-fonts.map.done      = fonts.map.done      or { }
-fonts.map.loaded    = fonts.map.loaded    or { }
-fonts.map.direct    = fonts.map.direct    or { }
-fonts.map.line      = fonts.map.line      or { }
+fonts     = fonts     or { }
+fonts.map = fonts.map or { }
 
-function fonts.map.line.pdfmapline(tag,str)
-    return "\\loadmapline[" .. tag .. "][" .. str .. "]"
-end
-
-function fonts.map.line.pdftex(e) -- so far no combination of slant and extend
-    if e.name and e.fontfile then
-        local fullname = e.fullname or ""
-        if e.slant and e.slant ~= 0 then
-            if e.encoding then
-                return fonts.map.line.pdfmapline("=",format('%s %s "%g SlantFont" <%s <%s',e.name,fullname,e.slant,e.encoding,e.fontfile))
-            else
-                return fonts.map.line.pdfmapline("=",format('%s %s "%g SlantFont" <%s',e.name,fullname,e.slant,e.fontfile))
-            end
-        elseif e.extend and e.extend ~= 1 and e.extend ~= 0 then
-            if e.encoding then
-                return fonts.map.line.pdfmapline("=",format('%s %s "%g ExtendFont" <%s <%s',e.name,fullname,e.extend,e.encoding,e.fontfile))
-            else
-                return fonts.map.line.pdfmapline("=",format('%s %s "%g ExtendFont" <%s',e.name,fullname,e.extend,e.fontfile))
-            end
-        else
-            if e.encoding then
-                return fonts.map.line.pdfmapline("=",format('%s %s <%s <%s',e.name,fullname,e.encoding,e.fontfile))
-            else
-                return fonts.map.line.pdfmapline("=",format('%s %s <%s',e.name,fullname,e.fontfile))
-            end
-        end
-    else
-        return nil
-    end
-end
-
-function fonts.map.flush(backend) -- will also erase the accumulated data
-    local flushline = fonts.map.line[backend or "pdftex"] or fonts.map.line.pdftex
-    for _, e in pairs(fonts.map.data) do
-        tex.sprint(ctxcatcodes,flushline(e))
-    end
-    fonts.map.data = { }
-end
-
-fonts.map.line.dvips     = fonts.map.line.pdftex
-fonts.map.line.dvipdfmx  = function() end
-
-function fonts.map.convert_entries(filename)
-    if not fonts.map.loaded[filename] then
-        fonts.map.data, fonts.map.encodings = fonts.map.load_file(filename,fonts.map.data, fonts.map.encodings)
-        fonts.map.loaded[filename] = true
-    end
-end
-
-function fonts.map.load_file(filename, entries, encodings)
-    entries   = entries   or { }
-    encodings = encodings or { }
-    local f = io.open(filename)
-    if f then
-        local data = f:read("*a")
-        if data then
-            for line in gmatch(data,"(.-)[\n\t]") do
-                if find(line,"^[%#%%%s]") then
-                    -- print(line)
-                else
-                    local extend, slant, name, fullname, fontfile, encoding
-                    line = gsub(line,'"(.+)"', function(s)
-                        extend = find(s,'"([^"]+) ExtendFont"')
-                        slant = find(s,'"([^"]+) SlantFont"')
-                        return ""
-                    end)
-                    if not name then
-                        -- name fullname encoding fontfile
-                        name, fullname, encoding, fontfile = match(line,"^(%S+)%s+(%S*)[%s<]+(%S*)[%s<]+(%S*)%s*$")
-                    end
-                    if not name then
-                        -- name fullname (flag) fontfile encoding
-                        name, fullname, fontfile, encoding = match(line,"^(%S+)%s+(%S*)[%d%s<]+(%S*)[%s<]+(%S*)%s*$")
-                    end
-                    if not name then
-                        -- name fontfile
-                        name, fontfile = match(line,"^(%S+)%s+[%d%s<]+(%S*)%s*$")
-                    end
-                    if name then
-                        if encoding == "" then encoding = nil end
-                        entries[name] = {
-                            name     = name, -- handy
-                            fullname = fullname,
-                            encoding = encoding,
-                            fontfile = fontfile,
-                            slant    = tonumber(slant),
-                            extend   = tonumber(extend)
-                        }
-                        encodings[name] = encoding
-                    elseif line ~= "" then
-                    --  print(line)
-                    end
-                end
-            end
-        end
-        f:close()
-    end
-    return entries, encodings
-end
-
-local function load_lum_table(filename)
+local function load_lum_table(filename) -- will move to font goodies
     local lumname = file.replacesuffix(file.basename(filename),"lum")
     local lumfile = resolvers.find_file(lumname,"map") or ""
     if lumfile ~= "" and lfs.isfile(lumfile) then
@@ -371,3 +265,106 @@ fonts.map.add_to_unicode = function(data,filename)
         logs.report("load otf","enhance: %s tounicode entries added (%s ligatures)",nl+ns, ns)
     end
 end
+
+-- the following is sort of obsolete
+--
+-- fonts.map.data      = fonts.map.data      or { }
+-- fonts.map.encodings = fonts.map.encodings or { }
+-- fonts.map.loaded    = fonts.map.loaded    or { }
+-- fonts.map.line      = fonts.map.line      or { }
+--
+-- function fonts.map.line.pdftex(e)
+--     if e.name and e.fontfile then
+--         local fullname = e.fullname or ""
+--         if e.slant and e.slant ~= 0 then
+--             if e.encoding then
+--                 pdf.mapline(format('= %s %s "%g SlantFont" <%s <%s',e.name,fullname,e.slant,e.encoding,e.fontfile)))
+--             else
+--                 pdf.mapline(format('= %s %s "%g SlantFont" <%s',e.name,fullname,e.slant,e.fontfile)))
+--             end
+--         elseif e.extend and e.extend ~= 1 and e.extend ~= 0 then
+--             if e.encoding then
+--                 pdf.mapline(format('= %s %s "%g ExtendFont" <%s <%s',e.name,fullname,e.extend,e.encoding,e.fontfile)))
+--             else
+--                 pdf.mapline(format('= %s %s "%g ExtendFont" <%s',e.name,fullname,e.extend,e.fontfile)))
+--             end
+--         else
+--             if e.encoding then
+--                 pdf.mapline(format('= %s %s <%s <%s',e.name,fullname,e.encoding,e.fontfile)))
+--             else
+--                 pdf.mapline(format('= %s %s <%s',e.name,fullname,e.fontfile)))
+--             end
+--         end
+--     else
+--         return nil
+--     end
+-- end
+--
+-- function fonts.map.flush(backend) -- will also erase the accumulated data
+--     local flushline = fonts.map.line[backend or "pdftex"] or fonts.map.line.pdftex
+--     for _, e in pairs(fonts.map.data) do
+--         flushline(e)
+--     end
+--     fonts.map.data = { }
+-- end
+--
+-- fonts.map.line.dvips     = fonts.map.line.pdftex
+-- fonts.map.line.dvipdfmx  = function() end
+--
+-- function fonts.map.convert_entries(filename)
+--     if not fonts.map.loaded[filename] then
+--         fonts.map.data, fonts.map.encodings = fonts.map.load_file(filename,fonts.map.data, fonts.map.encodings)
+--         fonts.map.loaded[filename] = true
+--     end
+-- end
+--
+-- function fonts.map.load_file(filename, entries, encodings)
+--     entries   = entries   or { }
+--     encodings = encodings or { }
+--     local f = io.open(filename)
+--     if f then
+--         local data = f:read("*a")
+--         if data then
+--             for line in gmatch(data,"(.-)[\n\t]") do
+--                 if find(line,"^[%#%%%s]") then
+--                     -- print(line)
+--                 else
+--                     local extend, slant, name, fullname, fontfile, encoding
+--                     line = gsub(line,'"(.+)"', function(s)
+--                         extend = find(s,'"([^"]+) ExtendFont"')
+--                         slant = find(s,'"([^"]+) SlantFont"')
+--                         return ""
+--                     end)
+--                     if not name then
+--                         -- name fullname encoding fontfile
+--                         name, fullname, encoding, fontfile = match(line,"^(%S+)%s+(%S*)[%s<]+(%S*)[%s<]+(%S*)%s*$")
+--                     end
+--                     if not name then
+--                         -- name fullname (flag) fontfile encoding
+--                         name, fullname, fontfile, encoding = match(line,"^(%S+)%s+(%S*)[%d%s<]+(%S*)[%s<]+(%S*)%s*$")
+--                     end
+--                     if not name then
+--                         -- name fontfile
+--                         name, fontfile = match(line,"^(%S+)%s+[%d%s<]+(%S*)%s*$")
+--                     end
+--                     if name then
+--                         if encoding == "" then encoding = nil end
+--                         entries[name] = {
+--                             name     = name, -- handy
+--                             fullname = fullname,
+--                             encoding = encoding,
+--                             fontfile = fontfile,
+--                             slant    = tonumber(slant),
+--                             extend   = tonumber(extend)
+--                         }
+--                         encodings[name] = encoding
+--                     elseif line ~= "" then
+--                     --  print(line)
+--                     end
+--                 end
+--             end
+--         end
+--         f:close()
+--     end
+--     return entries, encodings
+-- end
diff --git a/otfl-font-ota.lua b/otfl-font-ota.lua
index e2fa3f2..558e2fc 100644
--- a/otfl-font-ota.lua
+++ b/otfl-font-ota.lua
@@ -41,6 +41,8 @@ local insert_node_after  = node.insert_after
 local insert_node_before = node.insert_before
 local traverse_node_list = node.traverse
 
+local new_glue_node      = nodes.glue
+
 local fontdata = fonts.ids
 local state    = attributes.private('state')
 
@@ -302,11 +304,12 @@ function fonts.analyzers.methods.arab(head,font,attr) -- maybe make a special ve
     end
     first, last = finish(first,last)
     if removejoiners then
+        -- is never head
         for i=1,#joiners do
-            head = delete_node(head,joiners[i])
+            delete_node(head,joiners[i])
         end
         for i=1,#nonjoiners do
-            head = replace_node(head,nonjoiners[i],nodes.glue(0)) -- or maybe a kern
+            replace_node(head,nonjoiners[i],new_glue_node(0)) -- or maybe a kern
         end
     end
     return head, done
diff --git a/otfl-font-otf.lua b/otfl-font-otf.lua
index 9aa1a3a..29c56cd 100644
--- a/otfl-font-otf.lua
+++ b/otfl-font-otf.lua
@@ -247,6 +247,8 @@ function otf.load(filename,format,sub,featurefile)
                     logs.report("load otf","warning: %s",tostring(messages[m]))
                 end
             end
+        else
+            logs.report("load otf","font loaded okay")
         end
         if ff then
             load_featurefile(ff,featurefile)
diff --git a/otfl-font-otn.lua b/otfl-font-otn.lua
index 16ecc2d..3aa1927 100644
--- a/otfl-font-otn.lua
+++ b/otfl-font-otn.lua
@@ -10,6 +10,9 @@ if not modules then modules = { } end modules ['font-otn'] = {
 -- much functionality could only be implemented thanks to the husayni font
 -- of Idris Samawi Hamid to who we dedicate this module.
 
+-- I'm in the process of cleaning up the code (which happens in another
+-- file) so don't rely on things staying the same.
+
 -- some day when we can jit this, we can use more functions
 
 -- we can use more lpegs when lpeg is extended with function args and so
@@ -1538,6 +1541,7 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
     local skipmark, skipligature, skipbase = flags[1], flags[2], flags[3]
     local someskip = skipmark or skipligature or skipbase -- could be stored in flags for a fast test (hm, flags could be false !)
     local markclass = sequence.markclass -- todo, first we need a proper test
+    local skipped = false
     for k=1,#contexts do
         local match, current, last = true, start, start
         local ck = contexts[k]
@@ -1572,6 +1576,7 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                                     if ccd then
                                         local class = ccd.class
                                         if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
+                                            skipped = true
                                             if trace_skips then
                                                 show_skip(kind,chainname,char,ck,class)
                                             end
@@ -1616,6 +1621,7 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                                     if ccd then
                                         local class = ccd.class
                                         if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
+                                            skipped = true
                                             if trace_skips then
                                                 show_skip(kind,chainname,char,ck,class)
                                             end
@@ -1670,6 +1676,7 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                                     if ccd then
                                         local class = ccd.class
                                         if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
+                                            skipped = true
                                             if trace_skips then
                                                 show_skip(kind,chainname,char,ck,class)
                                             end
@@ -1735,8 +1742,47 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                     end
                  else
                     -- actually this needs a more complex treatment for which we will use chainmores
+--~                     local i = 1
+--~                     repeat
+--~                         local chainlookupname = chainlookups[i]
+--~                         local chainlookup = lookuptable[chainlookupname]
+--~                         local cp = chainmores[chainlookup.type]
+--~                         if cp then
+--~                             local ok, n
+--~                             start, ok, n = cp(start,last,kind,chainname,ck,cache,chainlookup,chainlookupname,i,sequence)
+--~                             -- messy since last can be changed !
+--~                             if ok then
+--~                                 done = true
+--~                                 start = start.next
+--~                                 if n then
+--~                                     -- skip next one(s) if ligature
+--~                                     i = i + n - 1
+--~                                 end
+--~                             end
+--~                         else
+--~                             logprocess("%s: multiple subchains for %s are not yet supported",cref(kind,chainname,chainlookupname),chainlookup.type)
+--~                         end
+--~                         i = i + 1
+--~                     until i > nofchainlookups
+
                     local i = 1
                     repeat
+if skipped then
+    while true do
+        local char = start.char
+        local ccd = descriptions[char]
+        if ccd then
+            local class = ccd.class
+            if class == skipmark or class == skipligature or class == skipbase or (markclass and class == "mark" and not markclass[char]) then
+                start = start.next
+            else
+                break
+            end
+        else
+            break
+        end
+    end
+end
                         local chainlookupname = chainlookups[i]
                         local chainlookup = lookuptable[chainlookupname]
                         local cp = chainmores[chainlookup.type]
@@ -1746,17 +1792,18 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                             -- messy since last can be changed !
                             if ok then
                                 done = true
-                                start = start.next
-                                if n then
-                                    -- skip next one(s) if ligature
-                                    i = i + n - 1
-                                end
+                                -- skip next one(s) if ligature
+                                i = i + (n or 1)
+                            else
+                                i = i + 1
                             end
                         else
                             logprocess("%s: multiple subchains for %s are not yet supported",cref(kind,chainname,chainlookupname),chainlookup.type)
+                            i = i + 1
                         end
-                        i = i + 1
+                        start = start.next
                     until i > nofchainlookups
+
                 end
             else
                 local replacements = ck[7]
@@ -1835,9 +1882,6 @@ local resolved = { } -- we only resolve a font,script,language pair once
 
 -- todo: pass all these 'locals' in a table
 
--- maybe some day i'll make an alternative that works on 'sub direction runs' which might be
--- more efficient for arabic but it has quite some consequences
-
 function fonts.methods.node.otf.features(head,font,attr)
     if trace_steps then
         checkstep(head)
@@ -1881,7 +1925,7 @@ function fonts.methods.node.otf.features(head,font,attr)
     local ra  = rl      [attr]     if ra == nil then ra  = { } rl      [attr]     = ra  end -- attr can be false
     -- sequences always > 1 so no need for optimization
     for s=1,#sequences do
-    local pardir, txtdir = 0, { }
+        local pardir, txtdir = 0, { }
         local success = false
         local sequence = sequences[s]
         local r = ra[s] -- cache
@@ -1907,12 +1951,10 @@ function fonts.methods.node.otf.features(head,font,attr)
                             -- only first attribute match check, so we assume simple fina's
                             -- default can become a font feature itself
                             if l[language] then
---~                                 valid, what = true, language
                                 valid, what = s_e or a_e, language
                         --  elseif l[default] then
                         --      valid, what = true, default
                             elseif l[wildcard] then
---~                                 valid, what = true, wildcard
                                 valid, what = s_e or a_e, wildcard
                             end
                             if valid then
@@ -1947,12 +1989,12 @@ function fonts.methods.node.otf.features(head,font,attr)
                 local handler = handlers[typ]
                 local thecache = featuredata[typ] or { }
                 -- we need to get rid of this slide !
-                start = find_node_tail(head) -- slow (we can store tail because there's always a skip at the end): todo
+                local start = find_node_tail(head) -- slow (we can store tail because there's always a skip at the end): todo
                 while start do
                     local id = start.id
                     if id == glyph then
---~                         if start.subtype<256 and start.font == font and (not attr or has_attribute(start,0,attr)) then
-                        if start.subtype<256 and start.font == font and has_attribute(start,0,attr) then
+                        if start.subtype<256 and start.font == font and (not attr or has_attribute(start,0,attr)) then
+--~                         if start.subtype<256 and start.font == font and has_attribute(start,0,attr) then
                             for i=1,#subtables do
                                 local lookupname = subtables[i]
                                 local lookupcache = thecache[lookupname]
@@ -1980,7 +2022,7 @@ function fonts.methods.node.otf.features(head,font,attr)
                 local handler = handlers[typ]
                 local ns = #subtables
                 local thecache = featuredata[typ] or { }
-                start = head -- local ?
+                local start = head -- local ?
                 rlmode = 0 -- to be checked ?
                 if ns == 1 then
                     local lookupname = subtables[1]
@@ -1988,19 +2030,16 @@ function fonts.methods.node.otf.features(head,font,attr)
                     if not lookupcache then
                         report_missing_cache(typ,lookupname)
                     else
---~ print(typ,lookupname,lookupcache,table.serialize(lookupcache))
                         while start do
                             local id = start.id
                             if id == glyph then
---~                                 if start.font == font and start.subtype<256 and (not attr or has_attribute(start,0,attr)) and (not attribute or has_attribute(start,state,attribute)) then
-                                if start.font == font and start.subtype<256 and has_attribute(start,0,attr) and (not attribute or has_attribute(start,state,attribute)) then
+--~                                 if start.font == font and start.subtype<256 and has_attribute(start,0,attr) and (not attribute or has_attribute(start,state,attribute)) then
+                                if start.font == font and start.subtype<256 and (not attr or has_attribute(start,0,attr)) and (not attribute or has_attribute(start,state,attribute)) then
                                     local lookupmatch = lookupcache[start.char]
                                     if lookupmatch then
                                         -- sequence kan weg
                                         local ok
---~ print("!!!")
                                         start, ok = handler(start,r[4],lookupname,lookupmatch,sequence,featuredata,1)
---~ texio.write_nl(tostring(lookupname),tostring(lookupmatch),tostring(ok))
                                         if ok then
                                             success = true
                                         end
@@ -2025,25 +2064,6 @@ function fonts.methods.node.otf.features(head,font,attr)
                             --         start = start.next
                             --     end
                             elseif id == whatsit then
---~                             if subtype == 7 then
---~                                 local dir = start.dir
---~                                 if dir == "+TRT" then
---~                                     rlmode = -1
---~                                 elseif dir == "+TLT" then
---~                                     rlmode = 1
---~                                 else
---~                                     rlmode = 0
---~                                 end
---~                             elseif subtype == 6 then
---~                                 local dir = start.dir
---~                                 if dir == "TRT" then
---~                                     rlmode = -1
---~                                 elseif dir == "TLT" then
---~                                     rlmode = 1
---~                                 else
---~                                     rlmode = 0
---~                                 end
---~                             end
                                 local subtype = start.subtype
                                 if subtype == 7 then
                                     local dir = start.dir
@@ -2088,8 +2108,8 @@ function fonts.methods.node.otf.features(head,font,attr)
                     while start do
                         local id = start.id
                         if id == glyph then
---~                             if start.subtype<256 and start.font == font and (not attr or has_attribute(start,0,attr)) and (not attribute or has_attribute(start,state,attribute)) then
-                            if start.subtype<256 and start.font == font and has_attribute(start,0,attr) and (not attribute or has_attribute(start,state,attribute)) then
+                            if start.subtype<256 and start.font == font and (not attr or has_attribute(start,0,attr)) and (not attribute or has_attribute(start,state,attribute)) then
+--~                             if start.subtype<256 and start.font == font and has_attribute(start,0,attr) and (not attribute or has_attribute(start,state,attribute)) then
                                 for i=1,ns do
                                     local lookupname = subtables[i]
                                     local lookupcache = thecache[lookupname]
@@ -2129,59 +2149,40 @@ function fonts.methods.node.otf.features(head,font,attr)
                         --     end
                         elseif id == whatsit then
                             local subtype = start.subtype
---~                             if subtype == 7 then
---~                                 local dir = start.dir
---~                                 if dir == "+TRT" then
---~                                     rlmode = -1
---~                                 elseif dir == "+TLT" then
---~                                     rlmode = 1
---~                                 else
---~                                     rlmode = 0
---~                                 end
---~                             elseif subtype == 6 then
---~                                 local dir = start.dir
---~                                 if dir == "TRT" then
---~                                     rlmode = -1
---~                                 elseif dir == "TLT" then
---~                                     rlmode = 1
---~                                 else
---~                                     rlmode = 0
---~                                 end
---~                             end
-                                local subtype = start.subtype
-                                if subtype == 7 then
-                                    local dir = start.dir
-                                    if     dir == "+TRT" or dir == "+TLT" then
-                                        insert(txtdir,dir)
-                                    elseif dir == "-TRT" or dir == "-TLT" then
-                                        remove(txtdir)
-                                    end
-                                    local d = txtdir[#txtdir]
-                                    if d == "+TRT" then
-                                        rlmode = -1
-                                    elseif d == "+TLT" then
-                                        rlmode = 1
-                                    else
-                                        rlmode = pardir
-                                    end
-                                    if trace_directions then
-                                        logs.report("fonts","directions after textdir %s: pardir=%s, txtdir=%s:%s, rlmode=%s",dir,pardir,#txtdir,txtdir[#txtdir] or "unset",rlmode)
-                                    end
-                                elseif subtype == 6 then
-                                    local dir = start.dir
-                                    if dir == "TRT" then
-                                        pardir = -1
-                                    elseif dir == "TLT" then
-                                        pardir = 1
-                                    else
-                                        pardir = 0
-                                    end
+                            local subtype = start.subtype
+                            if subtype == 7 then
+                                local dir = start.dir
+                                if     dir == "+TRT" or dir == "+TLT" then
+                                    insert(txtdir,dir)
+                                elseif dir == "-TRT" or dir == "-TLT" then
+                                    remove(txtdir)
+                                end
+                                local d = txtdir[#txtdir]
+                                if d == "+TRT" then
+                                    rlmode = -1
+                                elseif d == "+TLT" then
+                                    rlmode = 1
+                                else
                                     rlmode = pardir
-                                --~ txtdir = { }
+                                end
                                 if trace_directions then
-                                    logs.report("fonts","directions after pardir %s: pardir=%s, txtdir=%s:%s, rlmode=%s",dir,pardir,#txtdir,txtdir[#txtdir] or "unset",rlmode)
+                                    logs.report("fonts","directions after textdir %s: pardir=%s, txtdir=%s:%s, rlmode=%s",dir,pardir,#txtdir,txtdir[#txtdir] or "unset",rlmode)
+                                end
+                            elseif subtype == 6 then
+                                local dir = start.dir
+                                if dir == "TRT" then
+                                    pardir = -1
+                                elseif dir == "TLT" then
+                                    pardir = 1
+                                else
+                                    pardir = 0
                                 end
+                                rlmode = pardir
+                            --~ txtdir = { }
+                                if trace_directions then
+                                    logs.report("fonts","directions after pardir %s: pardir=%s, txtdir=%s:%s, rlmode=%s",dir,pardir,#txtdir,txtdir[#txtdir] or "unset",rlmode)
                                 end
+                            end
                             start = start.next
                         else
                             start = start.next
diff --git a/otfl-font-ott.lua b/otfl-font-ott.lua
index 55cb734..d26c55f 100644
--- a/otfl-font-ott.lua
+++ b/otfl-font-ott.lua
@@ -682,6 +682,14 @@ for k, v in pairs(to_features) do
     to_features[lower(k)] = v
 end
 
+otf.meanings.checkers = {
+    rand = function(v)
+        return v and "random"
+    end
+}
+
+local checkers = otf.meanings.checkers
+
 function otf.meanings.normalize(features)
     local h = { }
     for k,v in next, features do
@@ -710,7 +718,9 @@ function otf.meanings.normalize(features)
                     v = b
                 end
             end
-            h[to_features[k] or k] = v
+            k = to_features[k] or k
+            local c = checkers[k]
+            h[k] = c and c(v) or v
         end
     end
     return h
diff --git a/otfl-font-tfm.lua b/otfl-font-tfm.lua
index aa87398..4b217a3 100644
--- a/otfl-font-tfm.lua
+++ b/otfl-font-tfm.lua
@@ -205,29 +205,21 @@ local charactercache = { }
 -- a virtual font has italic correction make sure to set the
 -- has_italic flag. Some more flags will be added in the future.
 
-
 function tfm.calculate_scale(tfmtable, scaledpoints, relativeid)
     if scaledpoints < 0 then
         scaledpoints = (- scaledpoints/1000) * tfmtable.designsize -- already in sp
     end
-    local delta = scaledpoints/(tfmtable.units or 1000) -- brr, some open type fonts have 2048
-    return scaledpoints, delta
+    local units = tfmtable.units or 1000
+    local delta = scaledpoints/units -- brr, some open type fonts have 2048
+    return scaledpoints, delta, units
 end
 
 function tfm.do_scale(tfmtable, scaledpoints, relativeid)
  -- tfm.prepare_base_kerns(tfmtable) -- optimalization
-    local scaledpoints, delta = tfm.calculate_scale(tfmtable, scaledpoints, relativeid)
-    if enable_auto_r_scale and relativeid then -- for the moment this is rather context specific
-        local relativedata = fontdata[relativeid]
-        local id_x_height = relativedata and relativedata.parameters and relativedata.parameters.x_height
-        local tf_x_height = id_x_height and tfmtable.parameters and tfmtable.parameters.x_height * delta
-        if tf_x_height then
-            scaledpoints = (id_x_height/tf_x_height) * scaledpoints
-            delta = scaledpoints/(tfmtable.units or 1000)
-        end
-    end
+    local t = { } -- the new table
+    local scaledpoints, delta, units = tfm.calculate_scale(tfmtable, scaledpoints, relativeid)
+    t.units_per_em = units or 1000
     local hdelta, vdelta = delta, delta
-    local t = { }
     -- unicoded unique descriptions shared cidinfo characters changed parameters indices
     for k,v in next, tfmtable do
         if type(v) == "table" then
@@ -502,6 +494,8 @@ t.colorscheme = tfmtable.colorscheme
             local vc = v.commands
             if vc then
                 -- we assume non scaled commands here
+                -- tricky .. we need to scale pseudo math glyphs too
+                -- which is why we deal with rules too
                 local ok = false
                 for i=1,#vc do
                     local key = vc[i][1]
@@ -519,6 +513,8 @@ t.colorscheme = tfmtable.colorscheme
                             tt[#tt+1] = { key, ivc[2]*hdelta }
                         elseif key == "down" then
                             tt[#tt+1] = { key, ivc[2]*vdelta }
+                        elseif key == "rule" then
+                            tt[#tt+1] = { key, ivc[2]*vdelta, ivc[3]*hdelta }
                         else -- not comment
                             tt[#tt+1] = ivc -- shared since in cache and untouched
                         end
diff --git a/otfl-font-xtx.lua b/otfl-font-xtx.lua
index fd0a474..0fa2d69 100644
--- a/otfl-font-xtx.lua
+++ b/otfl-font-xtx.lua
@@ -189,6 +189,8 @@ local option     = spaces * (keyvalue + falsevalue + truevalue + somevalue) * sp
 local options    = lpeg.P(":") * spaces * (lpeg.P(";")^0  * option)^0
 local pattern    = (filename + fontname) * subvalue^0 * crapspec^0 * options^0
 
+local normalize_meanings = fonts.otf.meanings.normalize
+
 function fonts.define.specify.colonized(specification) -- xetex mode
     list = { }
     lpegmatch(pattern,specification.specification)
@@ -212,7 +214,8 @@ function fonts.define.specify.colonized(specification) -- xetex mode
         specification.sub = list.sub
         list.sub = nil
     end
-    specification.features.normal = list
+--  specification.features.normal = list
+    specification.features.normal = normalize_meanings(list)
     return specification
 end
 
diff --git a/otfl-node-fnt.lua b/otfl-node-fnt.lua
index 72bb140..f2d8e1d 100644
--- a/otfl-node-fnt.lua
+++ b/otfl-node-fnt.lua
@@ -51,7 +51,7 @@ function nodes.process_characters(head)
     local usedfonts, attrfonts, done = { }, { }, false
     local a, u, prevfont, prevattr = 0, 0, nil, 0
     for n in traverse_id(glyph,head) do
-        local font, attr = n.font, has_attribute(n,0) -- zero attribute is reserved for fonts, preset to 0 is faster (first match)
+        local font, attr = n.font, has_attribute(n,0) -- zero attribute is reserved for fonts in context
         if attr and attr > 0 then
             if font ~= prevfont or attr ~= prevattr then
                 local used = attrfonts[font]
@@ -107,7 +107,7 @@ function nodes.process_characters(head)
             head, done = h or head, done or d
             if n > 1 then
                 for i=2,n do
-                    local h, d = processors[i](head,font,0) -- false)
+                    local h, d = processors[i](head,font,false)
                     head, done = h or head, done or d
                 end
             end
@@ -119,7 +119,7 @@ function nodes.process_characters(head)
             head, done = h or head, done or d
             if n > 1 then
                 for i=2,n do
-                    local h, d = processors[i](head,font,0) -- false)
+                    local h, d = processors[i](head,font,false)
                     head, done = h or head, done or d
                 end
             end
diff --git a/otfl-node-ini.lua b/otfl-node-ini.lua
index 741e53d..36e2402 100644
--- a/otfl-node-ini.lua
+++ b/otfl-node-ini.lua
@@ -108,10 +108,12 @@ local penalty = node.id('penalty')
 local kern    = node.id('kern')
 local whatsit = node.id('whatsit')
 
-local traverse_id = node.traverse_id
-local traverse    = node.traverse
-local free_node   = node.free
-local remove_node = node.remove
+local traverse_id        = node.traverse_id
+local traverse           = node.traverse
+local free_node          = node.free
+local remove_node        = node.remove
+local insert_node_before = node.insert_before
+local insert_node_after  = node.insert_after
 
 function nodes.remove(head, current, free_too)
    local t = current
@@ -131,8 +133,8 @@ function nodes.delete(head,current)
     return nodes.remove(head,current,true)
 end
 
-nodes.before = node.insert_before
-nodes.after  = node.insert_after
+nodes.before = insert_node_before
+nodes.after  = insert_node_after
 
 -- we need to test this, as it might be fixed now
 
@@ -172,21 +174,31 @@ function nodes.after(h,c,n)
     return n, n
 end
 
-function nodes.replace(head,current,new)
-    if current and next then
-        local p, n = current.prev, current.next
-        new.prev, new.next = p, n
-        if p then
-            p.next = new
-        else
+-- local h, c = nodes.replace(head,current,new)
+-- local c = nodes.replace(false,current,new)
+-- local c = nodes.replace(current,new)
+
+function nodes.replace(head,current,new) -- no head returned if false
+    if not new then
+        head, current, new = false, head, current
+    end
+    local prev, next = current.prev, current.next
+    if next then
+        new.next, next.prev = next, new
+    end
+    if prev then
+        new.prev, prev.next = prev, new
+    end
+    if head then
+        if head == current then
             head = new
         end
-        if n then
-            n.prev = new
-        end
         free_node(current)
+        return head, new
+    else
+        free_node(current)
+        return new
     end
-    return head, current
 end
 
 -- will move
@@ -212,7 +224,7 @@ end
 
 nodes.count = count
 
--- new
+-- new, will move
 
 function attributes.ofnode(n)
     local a = n.attr
diff --git a/otfl-node-res.lua b/otfl-node-res.lua
index 5398d84..2657dfa 100644
--- a/otfl-node-res.lua
+++ b/otfl-node-res.lua
@@ -7,7 +7,10 @@ if not modules then modules = { } end modules ['node-res'] = {
 }
 
 local gmatch, format = string.gmatch, string.format
-local copy_node, free_node, free_list, new_node = node.copy, node.free, node.flush_list, node.new
+local copy_node, free_node, free_list, new_node, node_type, node_id = node.copy, node.free, node.flush_list, node.new, node.type, node.id
+local tonumber, round = tonumber, math.round
+
+local glyph_node = node_id("glyph")
 
 --[[ldx--
 <p>The next function is not that much needed but in <l n='context'/> we use
@@ -25,16 +28,21 @@ for k, v in pairs(node.whatsits()) do
     whatsits[k], whatsits[v] = v, k -- two way
 end
 
-function nodes.register(n)
+local function register_node(n)
     reserved[#reserved+1] = n
     return n
 end
 
+nodes.register = register_node
+
 function nodes.cleanup_reserved(nofboxes) -- todo
     nodes.tracers.steppers.reset() -- todo: make a registration subsystem
     local nr, nl = #reserved, 0
     for i=1,nr do
-        free_node(reserved[i])
+        local ri = reserved[i]
+    --  if not (ri.id == glue_spec and not ri.is_writable) then
+            free_node(reserved[i])
+    --  end
     end
     if nofboxes then
         local tb = tex.box
@@ -58,19 +66,37 @@ function nodes.usage()
     return t
 end
 
-local disc       = nodes.register(new_node("disc"))
-local kern       = nodes.register(new_node("kern",1))
-local penalty    = nodes.register(new_node("penalty"))
-local glue       = nodes.register(new_node("glue"))
-local glue_spec  = nodes.register(new_node("glue_spec"))
-local glyph      = nodes.register(new_node("glyph",0))
-local textdir    = nodes.register(new_node("whatsit",whatsits.dir)) -- 7
-local rule       = nodes.register(new_node("rule"))
-local latelua    = nodes.register(new_node("whatsit",whatsits.late_lua)) -- 35
-local user_n     = nodes.register(new_node("whatsit",whatsits.user_defined)) user_n.type = 100 -- 44
-local user_l     = nodes.register(new_node("whatsit",whatsits.user_defined)) user_l.type = 110 -- 44
-local user_s     = nodes.register(new_node("whatsit",whatsits.user_defined)) user_s.type = 115 -- 44
-local user_t     = nodes.register(new_node("whatsit",whatsits.user_defined)) user_t.type = 116 -- 44
+local disc              = register_node(new_node("disc"))
+local kern              = register_node(new_node("kern",1))
+local penalty           = register_node(new_node("penalty"))
+local glue              = register_node(new_node("glue")) -- glue.spec = nil
+local glue_spec         = register_node(new_node("glue_spec"))
+local glyph             = register_node(new_node("glyph",0))
+local textdir           = register_node(new_node("whatsit",whatsits.dir)) -- 7 (6 is local par node)
+local rule              = register_node(new_node("rule"))
+local latelua           = register_node(new_node("whatsit",whatsits.late_lua)) -- 35
+local user_n            = register_node(new_node("whatsit",whatsits.user_defined)) user_n.type = 100 -- 44
+local user_l            = register_node(new_node("whatsit",whatsits.user_defined)) user_l.type = 110 -- 44
+local user_s            = register_node(new_node("whatsit",whatsits.user_defined)) user_s.type = 115 -- 44
+local user_t            = register_node(new_node("whatsit",whatsits.user_defined)) user_t.type = 116 -- 44
+local left_margin_kern  = register_node(new_node("margin_kern",0))
+local right_margin_kern = register_node(new_node("margin_kern",1))
+local lineskip          = register_node(new_node("glue",1))
+local baselineskip      = register_node(new_node("glue",2))
+local leftskip          = register_node(new_node("glue",8))
+local rightskip         = register_node(new_node("glue",9))
+local temp              = register_node(new_node("temp",0))
+
+function nodes.zeroglue(n)
+    local s = n.spec
+    return not writable or (
+                     s.width == 0
+         and       s.stretch == 0
+         and        s.shrink == 0
+         and s.stretch_order == 0
+         and  s.shrink_order == 0
+        )
+end
 
 function nodes.glyph(fnt,chr)
     local n = copy_node(glyph)
@@ -78,48 +104,156 @@ function nodes.glyph(fnt,chr)
     if chr then n.char = chr end
     return n
 end
+
 function nodes.penalty(p)
     local n = copy_node(penalty)
     n.penalty = p
     return n
 end
+
 function nodes.kern(k)
     local n = copy_node(kern)
     n.kern = k
     return n
 end
-function nodes.glue(width,stretch,shrink)
-    local n, s = copy_node(glue), copy_node(glue_spec)
-    s.width, s.stretch, s.shrink = width, stretch, shrink
-    n.spec = s
-    return n
-end
+
 function nodes.glue_spec(width,stretch,shrink)
     local s = copy_node(glue_spec)
     s.width, s.stretch, s.shrink = width, stretch, shrink
     return s
 end
+
+local function someskip(skip,width,stretch,shrink)
+    local n = copy_node(skip)
+    if not width then
+        -- no spec
+    elseif tonumber(width) then
+        local s = copy_node(glue_spec)
+        s.width, s.stretch, s.shrink = width, stretch, shrink
+        n.spec = s
+    else
+        -- shared
+        n.spec = copy_node(width)
+    end
+    return n
+end
+
+function nodes.glue(width,stretch,shrink)
+    return someskip(glue,width,stretch,shrink)
+end
+function nodes.leftskip(width,stretch,shrink)
+    return someskip(leftskip,width,stretch,shrink)
+end
+function nodes.rightskip(width,stretch,shrink)
+    return someskip(rightskip,width,stretch,shrink)
+end
+function nodes.lineskip(width,stretch,shrink)
+    return someskip(lineskip,width,stretch,shrink)
+end
+function nodes.baselineskip(width,stretch,shrink)
+    return someskip(baselineskip,width,stretch,shrink)
+end
+
 function nodes.disc()
     return copy_node(disc)
 end
+
 function nodes.textdir(dir)
     local t = copy_node(textdir)
     t.dir = dir
     return t
 end
-function nodes.rule(w,h,d)
+
+function nodes.rule(width,height,depth,dir)
     local n = copy_node(rule)
-    if w then n.width  = w end
-    if h then n.height = h end
-    if d then n.depth  = d end
+    if width  then n.width  = width  end
+    if height then n.height = height end
+    if depth  then n.depth  = depth  end
+    if dir    then n.dir    = dir    end
     return n
 end
+
 function nodes.latelua(code)
     local n = copy_node(latelua)
     n.data = code
     return n
 end
 
+function nodes.leftmarginkern(glyph,width)
+    local n = copy_node(left_margin_kern)
+    if not glyph then
+        logs.fatal("nodes","invalid pointer to left margin glyph node")
+    elseif glyph.id ~= glyph_node then
+        logs.fatal("nodes","invalid node type %s for left margin glyph node",node_type(glyph))
+    else
+        n.glyph = glyph
+    end
+    if width then
+        n.width = width
+    end
+    return n
+end
+
+function nodes.rightmarginkern(glyph,width)
+    local n = copy_node(right_margin_kern)
+    if not glyph then
+        logs.fatal("nodes","invalid pointer to right margin glyph node")
+    elseif glyph.id ~= glyph_node then
+        logs.fatal("nodes","invalid node type %s for right margin glyph node",node_type(p))
+    else
+        n.glyph = glyph
+    end
+    if width then
+        n.width = width
+    end
+    return n
+end
+
+function nodes.temp()
+    return copy_node(temp)
+end
+--[[
+<p>At some point we ran into a problem that the glue specification
+of the zeropoint dimension was overwritten when adapting a glue spec
+node. This is a side effect of glue specs being shared. After a
+couple of hours tracing and debugging Taco and I came to the
+conclusion that it made no sense to complicate the spec allocator
+and settled on a writable flag. This all is a side effect of the
+fact that some glues use reserved memory slots (with the zeropoint
+glue being a noticeable one). So, next we wrap this into a function
+and hide it for the user. And yes, LuaTeX now gives a warning as
+well.</p>
+]]--
+
+if tex.luatexversion > 51 then
+
+    function nodes.writable_spec(n)
+        local spec = n.spec
+        if not spec then
+            spec = copy_node(glue_spec)
+            n.spec = spec
+        elseif not spec.writable then
+            spec = copy_node(spec)
+            n.spec = spec
+        end
+        return spec
+    end
+
+else
+
+    function nodes.writable_spec(n)
+        local spec = n.spec
+        if not spec then
+            spec = copy_node(glue_spec)
+        else
+            spec = copy_node(spec)
+        end
+        n.spec = spec
+        return spec
+    end
+
+end
+
 local cache = { }
 
 function nodes.usernumber(num)