2015-07-12 00:04:00

13 files changed, 227 insertions, 111 deletions

diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua
index dafe2ad0e..1caf3b752 100644
--- a/tex/context/base/buff-ini.lua
+++ b/tex/context/base/buff-ini.lua
@@ -50,7 +50,12 @@ local registertempfile  = luatex.registertempfile
 
 local v_yes             = variables.yes
 
-local p_whitespace      = patterns.whitespace
+local eol               = patterns.eol
+local space             = patterns.space
+local whitespace        = patterns.whitespace
+local blackspace        = whitespace - eol
+local whatever          = (1-eol)^1 * eol^0
+local emptyline         = space^0 * eol
 
 local catcodenumbers    = catcodes.numbers
 
@@ -264,25 +269,24 @@ local continue   = false
 
 -- how about tabs
 
-local getmargin = (Cs(P(" ")^1)*P(-1)+1)^1 -- 1 or utf8character
-local eol       = patterns.eol
-local whatever  = (P(1)-eol)^0 * eol^1
+local strippers  = { }
+local nofspaces  = 0
 
-local strippers = { }
+local normalline = space^0 / function(s) local n = #s if n < nofspaces then nofspaces = n end end
+                 * whatever
+
+local getmargin = (emptyline + normalline)^1
 
 local function undent(str) -- new version, needs testing: todo: not always needed, like in xtables
+    nofspaces = #str
     local margin = lpegmatch(getmargin,str)
-    if type(margin) ~= "string" then
-        return str
-    end
-    local indent = #margin
-    if indent == 0 then
+    if nofspaces == #str or nofspaces ==0 then
         return str
     end
-    local stripper = strippers[indent]
+    local stripper = strippers[nofspaces]
     if not stripper then
-        stripper = Cs((P(margin)/"" * whatever + eol^1)^1)
-        strippers[indent] = stripper
+        stripper = Cs(((space^-nofspaces)/"" * whatever + emptyline)^1)
+        strippers[nofspaces] = stripper
     end
     return lpegmatch(stripper,str) or str
 end
@@ -334,54 +338,61 @@ buffers.undent = undent
 -- end
 
 function tokens.pickup(start,stop)
-    local stoplist    = totable(stop)
-    local stoplength  = #stoplist
-    local stoplast    = stoplist[stoplength]
-    local startlist   = totable(start)
-    local startlength = #startlist
-    local startlast   = startlist[startlength]
-    local list        = { }
-    local size        = 0
-    local depth       = 0
+    local stoplist     = totable(stop)
+    local stoplength   = #stoplist
+    local stoplast     = stoplist[stoplength]
+    local startlist    = totable(start)
+    local startlength  = #startlist
+    local startlast    = startlist[startlength]
+    local list         = { }
+    local size         = 0
+    local depth        = 0
+--     local done         = 32
     while true do -- or use depth
         local char = scancode()
         if char then
-            char = utfchar(char)
-            size = size + 1
-            list[size] = char
-            if char == stoplast and size >= stoplength then
-                local done = true
-                local last = size
-                for i=stoplength,1,-1 do
-                    if stoplist[i] ~= list[last] then
-                        done = false
-                        break
+--             if char < done then
+--                 -- we skip leading control characters so that we can use them to
+--                 -- obey spaces (a dirty trick)
+--             else
+--                 done = 0
+                char = utfchar(char)
+                size = size + 1
+                list[size] = char
+                if char == stoplast and size >= stoplength then
+                    local done = true
+                    local last = size
+                    for i=stoplength,1,-1 do
+                        if stoplist[i] ~= list[last] then
+                            done = false
+                            break
+                        end
+                        last = last - 1
                     end
-                    last = last - 1
-                end
-                if done then
-                    if depth > 0 then
-                        depth = depth - 1
-                    else
-                        break
+                    if done then
+                        if depth > 0 then
+                            depth = depth - 1
+                        else
+                            break
+                        end
+                        char = false -- trick: let's skip the next (start) test
                     end
-                    char = false -- trick: let's skip the next (start) test
                 end
-            end
-            if char == startlast and size >= startlength then
-                local done = true
-                local last = size
-                for i=startlength,1,-1 do
-                    if startlist[i] ~= list[last] then
-                        done = false
-                        break
+                if char == startlast and size >= startlength then
+                    local done = true
+                    local last = size
+                    for i=startlength,1,-1 do
+                        if startlist[i] ~= list[last] then
+                            done = false
+                            break
+                        end
+                        last = last - 1
+                    end
+                    if done then
+                        depth = depth + 1
                     end
-                    last = last - 1
-                end
-                if done then
-                    depth = depth + 1
                 end
-            end
+--             end
         else
          -- local t = scantoken()
             local t = gettoken()
@@ -394,15 +405,21 @@ function tokens.pickup(start,stop)
     end
     local start = 1
     local stop  = size-stoplength-1
+    -- not good enough: only empty lines, but even then we miss the leading
+    -- for verbatim
     for i=start,stop do
-        if lpegmatch(p_whitespace,list[i]) then
+        local li = list[i]
+        if lpegmatch(blackspace,li) then
+            -- keep going
+        elseif lpegmatch(eol,li) then
+            -- okay
             start = i + 1
         else
             break
         end
     end
     for i=stop,start,-1 do
-        if lpegmatch(p_whitespace,list[i]) then
+        if lpegmatch(whitespace,list[i]) then
             stop = i - 1
         else
             break
@@ -435,7 +452,7 @@ scanners.pickupbuffer = function()
     local doundent = scanboolean()
     local data = tokens.pickup(start,stop)
     if doundent or (autoundent and doundent == nil) then
-        data = buffers.undent(data)
+        data = undent(data)
     end
     buffers.assign(name,data,catcodes)
  -- context[finish]()
diff --git a/tex/context/base/buff-ini.mkiv b/tex/context/base/buff-ini.mkiv
index c10680085..cd6b7bfe5 100644
--- a/tex/context/base/buff-ini.mkiv
+++ b/tex/context/base/buff-ini.mkiv
@@ -118,7 +118,7 @@
      \scratchcounter
      \ifnum#6=\plusone\s!true\else\s!false\fi
  % \relax}
-   \expandafter\relax\string}
+   \expandafter\relax\string} % maybe \normalexpanded{\relax\utfchar{7}}} signal
 
 \unexpanded\def\buff_stop#1%
   {\endgroup % (3 & 4 & 5 & 6)
@@ -170,7 +170,7 @@
 
 \unexpanded\def\buff_get[#1]% [name]
   {\namedbufferparameter\empty\c!before
-   \doifelsenothing{#1}
+%    \doifelsenothing{#1}
      {\buff_get_stored_indeed\empty}
      {\processcommalist[#1]\buff_get_stored_indeed}%
    \namedbufferparameter\empty\c!after}
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index d8017a271..a344eee5c 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2015.07.10 00:05}
+\newcontextversion{2015.07.12 00:00}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 45307850a..7154b629d 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 9587f0464..8d17530f3 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -39,7 +39,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2015.07.10 00:05}
+\edef\contextversion{2015.07.12 00:00}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/font-otc.lua b/tex/context/base/font-otc.lua
index c2ccf42b0..4a71e3df6 100644
--- a/tex/context/base/font-otc.lua
+++ b/tex/context/base/font-otc.lua
@@ -21,12 +21,6 @@ local otf                 = fonts.handlers.otf
 local registerotffeature  = otf.features.register
 local setmetatableindex   = table.setmetatableindex
 
--- In the userdata interface we can not longer tweak the loaded font as
--- conveniently as before. For instance, instead of pushing extra data in
--- in the table using the original structure, we now have to operate on
--- the mkiv representation. And as the fontloader interface is modelled
--- after fontforge we cannot change that one too much either.
-
 local normalized = {
     substitution = "substitution",
     single       = "substitution",
@@ -37,7 +31,6 @@ local normalized = {
 }
 
 local types = {
-    single       = "gsub_single",
     substitution = "gsub_single",
     ligature     = "gsub_ligature",
     alternate    = "gsub_alternate",
diff --git a/tex/context/base/m-newotf.mkiv b/tex/context/base/m-newotf.mkiv
new file mode 100644
index 000000000..8348f600f
--- /dev/null
+++ b/tex/context/base/m-newotf.mkiv
@@ -0,0 +1,39 @@
+%D \module
+%D   [       file=m-newotf,
+%D        version=2015.07.08,
+%D          title=\CONTEXT\ Extra Modules,
+%D       subtitle=Experimental OTF Loader,
+%D         author=Hans Hagen,
+%D           date=\currentdate,
+%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
+%C
+%C This module is part of the \CONTEXT\ macro||package and is
+%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
+%C details.
+
+%D This module will go away as soon as we use the new loader code by default.
+%D That will happen after extensive testing. Generic support will happen after
+%D that.
+
+\unprotect
+
+\startluacode
+    local files = {
+        "font-otr", "font-cff", "font-ttf", "font-dsp", "font-oup",
+        "font-otl", "font-ots", "font-oto", "font-otd", "font-otc",
+        "font-osd", "font-map", "font-fbk", "font-gds",
+    }
+
+    logs.report("fonts")
+    if tex.modes.atpragma then
+        logs.report("fonts","replacing font loader code by experimental code (at pragma)")
+        logs.report("fonts")
+        for i=1,#files do require(file.join("t:/sources",files[i])) end
+    else
+        logs.report("fonts","replacing font loader code by experimental code")
+        logs.report("fonts")
+        for i=1,#files do require(files[i]) end
+    end
+\stopluacode
+
+\protect \endinput
diff --git a/tex/context/base/math-noa.lua b/tex/context/base/math-noa.lua
index a1b3bc855..8231738bc 100644
--- a/tex/context/base/math-noa.lua
+++ b/tex/context/base/math-noa.lua
@@ -6,9 +6,10 @@ if not modules then modules = { } end modules ['math-noa'] = {
     license   = "see context related readme files"
 }
 
--- beware: this is experimental code and there will be a more
--- generic (attribute value driven) interface too but for the
--- moment this is ok
+-- beware: this is experimental code and there will be a more generic (attribute value
+-- driven) interface too but for the moment this is ok (sometime in 2015-2016 i will
+-- start cleaning up as by then the bigger picture is clear and code has been used for
+-- years; the main handlers will get some extensions)
 --
 -- we will also make dedicated processors (faster)
 --
@@ -21,6 +22,7 @@ if not modules then modules = { } end modules ['math-noa'] = {
 local utfchar, utfbyte = utf.char, utf.byte
 local formatters = string.formatters
 local sortedhash = table.sortedhash
+local insert, remove = table.insert, table.remove
 local div = math.div
 
 local fonts, nodes, node, mathematics = fonts, nodes, node, mathematics
@@ -65,6 +67,7 @@ local a_exportstatus      = privateattribute("exportstatus")
 local nuts                = nodes.nuts
 local nodepool            = nuts.pool
 local tonut               = nuts.tonut
+local tonode              = nuts.tonode
 local nutstring           = nuts.tostring
 
 local getfield            = nuts.getfield
@@ -83,6 +86,7 @@ local insert_node_before  = nuts.insert_before
 local free_node           = nuts.free
 local new_node            = nuts.new -- todo: pool: math_noad math_sub
 local copy_node           = nuts.copy
+local slide_nodes         = nuts.slide
 
 local mlist_to_hlist      = nodes.mlist_to_hlist
 
@@ -119,6 +123,7 @@ local tasks               = nodes.tasks
 
 local nodecodes           = nodes.nodecodes
 local noadcodes           = nodes.noadcodes
+local fencecodes          = nodes.fencecodes
 
 local noad_ord            = noadcodes.ord
 local noad_rel            = noadcodes.rel
@@ -126,6 +131,7 @@ local noad_punct          = noadcodes.punct
 local noad_opdisplaylimits= noadcodes.opdisplaylimits
 local noad_oplimits       = noadcodes.oplimits
 local noad_opnolimits     = noadcodes.opnolimits
+local noad_inner          = noadcodes.inner
 
 local math_noad           = nodecodes.noad           -- attr nucleus sub sup
 local math_accent         = nodecodes.accent         -- attr nucleus sub sup accent
@@ -143,12 +149,24 @@ local math_fence          = nodecodes.fence          -- attr subtype
 local hlist_code          = nodecodes.hlist
 local glyph_code          = nodecodes.glyph
 
-local left_fence_code     = 1
-local right_fence_code    = 3
+local left_fence_code     = fencecodes.left
+local middle_fence_code   = fencecodes.middle
+local right_fence_code    = fencecodes.right
+
+-- this initial stuff is tricky as we can have removed and new nodes with the same address
+-- the only way out is a free-per-page list of nodes (not bad anyway)
 
 local function process(start,what,n,parent)
-    if n then n = n + 1 else n = 0 end
-    local prev = nil
+    if n then
+        n = n + 1
+    else
+        n = 0
+    end
+    --
+    local initial = start
+    --
+    slide_nodes(start) -- we still miss a prev in noads -- fences test code
+    --
     while start do
         local id = getid(start)
         if trace_processing then
@@ -166,21 +184,27 @@ local function process(start,what,n,parent)
         local proc = what[id]
         if proc then
          -- report_processing("start processing")
-            local done, newstart = proc(start,what,n,parent) -- prev is bugged:  or getprev(start)
-            if newstart then
-                start = newstart
-             -- report_processing("stop processing (new start)")
+            local done, newstart, newinitial = proc(start,what,n,parent) -- prev is bugged:  or getprev(start)
+            if newinitial then
+                initial = newinitial -- temp hack .. we will make all return head
+                if newstart then
+                    start = newstart
+                 -- report_processing("stop processing (new start)")
+                else
+                 -- report_processing("quit processing (done)")
+                    break
+                end
             else
-             -- report_processing("stop processing")
+                if newstart then
+                    start = newstart
+                 -- report_processing("stop processing (new start)")
+                else
+                 -- report_processing("stop processing")
+                end
             end
         elseif id == math_char or id == math_textchar or id == math_delim then
             break
         elseif id == math_noad then
-if prev then
-    -- we have no proper prev in math nodes yet
-    setfield(start,"prev",prev)
-end
-
             local noad = getfield(start,"nucleus")      if noad then process(noad,what,n,start) end -- list
                   noad = getfield(start,"sup")          if noad then process(noad,what,n,start) end -- list
                   noad = getfield(start,"sub")          if noad then process(noad,what,n,start) end -- list
@@ -210,27 +234,69 @@ end
                   noad = getfield(start,"sub")          if noad then process(noad,what,n,start) end -- list
                   noad = getfield(start,"accent")       if noad then process(noad,what,n,start) end -- list
                   noad = getfield(start,"bot_accent")   if noad then process(noad,what,n,start) end -- list
-        elseif id == math_style then
-            -- has a next
-        else
-            -- glue, penalty, etc
+     -- elseif id == math_style then
+     --     -- has a next
+     -- else
+     --     -- glue, penalty, etc
         end
-prev = start
         start = getnext(start)
     end
+    if not parent then
+        return initial, true -- only first level -- for now
+    end
+end
+
+local function processnested(current,what)
+    local noad = nil
+    local id   = getid(current)
+    if id == math_noad then
+        noad = getfield(current,"nucleus")      if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"sup")          if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"sub")          if noad then return process(noad,what,n,current) end -- list
+    elseif id == math_box or id == math_sub then
+        noad = getfield(current,"list")         if noad then return process(noad,what,n,current) end -- list (not getlist !)
+    elseif id == math_fraction then
+        noad = getfield(current,"num")          if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"denom")        if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"left")         if noad then return process(noad,what,n,current) end -- delimiter
+        noad = getfield(current,"right")        if noad then return process(noad,what,n,current) end -- delimiter
+    elseif id == math_choice then
+        noad = getfield(current,"display")      if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"text")         if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"script")       if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"scriptscript") if noad then return process(noad,what,n,current) end -- list
+    elseif id == math_fence then
+        noad = getfield(current,"delim")        if noad then return process(noad,what,n,current) end -- delimiter
+    elseif id == math_radical then
+        noad = getfield(current,"nucleus")      if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"sup")          if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"sub")          if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"left")         if noad then return process(noad,what,n,current) end -- delimiter
+        noad = getfield(current,"degree")       if noad then return process(noad,what,n,current) end -- list
+    elseif id == math_accent then
+        noad = getfield(current,"nucleus")      if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"sup")          if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"sub")          if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"accent")       if noad then return process(noad,what,n,current) end -- list
+        noad = getfield(current,"bot_accent")   if noad then return process(noad,what,n,current) end -- list
+    end
 end
 
 local function processnoads(head,actions,banner)
+    local h, d
     if trace_processing then
         report_processing("start %a",banner)
-        process(tonut(head),actions)
+        h, d = process(tonut(head),actions)
         report_processing("stop %a",banner)
     else
-        process(tonut(head),actions)
+        h, d = process(tonut(head),actions)
     end
+    return h and tonode(h) or head, d == nil and true or d
 end
 
-noads.process = processnoads
+noads.process       = processnoads
+noads.processnested = processnested
+noads.processouter  = process
 
 --
 
@@ -539,7 +605,7 @@ end
 -- todo: just replace the character by an ord noad
 -- and remove the right delimiter as well
 
-local mathsize = privateattribute("mathsize")
+local mathsize = privateattribute("mathsize") -- this might move into other fence code
 
 local resize = { } processors.resize = resize
 
@@ -790,8 +856,8 @@ alternate[math_char] = function(pointer)
     end
 end
 
-function handlers.check(head,style,penalties)
-    processnoads(head,alternate,"check")
+function handlers.alternates(head,style,penalties)
+    processnoads(head,alternate,"alternate")
     return true
 end
 
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 9e26a6d32..96c4157a8 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index ff04d11ba..ff68f540b 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/task-ini.lua b/tex/context/base/task-ini.lua
index 062f0208f..5fef5633c 100644
--- a/tex/context/base/task-ini.lua
+++ b/tex/context/base/task-ini.lua
@@ -88,21 +88,21 @@ appendaction("shipouts",     "finishers",   "attributes.viewerlayers.handler")
 
 --maybe integrate relocate and families
 
-appendaction("math",         "normalizers", "noads.handlers.showtree", nil, "nohead")
-
-appendaction("math",         "normalizers", "noads.handlers.unscript", nil, "nohead")            -- always on (maybe disabled)
-appendaction("math",         "normalizers", "noads.handlers.variants", nil, "nohead")            -- always on
-appendaction("math",         "normalizers", "noads.handlers.relocate", nil, "nohead")            -- always on
-appendaction("math",         "normalizers", "noads.handlers.families", nil, "nohead")            -- always on
-
-appendaction("math",         "normalizers", "noads.handlers.render",   nil, "nohead")            -- always on
-appendaction("math",         "normalizers", "noads.handlers.collapse", nil, "nohead")            -- always on
-appendaction("math",         "normalizers", "noads.handlers.resize",   nil, "nohead")            -- always on
-------------("math",         "normalizers", "noads.handlers.respace",  nil, "nohead")            -- always on
-appendaction("math",         "normalizers", "noads.handlers.check",    nil, "nohead")            -- always on
-appendaction("math",         "normalizers", "noads.handlers.tags",     nil, "nohead")            -- disabled
-appendaction("math",         "normalizers", "noads.handlers.italics",  nil, "nohead")            -- disabled
-appendaction("math",         "normalizers", "noads.handlers.classes",  nil, "nohead")            -- disabled
+appendaction("math",         "normalizers", "noads.handlers.showtree",  nil, "nohead")
+
+appendaction("math",         "normalizers", "noads.handlers.unscript",  nil, "nohead")            -- always on (maybe disabled)
+appendaction("math",         "normalizers", "noads.handlers.variants",  nil, "nohead")            -- always on
+appendaction("math",         "normalizers", "noads.handlers.relocate",  nil, "nohead")            -- always on
+appendaction("math",         "normalizers", "noads.handlers.families",  nil, "nohead")            -- always on
+
+appendaction("math",         "normalizers", "noads.handlers.render",    nil, "nohead")            -- always on
+appendaction("math",         "normalizers", "noads.handlers.collapse",  nil, "nohead")            -- always on
+appendaction("math",         "normalizers", "noads.handlers.resize",    nil, "nohead")            -- always on
+------------("math",         "normalizers", "noads.handlers.respace",   nil, "nohead")            -- always on
+appendaction("math",         "normalizers", "noads.handlers.alternates",nil, "nohead")            -- always on
+appendaction("math",         "normalizers", "noads.handlers.tags",      nil, "nohead")            -- disabled
+appendaction("math",         "normalizers", "noads.handlers.italics",   nil, "nohead")            -- disabled
+appendaction("math",         "normalizers", "noads.handlers.classes",   nil, "nohead")            -- disabled
 
 appendaction("math",         "builders",    "builders.kernel.mlist_to_hlist")                    -- always on
 ------------("math",         "builders",    "noads.handlers.italics",  nil, "nohead")            -- disabled
diff --git a/tex/context/base/util-seq.lua b/tex/context/base/util-seq.lua
index 08fc4e95c..32fe59096 100644
--- a/tex/context/base/util-seq.lua
+++ b/tex/context/base/util-seq.lua
@@ -347,5 +347,6 @@ function sequencers.nodeprocessor(t,nofarguments) -- todo: handle 'kind' in plug
         end
     end
     local processor = #calls > 0 and formatters[template_yes](concat(vars,"\n"),args,concat(calls,"\n")) or template_nop
+-- print(processor)
     return processor
 end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index bf337eff7..c88a52855 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 07/10/15 00:05:49
+-- merge date  : 07/12/15 00:00:22
 
 do -- begin closure to overcome local limits and interference