beta 2013.09.19 17:08

9 files changed, 279 insertions, 95 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index f9b31f31b..4b542c0d8 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.09.19 13:52}
+\newcontextversion{2013.09.19 17:08}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 93c46df4c..8aca2aa54 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index ad67b3668..8866ecfe9 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.09.19 13:52}
+\edef\contextversion{2013.09.19 17:08}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/math-ini.mkiv b/tex/context/base/math-ini.mkiv
index bbec7d77d..f2327c22c 100644
--- a/tex/context/base/math-ini.mkiv
+++ b/tex/context/base/math-ini.mkiv
@@ -414,6 +414,17 @@
   {\ifdefined\normalhbox\else\let\normalhbox\hbox\fi % ?
    \let\hbox\mbox}
 
+\unexpanded\def\snappedmath#1% sort of \struttedbox
+  {\dontleavehmode
+   \begingroup
+   \setbox\scratchbox\hbox\bgroup
+     \startimath#1\stopimath
+   \egroup
+   \ht\scratchbox\strutht
+   \dp\scratchbox\strutht
+   \box\scratchbox
+   \endgroup}
+
 %D The next hack is needed needed for sine, cosine etc.
 
 \let\mathfunction\firstofoneunexpanded
diff --git a/tex/context/base/page-mix.lua b/tex/context/base/page-mix.lua
index f86e2f279..ef5308546 100644
--- a/tex/context/base/page-mix.lua
+++ b/tex/context/base/page-mix.lua
@@ -135,16 +135,11 @@ local function discardtopglue(current,discarded)
         elseif id == penalty_code then
             if current.penalty == forcedbreak then
                 discarded[#discarded+1] = current
-                current = current.next
-                while current do
-                    local id = current.id
-                    if id == glue_code then
-                        size = size + current.spec.width
-                        discarded[#discarded+1] = current
-                        current = current.next
-                    else
-                        break
-                    end
+                current = getnext(current)
+                while current and current.id == glue_code do
+                    size = size + current.spec.width
+                    discarded[#discarded+1] = current
+                    current = current.next
                 end
             else
                 discarded[#discarded+1] = current
@@ -221,6 +216,10 @@ local function setsplit(specification) -- a rather large function
     local height = 0
     local depth = 0
     local skip = 0
+    local splitmethod = specification.splitmethod or false
+    if splitmethod == v_none then
+        splitmethod = false
+    end
     local options = settings_to_hash(specification.option or "")
     local stripbottom = specification.alternative == v_local
     local cycle = specification.cycle or 1
@@ -256,12 +255,35 @@ local function setsplit(specification) -- a rather large function
             delta   = 0,
         }
     end
+
     local column   = 1
     local line     = 0
-    local result   = results[column]
+    local result   = results[1]
     local lasthead = nil
     local rest     = nil
+
+    if trace_state then
+        report_state("setting collector to column %s",column)
+    end
+
+    local lastlocked  = nil
+    local lastcurrent = nil
+
+    local backtracked = false
+
     local function gotonext()
+        if lastcurrent then
+            if current ~= lastcurrent then
+                if trace_state then
+                    report_state("backtracking to preferred break in column %s",column)
+                end
+                -- todo: also remember height/depth
+                current = lastcurrent
+                backtracked = true
+            end
+            lastcurrent = nil
+            lastlocked = nil
+        end
         if head == lasthead then
             if trace_state then
                 report_state("empty column %s, needs more work",column)
@@ -290,12 +312,16 @@ local function setsplit(specification) -- a rather large function
             local skipped
             column = column + 1
             result = results[column]
+            if trace_state then
+                report_state("setting collector to column %s",column)
+            end
             current, skipped = discardtopglue(current,discarded)
             head = current
             return true, skipped
         end
     end
-    local function checked(advance,where)
+
+    local function checked(advance,where,locked)
         local total   = skip + height + depth + advance
         local delta   = total - target
         local state   = "same"
@@ -317,47 +343,34 @@ local function setsplit(specification) -- a rather large function
         end
         return state, skipped
     end
+
     current, skipped = discardtopglue(current,discarded)
     if trace_detail and skipped ~= 0 then
         report_state("check > column 1, discarded %p",skipped)
     end
+
+    -- problem: when we cannot break after a list (and we only can expect same-page situations as we don't
+    -- care too much about weighted breaks here) we should sort of look ahead or otherwise be able to push
+    -- back inserts and so
+    --
+    -- ok, we could use vsplit but we don't have that one opened up yet .. maybe i should look into the c-code
+    -- .. something that i try to avoid so let's experiment more before we entry dirty trick mode
+
     head = current
-    while current do
-        local id = current.id
-        local nxt = current.next
-        local lastcolumn = column
-        if id == hlist_code or id == vlist_code then
-            line = line + 1
-            local nxtid = nxt and nxt.id
-            local inserts, currentskips, nextskips, inserttotal = nil, 0, 0, 0
-            local advance = current.height -- + current.depth
-            if nxt and (nxtid == insert_code or nxtid == mark_code) then
-                nxt, inserts, localskips, insertskips, inserttotal = collectinserts(result,nxt,nxtid)
-            end
-            local state, skipped = checked(advance+inserttotal+currentskips,"line")
-            if trace_state then
-                report_state("%-7s > column %s, state %a, line %s, advance %p, insert %p, height %p","line",column,state,line,advance,inserttotal,height)
-                if skipped ~= 0 then
-                    report_state("%-7s > column %s, discarded %p","line",column,skipped)
-                end
-            end
-            if state == "quit" then
-                break
-            else
-                height = height + depth + skip + advance + inserttotal
-                if state == "next" then
-                    height = height + nextskips
-                else
-                    height = height + currentskips
+
+    local function process_skip(current,nxt)
+        local advance = current.spec.width
+            local prv = current.prev
+            if prv.id == penalty_code then
+                local penalty = prv.penalty
+                if penalty < 4000 then
+                    lastlocked  = nil
+                    lastcurrent = nil
                 end
+            elseif current.subtype ~= lineskip_code then
+                lastlocked  = nil
+                lastcurrent = nil
             end
-            depth = current.depth
-            skip  = 0
-            if inserts then
-                appendinserts(result.inserts,inserts)
-            end
-        elseif id == glue_code then
-            local advance = current.spec.width
             if advance ~= 0 then
                 local state, skipped = checked(advance,"glue")
                 if trace_state then
@@ -367,60 +380,222 @@ local function setsplit(specification) -- a rather large function
                     end
                 end
                 if state == "quit" then
-                    break
+                    return true
                 end
                 height = height + depth + skip
                 depth  = 0
                 skip   = height > 0 and advance or 0
-            end
-        elseif id == kern_code then
-            local advance = current.kern
-            if advance ~= 0 then
-                local state, skipped = checked(advance,"kern")
                 if trace_state then
-                    report_state("%-7s > column %s, state %a, advance %p, height %p, state %a","kern",column,state,advance,height)
-                    if skipped ~= 0 then
-                        report_state("%-7s > column %s, discarded %p","kern",column,skipped)
-                    end
+                    report_state("%-7s > column %s, height %p, depth %p, skip %p","glue",column,height,depth,skip)
                 end
-                if state == "quit" then
-                    break
+            else
+                -- what else? ignore? treat as valid as usual?
+            end
+    end
+
+    local function process_kern(current,nxt)
+        local advance = current.kern
+        if advance ~= 0 then
+            local state, skipped = checked(advance,"kern")
+            if trace_state then
+                report_state("%-7s > column %s, state %a, advance %p, height %p, state %a","kern",column,state,advance,height)
+                if skipped ~= 0 then
+                    report_state("%-7s > column %s, discarded %p","kern",column,skipped)
                 end
-                height = height + depth + skip + advance
-                depth  = 0
-                skip   = 0
             end
-        elseif id == penalty_code then
-            local penalty = current.penalty
-            if penalty == 0 then
-                -- don't bother
-            elseif penalty == forcedbreak then
-                local okay, skipped = gotonext()
-                if okay then
-                    if trace_state then
-                        report_state("cycle: %s, forced column break (same page)",cycle)
-                        if skipped ~= 0 then
-                            report_state("%-7s > column %s, discarded %p","penalty",column,skipped)
-                        end
-                    end
-                else
-                    if trace_state then
-                        report_state("cycle: %s, forced column break (next page)",cycle)
-                        if skipped ~= 0 then
-                            report_state("%-7s > column %s, discarded %p","penalty",column,skipped)
-                        end
+            if state == "quit" then
+                return true
+            end
+            height = height + depth + skip + advance
+            depth  = 0
+            skip   = 0
+            if trace_state then
+                report_state("%-7s > column %s, height %p, depth %p, skip %p","kern",column,height,depth,skip)
+            end
+        end
+    end
+
+    local function process_rule(current,nxt)
+        -- simple variant of h|vlist
+        local advance = current.height -- + current.depth
+        local state, skipped = checked(advance+currentskips,"rule")
+        if trace_state then
+            report_state("%-7s > column %s, state %a, rule, advance %p, height %p","line",column,state,advance,inserttotal,height)
+            if skipped ~= 0 then
+                report_state("%-7s > column %s, discarded %p","rule",column,skipped)
+            end
+        end
+        if state == "quit" then
+            return true
+        end
+        height = height + depth + skip + advance
+        if state == "next" then
+            height = height + nextskips
+        else
+            height = height + currentskips
+        end
+        depth = current.depth
+        skip  = 0
+    end
+
+    -- okay, here we could do some badness like magic but we want something
+    -- predictable and even better: strategies .. so eventually this will
+    -- become installable
+    --
+    -- [chapter] [penalty] [section] [penalty] [first line]
+    --
+    -- we need some nice use cases so the next is just for me to play with
+
+    -- todo: presets:
+    --
+    -- fixed :  criterium=4000  check=no
+    -- large :  criterium=4000  check=more
+    -- auto  :  criterium=0     check=more
+
+    local lockcriterium = 4000
+
+    local function prevprev(current)
+        local p = current.prev
+        return p and p.prev
+    end
+
+    local function reassess(current,penalty)
+        if splitmethod == v_fixed then
+         -- quite ok, a magic number: used as samepage (in sectioning)
+            if penalty >= lockcriterium then
+                if not lastlocked then
+                    lastcurrent = prevprev(current)
+                    lastlocked  = lastcurrent and penalty
+                end
+                return
+            end
+        elseif splitmethod == v_more then
+            -- experiment, might change
+            if penalty >= lockcriterium then
+                if not lastlocked or penalty >= lastlocked then
+                    lastcurrent = prevprev(current)
+                    lastlocked  = lastcurrent and penalty
+                end
+                return
+            end
+        elseif splitmethod == v_auto then
+            if penalty > 0 then
+                if not lastlocked or penalty > lastlocked then
+                    lastcurrent = prevprev(current)
+                    lastlocked  = lastcurrent and penalty
+                end
+                return
+            end
+        end
+        lastlocked  = nil
+        lastcurrent = nil
+    end
+
+    local function process_penalty(current,nxt)
+        local penalty = current.penalty
+        if penalty == 0 then
+            lastlocked  = nil
+            lastcurrent = nil
+        elseif penalty == forcedbreak then
+            lastlocked  = nil
+            lastcurrent = nil
+            local okay, skipped = gotonext()
+            if okay then
+                if trace_state then
+                    report_state("cycle: %s, forced column break, same page",cycle)
+                    if skipped ~= 0 then
+                        report_state("%-7s > column %s, discarded %p","penalty",column,skipped)
                     end
-                    break
                 end
             else
-                -- todo: nobreak etc ... we might need to backtrack so we need to remember
-                -- the last acceptable break
-                -- club and widow and such i.e. resulting penalties (if we care)
+                if trace_state then
+                    report_state("cycle: %s, forced column break, next page",cycle)
+                    if skipped ~= 0 then
+                        report_state("%-7s > column %s, discarded %p","penalty",column,skipped)
+                    end
+                end
+                return true
+            end
+        elseif penalty < 0 then
+            -- we don't care too much
+            lastlocked  = nil
+            lastcurrent = nil
+        elseif splitmethod then
+            reassess(current,penalty)
+        else
+            lastlocked  = nil
+            lastcurrent = nil
+        end
+    end
+
+    local function process_list(current,nxt)
+        local nxtid = nxt and nxt.id
+        line = line + 1
+        local inserts, currentskips, nextskips, inserttotal = nil, 0, 0, 0
+        local advance = current.height -- + current.depth
+        if trace_state then
+            report_state("%-7s > column %s, content: %s","line",column,listtoutf(current.list,true,true))
+        end
+        if nxt and (nxtid == insert_code or nxtid == mark_code) then
+            nxt, inserts, localskips, insertskips, inserttotal = collectinserts(result,nxt,nxtid)
+        end
+        local state, skipped = checked(advance+inserttotal+currentskips,"line",lastlocked)
+        if trace_state then
+            report_state("%-7s > column %s, state %a, line %s, advance %p, insert %p, height %p","line",column,state,line,advance,inserttotal,height)
+            if skipped ~= 0 then
+                report_state("%-7s > column %s, discarded %p","line",column,skipped)
             end
         end
-        if current then -- lastcolumn == column then
-            nxt = current.next -- can have changed
+        if state == "quit" then
+            return true
         end
+        height = height + depth + skip + advance + inserttotal
+        if state == "next" then
+            height = height + nextskips
+        else
+            height = height + currentskips
+        end
+        depth = current.depth
+        skip  = 0
+        if inserts then
+            -- so we already collect them ... makes backtracking tricky ... alternatively
+            -- we can do that in a separate loop ... no big deal either
+            appendinserts(result.inserts,inserts)
+        end
+        if trace_state then
+            report_state("%-7s > column %s, height %p, depth %p, skip %p","line",column,height,depth,skip)
+        end
+    end
+
+    while current do
+
+        local id  = current.id
+        local nxt = current.next
+
+        backtracked = false
+
+     -- print("process",nodetostring(current))
+
+        if id == hlist_code or id == vlist_code then
+            if process_list(current,nxt) then break end
+        elseif id == glue_code then
+            if process_skip(current,nxt) then break end
+        elseif id == kern_code then
+            if process_kern(current,nxt) then break end
+        elseif id == penalty_code then
+            if process_penalty(current,nxt) then break end
+        elseif id == rule_code then
+            if process_rule(current,nxt) then break end
+        else
+        end
+
+        if backtracked then
+         -- print("pickup",nodetostring(current))
+            nxt = current
+        else
+         -- print("move on",nodetostring(current))
+        end
+
         if nxt then
             current = nxt
         elseif head == lasthead then
@@ -438,6 +613,7 @@ local function setsplit(specification) -- a rather large function
             break
         end
     end
+
     if not current then
         if trace_state then
             report_state("nilling rest")
@@ -557,9 +733,6 @@ function mixedcolumns.setsplit(specification)
     end
 end
 
-local topskip_code      = gluecodes.topskip
-local baselineskip_code = gluecodes.baselineskip
-
 function mixedcolumns.getsplit(result,n)
     if not result then
         report_state("flush, column %s, no result",n)
@@ -616,7 +789,8 @@ function mixedcolumns.getsplit(result,n)
     end
 
     for c, list in next, r.inserts do
-        local b = vpack(concatnodes(list)) -- multiple arguments
+        local l = concatnodes(list)
+        local b = vpack(l) -- multiple arguments, todo: fastvpack
      -- texsetbox("global",c,b)
         texsetbox(c,b)
         r.inserts[c] = nil
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 90791c6b9..26abecfb3 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.log b/tex/context/base/status-lua.log
index 27cb2d27d..51d51e0b0 100644
--- a/tex/context/base/status-lua.log
+++ b/tex/context/base/status-lua.log
@@ -1,6 +1,6 @@
 (cont-yes.mkiv
 
-ConTeXt  ver: 2013.09.19 13:52 MKIV beta  fmt: 2013.9.19  int: english/english
+ConTeXt  ver: 2013.09.19 17:08 MKIV beta  fmt: 2013.9.19  int: english/english
 
 system          > 'cont-new.mkiv' loaded
 (cont-new.mkiv)
diff --git a/tex/context/base/strc-itm.mkvi b/tex/context/base/strc-itm.mkvi
index 6d8745b19..b4b19eb2d 100644
--- a/tex/context/base/strc-itm.mkvi
+++ b/tex/context/base/strc-itm.mkvi
@@ -408,7 +408,6 @@
                                            \settrue\c_strc_itemgroups_inline
                                            \settrue\c_strc_itemgroups_joined
                                            \strc_itemgroups_process_set_option_pack}
-\setvalue{\??itemgroupkeyword\v!columns  }{\strc_itemgroups_process_set_option_pack}
 \setvalue{\??itemgroupkeyword\v!before   }{\settrue\c_strc_itemgroups_before}
 \setvalue{\??itemgroupkeyword\v!after    }{\settrue\c_strc_itemgroups_after}
 \setvalue{\??itemgroupkeyword\v!nowhite  }{\settrue\c_strc_itemgroups_nowhite}
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 6251ec816..b90de920f 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 09/19/13 13:52:26
+-- merge date  : 09/19/13 17:08:32
 
 do -- begin closure to overcome local limits and interference