2014-12-09 13:05:00

11 files changed, 111 insertions, 55 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index f710d16af..c564529fe 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2014.12.08 23:47}
+\newcontextversion{2014.12.09 13:03}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index ed72584b7..8431af5b4 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index ecd372313..ebffaceeb 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -28,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2014.12.08 23:47}
+\edef\contextversion{2014.12.09 13:03}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/lang-ini.lua b/tex/context/base/lang-ini.lua
index d0fbced4d..302f5b34d 100644
--- a/tex/context/base/lang-ini.lua
+++ b/tex/context/base/lang-ini.lua
@@ -76,6 +76,9 @@ local function resolve(tag)
 end
 
 local function tolang(what) -- returns lang object
+    if not what then
+        what = tex.language
+    end
     local tag = numbers[what]
     local data = tag and registered[tag] or registered[what]
     if data then
@@ -389,3 +392,17 @@ commands.definelanguage        = languages.define
 commands.setlanguagesynonym    = languages.setsynonym
 commands.unloadlanguage        = languages.unload
 commands.setlanguageexceptions = languages.setexceptions
+
+function commands.prehyphenchar(l)
+    local c = prehyphenchar(tolang(l))
+    if c and c > 0 then
+        context.char(c)
+    end
+end
+
+function commands.posthyphenchar(l)
+    local c = posthyphenchar(tolang(l))
+    if c and c > 0 then
+        context.char(c)
+    end
+end
diff --git a/tex/context/base/lang-ini.mkiv b/tex/context/base/lang-ini.mkiv
index d40136f9c..bf2e42a75 100644
--- a/tex/context/base/lang-ini.mkiv
+++ b/tex/context/base/lang-ini.mkiv
@@ -693,4 +693,16 @@
 \uchyph        1 % also treat uppercase
 \exhyphenchar 45 % to permit breaking at explicit hyphens
 
+%D New:
+
+\unexpanded\def\traceddiscretionary#1#2#3%
+  {\dontleavehmode
+   \discretionary{\darkred#1}{\darkgreen#2}{\darkblue#3}}
+
+\unexpanded\def\samplediscretionary
+  {\traceddiscretionary
+     {pre\ctxcommand{prehyphenchar()}}%
+     {\ctxcommand{posthyphenchar()}post}%
+     {replace}}
+
 \protect \endinput
diff --git a/tex/context/base/node-fin.lua b/tex/context/base/node-fin.lua
index f38f0bf85..8aa7a5b18 100644
--- a/tex/context/base/node-fin.lua
+++ b/tex/context/base/node-fin.lua
@@ -180,8 +180,8 @@ local function process(namespace,attribute,head,inheritance,default) -- one attr
         local id = getid(stack)
         if id == glyph_code then
             check = true
-        elseif id == disc_code then
-            check = true -- indeed
+     -- elseif id == disc_code then
+     --     check = true -- no longer needed as we flatten replace
         elseif id == glue_code then
             leader = getleader(stack)
             if leader then
@@ -295,8 +295,8 @@ local function selective(namespace,attribute,head,inheritance,default) -- two at
         local id = getid(stack)
         if id == glyph_code then
             check = true
-        elseif id == disc_code then
-            check = true -- indeed
+     -- elseif id == disc_code then
+     --     check = true -- no longer needed as we flatten replace
         elseif id == glue_code then
             leader = getleader(stack)
             if leader then
diff --git a/tex/context/base/node-shp.lua b/tex/context/base/node-shp.lua
index 6ebfd767f..42b622878 100644
--- a/tex/context/base/node-shp.lua
+++ b/tex/context/base/node-shp.lua
@@ -15,6 +15,8 @@ local setmetatableindex = table.setmetatableindex
 
 local nodecodes      = nodes.nodecodes
 local whatsitcodes   = nodes.whatsitcodes
+local disccodes      = nodes.disccodes
+
 local tasks          = nodes.tasks
 local handlers       = nodes.handlers
 
@@ -26,11 +28,25 @@ local kern_code      = nodecodes.kern
 local glue_code      = nodecodes.glue
 local whatsit_code   = nodecodes.whatsit
 
+local fulldisc_code  = disccodes.discretionary
+
 local texgetbox      = tex.getbox
 
-local free_node      = node.free
-local remove_node    = node.remove
-local traverse_nodes = node.traverse
+local nuts           = nodes.nuts
+local tonut          = nuts.tonut
+local tonode         = nuts.tonode
+local free_node      = nuts.free
+local remove_node    = nuts.remove
+local traverse_nodes = nuts.traverse
+local find_tail      = nuts.tail
+
+local getfield       = nuts.getfield
+local setfield       = nuts.setfield
+local getid          = nuts.getid
+local getnext        = nuts.getnext
+local getprev        = nuts.getprev
+local getlist        = nuts.getlist
+local getsubtype     = nuts.getsubtype
 
 local removables     = {
     [whatsitcodes.open]       = true,
@@ -41,38 +57,50 @@ local removables     = {
     [whatsitcodes.latelua]    = true,
 }
 
-local function cleanup_redundant(head)
+-- About 10% of the nodes make no sense for the backend. By (at least)
+-- removing the replace disc nodes, we can omit extensive checking in
+-- the finalizer code (e.g. colors in disc nodes). Removing more nodes
+-- (like marks) is not saving much and removing empty boxes is even
+-- dangerous because we can rely on dimensions (e.g. in references).
+
+local wipedisc = false -- we can use them in the export ... can be option
+
+local function cleanup_redundant(head) -- better name is: flatten_page
     local start = head
     while start do
-        local id = start.id
+        local id = getid(start)
         if id == disc_code then
-            head, start = remove_node(head,start,true)
-     -- elseif id == glue_code then
-     --     if start.writable then
-     --         start = start.next
-     --     elseif some_complex_check_on_glue_spec then
-     --         head, start = remove_node(head,start,true)
-     --     else
-     --         start = start.next
-     --     end
-        elseif id == kern_code then
-            if start.kern == 0 then
-                head, start = remove_node(head,start,true)
+            if getsubtype(start) == fulldisc_code then
+                local replace = getfield(start,"replace")
+                if replace then
+                    setfield(start,"replace",nil)
+                    head, start = remove_node(head,start,true)
+                    local tail = find_tail(replace)
+                    local prev = getprev(start)
+                    setfield(tail,"next",start)
+                    setfield(start,"prev",tail)
+                    setfield(prev,"next",replace)
+                    setfield(replace,"prev",prev)
+                elseif wipedisc then
+                    -- pre and post can have values
+                    head, start = remove_node(head,start,true)
+                else
+                    start = getnext(start)
+                end
             else
-                start = start.next
+                start = getnext(start)
             end
-        elseif id == mark_code then
-            head, start = remove_node(head,start,true)
         elseif id == hlist_code or id == vlist_code then
-            local sl = start.list
+            local sl = getlist(start)
             if sl then
-                start.list = cleanup_redundant(sl)
-                start = start.next
-            else
-                head, start = remove_node(head,start,true)
+                local rl = cleanup_redundant(sl)
+                if rl ~= sl then
+                    setfield(start,"list",rl)
+                end
             end
+            start = getnext(start)
         else
-            start = start.next
+            start = getnext(start)
         end
     end
     return head
@@ -81,37 +109,37 @@ end
 local function cleanup_flushed(head) -- rough
     local start = head
     while start do
-        local id = start.id
-        if id == whatsit_code and removables[start.subtype] then
-            head, start = remove_node(head,start,true)
+        local id = getid(start)
+        if id == whatsit_code then
+            if removables[getsubtype(start)] then
+                head, start = remove_node(head,start,true)
+            else
+                start = getnext(start)
+            end
         elseif id == hlist_code or id == vlist_code then
-            local sl = start.list
+            local sl = getlist(start)
             if sl then
-                start.list = cleanup_flushed(sl)
-                start = start.next
-            else
-                head, start = remove_node(head,start,true)
+                local rl = cleanup_flushed(sl)
+                if rl ~= sl then
+                    setfield(start,"list",rl)
+                end
             end
+            start = getnext(start)
         else
-            start = start.next
+            start = getnext(start)
         end
     end
     return head
 end
 
 function handlers.cleanuppage(head)
-    -- about 10% of the nodes make no sense for the backend
-    return cleanup_redundant(head), true
+    return tonode(cleanup_redundant(tonut(head))), true
 end
 
 function handlers.cleanupbox(head)
-    return cleanup_flushed(head), true
+    return tonode(cleanup_flushed(tonut(head))), true
 end
 
-directives.register("backend.cleanup", function()
-    tasks.enableaction("shipouts","nodes.handlers.cleanuppage")
-end)
-
 local actions = tasks.actions("shipouts")  -- no extra arguments
 
 function handlers.finalize(head) -- problem, attr loaded before node, todo ...
@@ -158,12 +186,12 @@ local function count(head,data,subcategory)
     -- no components, pre, post, replace .. can maybe an option .. but
     -- we use this for optimization so it makes sense to look the the
     -- main node only
-    for n in traverse_nodes(head) do
-        local id = n.id
-        local dn = data[nodecodes[n.id]]
+    for n in traverse_nodes(tonut(head)) do
+        local id = getid(n)
+        local dn = data[nodecodes[id]] -- we could use id and then later convert to nodecodes
         dn[subcategory] = dn[subcategory] + 1
         if id == hlist_code or id == vlist_code then
-            count(n.list,data,subcategory)
+            count(getfield(n,"list"),data,subcategory)
         end
     end
 end
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index a78f98d80..e42667002 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 3432e2430..e9a91348d 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/task-ini.lua b/tex/context/base/task-ini.lua
index 4087caafe..d9ede85c4 100644
--- a/tex/context/base/task-ini.lua
+++ b/tex/context/base/task-ini.lua
@@ -62,8 +62,8 @@ appendaction("processors",   "lists",       "typesetters.italics.handler")
 
 -- appendaction("processors",   "lists",       "typesetters.initials.handler")                      -- disabled
 
-appendaction("shipouts",     "normalizers", "builders.paragraphs.expansion.trace")               -- disabled
 appendaction("shipouts",     "normalizers", "nodes.handlers.cleanuppage")                        -- disabled
+appendaction("shipouts",     "normalizers", "builders.paragraphs.expansion.trace")               -- disabled
 appendaction("shipouts",     "normalizers", "typesetters.alignments.handler")
 appendaction("shipouts",     "normalizers", "nodes.references.handler")                          -- disabled
 appendaction("shipouts",     "normalizers", "nodes.destinations.handler")                        -- disabled
@@ -168,7 +168,6 @@ disableaction("shipouts",    "nodes.visualizers.handler")
 disableaction("shipouts",    "nodes.handlers.accessibility")
 disableaction("shipouts",    "nodes.handlers.backgrounds")
 disableaction("shipouts",    "nodes.handlers.alignbackgrounds")
-disableaction("shipouts",    "nodes.handlers.cleanuppage")
 
 disableaction("shipouts",    "nodes.references.handler")
 disableaction("shipouts",    "nodes.destinations.handler")
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 5cb4f4890..53386b5d6 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 12/08/14 23:47:31
+-- merge date  : 12/09/14 13:03:29
 
 do -- begin closure to overcome local limits and interference