beta 2013.08.29 20:45

12 files changed, 99 insertions, 33 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 2d8f76d93..a33c1c93f 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.08.28 23:08}
+\newcontextversion{2013.08.29 20:45}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 465b5c002..b61f77a54 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 4a3fd8877..e7b7d92e0 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.08.28 23:08}
+\edef\contextversion{2013.08.29 20:45}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index b33df96b7..78b503b8e 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -109,11 +109,11 @@ local space            = P(" ")
 local period           = P(".")
 local comma            = P(",")
 
-local utfbom_32_be     = P('\000\000\254\255')
-local utfbom_32_le     = P('\255\254\000\000')
-local utfbom_16_be     = P('\255\254')
-local utfbom_16_le     = P('\254\255')
-local utfbom_8         = P('\239\187\191')
+local utfbom_32_be     = P('\000\000\254\255') -- 00 00 FE FF
+local utfbom_32_le     = P('\255\254\000\000') -- FF FE 00 00
+local utfbom_16_be     = P('\254\255')         -- FE FF
+local utfbom_16_le     = P('\255\254')         -- FF FE
+local utfbom_8         = P('\239\187\191')     -- EF BB BF
 local utfbom           = utfbom_32_be + utfbom_32_le
                        + utfbom_16_be + utfbom_16_le
                        + utfbom_8
diff --git a/tex/context/base/l-unicode.lua b/tex/context/base/l-unicode.lua
index 813ffd54b..cf0c93d4f 100644
--- a/tex/context/base/l-unicode.lua
+++ b/tex/context/base/l-unicode.lua
@@ -777,11 +777,19 @@ end
 local _, l_remap = utf.remapper(little)
 local _, b_remap = utf.remapper(big)
 
+function utf.utf8_to_utf16_be(str)
+    return char(254,255) .. lpegmatch(b_remap,str)
+end
+
+function utf.utf8_to_utf16_le(str)
+    return char(255,254) .. lpegmatch(l_remap,str)
+end
+
 function utf.utf8_to_utf16(str,littleendian)
     if littleendian then
-        return char(255,254) .. lpegmatch(l_remap,str)
+        return utf.utf8_to_utf16_le(str)
     else
-        return char(254,255) .. lpegmatch(b_remap,str)
+        return utf.utf8_to_utf16_be(str)
     end
 end
 
diff --git a/tex/context/base/meta-ini.mkiv b/tex/context/base/meta-ini.mkiv
index 16ecb4ca1..e3e694448 100644
--- a/tex/context/base/meta-ini.mkiv
+++ b/tex/context/base/meta-ini.mkiv
@@ -436,9 +436,9 @@
 
 % watch out, this is a type1 font because mp can only handle 8 bit fonts
 
-\startMPinitializations % scale is not yet ok
-    defaultfont:="rm-lmtt10";
-\stopMPinitializations
+% \startMPinitializations % scale is not yet ok
+%     defaultfont:="rm-lmtt10";
+% \stopMPinitializations
 
 %D A signal that we're in combined \CONTEXT||\METAFUN mode:
 
diff --git a/tex/context/base/node-fin.mkiv b/tex/context/base/node-fin.mkiv
index 09bac6c08..2eb033fc1 100644
--- a/tex/context/base/node-fin.mkiv
+++ b/tex/context/base/node-fin.mkiv
@@ -23,8 +23,12 @@
 
 % we might have two variants at some point (efficiency)
 
-\def\finalizeobjectbox #1{\ctxcommand{finalizebox(\number#1)}}
-\def\finalizeshipoutbox#1{\ctxcommand{finalizebox(\number#1)}}
+\unexpanded\def\finalizeobjectbox #1{\ctxcommand{finalizebox(\number#1)}}
+\unexpanded\def\finalizeshipoutbox#1{\ctxcommand{finalizebox(\number#1)}}
+
+% Experimental (for Aditya):
+
+\unexpanded\def\cleanupbox#1{\ctxcommand{cleanupbox(\number#1)}}
 
 % Tricky stuff: this might become obsolete.
 
diff --git a/tex/context/base/node-shp.lua b/tex/context/base/node-shp.lua
index 32024e5da..6ebfd767f 100644
--- a/tex/context/base/node-shp.lua
+++ b/tex/context/base/node-shp.lua
@@ -14,6 +14,7 @@ local concat, sortedpairs = table.concat, table.sortedpairs
 local setmetatableindex = table.setmetatableindex
 
 local nodecodes      = nodes.nodecodes
+local whatsitcodes   = nodes.whatsitcodes
 local tasks          = nodes.tasks
 local handlers       = nodes.handlers
 
@@ -23,6 +24,7 @@ local disc_code      = nodecodes.disc
 local mark_code      = nodecodes.mark
 local kern_code      = nodecodes.kern
 local glue_code      = nodecodes.glue
+local whatsit_code   = nodecodes.whatsit
 
 local texgetbox      = tex.getbox
 
@@ -30,21 +32,44 @@ local free_node      = node.free
 local remove_node    = node.remove
 local traverse_nodes = node.traverse
 
-local function cleanup(head) -- rough
+local removables     = {
+    [whatsitcodes.open]       = true,
+    [whatsitcodes.close]      = true,
+    [whatsitcodes.write]      = true,
+    [whatsitcodes.pdfdest]    = true,
+    [whatsitcodes.pdfsavepos] = true,
+    [whatsitcodes.latelua]    = true,
+}
+
+local function cleanup_redundant(head)
     local start = head
     while start do
         local id = start.id
-        if id == disc_code or (id == glue_code and not start.writable) or (id == kern_code and start.kern == 0) or id == mark_code then
-            head, start, tmp = remove_node(head,start)
-            free_node(tmp)
+        if id == disc_code then
+            head, start = remove_node(head,start,true)
+     -- elseif id == glue_code then
+     --     if start.writable then
+     --         start = start.next
+     --     elseif some_complex_check_on_glue_spec then
+     --         head, start = remove_node(head,start,true)
+     --     else
+     --         start = start.next
+     --     end
+        elseif id == kern_code then
+            if start.kern == 0 then
+                head, start = remove_node(head,start,true)
+            else
+                start = start.next
+            end
+        elseif id == mark_code then
+            head, start = remove_node(head,start,true)
         elseif id == hlist_code or id == vlist_code then
             local sl = start.list
             if sl then
-                start.list = cleanup(sl)
+                start.list = cleanup_redundant(sl)
                 start = start.next
             else
-                head, start, tmp = remove_node(head,start)
-                free_node(tmp)
+                head, start = remove_node(head,start,true)
             end
         else
             start = start.next
@@ -53,21 +78,50 @@ local function cleanup(head) -- rough
     return head
 end
 
-directives.register("backend.cleanup", function()
-    tasks.enableaction("shipouts","nodes.handlers.cleanuppage")
-end)
+local function cleanup_flushed(head) -- rough
+    local start = head
+    while start do
+        local id = start.id
+        if id == whatsit_code and removables[start.subtype] then
+            head, start = remove_node(head,start,true)
+        elseif id == hlist_code or id == vlist_code then
+            local sl = start.list
+            if sl then
+                start.list = cleanup_flushed(sl)
+                start = start.next
+            else
+                head, start = remove_node(head,start,true)
+            end
+        else
+            start = start.next
+        end
+    end
+    return head
+end
 
 function handlers.cleanuppage(head)
     -- about 10% of the nodes make no sense for the backend
-    return cleanup(head), true
+    return cleanup_redundant(head), true
 end
 
+function handlers.cleanupbox(head)
+    return cleanup_flushed(head), true
+end
+
+directives.register("backend.cleanup", function()
+    tasks.enableaction("shipouts","nodes.handlers.cleanuppage")
+end)
+
 local actions = tasks.actions("shipouts")  -- no extra arguments
 
 function handlers.finalize(head) -- problem, attr loaded before node, todo ...
     return actions(head)
 end
 
+function commands.cleanupbox(n)
+    cleanup_flushed(texgetbox(n))
+end
+
 -- handlers.finalize = actions
 
 -- interface
diff --git a/tex/context/base/page-imp.mkiv b/tex/context/base/page-imp.mkiv
index c22e9e646..cfa535ab2 100644
--- a/tex/context/base/page-imp.mkiv
+++ b/tex/context/base/page-imp.mkiv
@@ -293,7 +293,7 @@
      \fi
    \fi}
 
-\def\installpagearrangement #1 % will lchange, no space
+\def\installpagearrangement #1 % will change, no space
   {\setgvalue{\??layoutarranger#1}}
 
 \def\checkinstalledpagearrangement#1% can be empty: aaa,,bbb
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index a984f1c6b..88393860d 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.log b/tex/context/base/status-lua.log
index ae1d43e6c..2afe02313 100644
--- a/tex/context/base/status-lua.log
+++ b/tex/context/base/status-lua.log
@@ -1,6 +1,6 @@
 (cont-yes.mkiv
 
-ConTeXt  ver: 2013.08.28 23:08 MKIV beta  fmt: 2013.8.28  int: english/english
+ConTeXt  ver: 2013.08.29 20:45 MKIV beta  fmt: 2013.8.29  int: english/english
 
 system          > 'cont-new.mkiv' loaded
 (cont-new.mkiv)
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index ff4758068..88b45182e 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 08/28/13 23:08:58
+-- merge date  : 08/29/13 20:45:59
 
 do -- begin closure to overcome local limits and interference
 
@@ -127,11 +127,11 @@ local dquote=P('"')
 local space=P(" ")
 local period=P(".")
 local comma=P(",")
-local utfbom_32_be=P('\000\000\254\255')
-local utfbom_32_le=P('\255\254\000\000')
-local utfbom_16_be=P('\255\254')
-local utfbom_16_le=P('\254\255')
-local utfbom_8=P('\239\187\191')
+local utfbom_32_be=P('\000\000\254\255') 
+local utfbom_32_le=P('\255\254\000\000') 
+local utfbom_16_be=P('\254\255')     
+local utfbom_16_le=P('\255\254')     
+local utfbom_8=P('\239\187\191')   
 local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8
 local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") 
 local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0)