beta 2011.07.21 18:09

24 files changed, 446 insertions, 147 deletions

diff --git a/scripts/context/lua/mtx-pdf.lua b/scripts/context/lua/mtx-pdf.lua
new file mode 100644
index 000000000..5654b8bc4
--- /dev/null
+++ b/scripts/context/lua/mtx-pdf.lua
@@ -0,0 +1,117 @@
+if not modules then modules = { } end modules ['mtx-pdf'] = {
+    version   = 1.001,
+    comment   = "companion to mtxrun.lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local helpinfo = [[
+--info                show some info about the given file
+--metadata            show metadata xml blob
+]]
+
+local application = logs.application {
+    name     = "mtx-pdf",
+    banner   = "ConTeXt PDF Helpers 0.01",
+    helpinfo = helpinfo,
+}
+
+local report = application.report
+
+dofile(resolvers.findfile("lpdf-epd.lua","tex"))
+
+scripts     = scripts     or { }
+scripts.pdf = scripts.pdf or { }
+
+local function loadpdffile(filename)
+    if not filename or filename == "" then
+        report("no filename given")
+    elseif not lfs.isfile(filename) then
+        report("unknown file '%s'",filename)
+    else
+        local pdffile  = lpdf.epdf.load(filename)
+        if pdffile then
+            return pdffile
+        else
+            report("no valid pdf file '%s'",filename)
+        end
+    end
+end
+
+function scripts.pdf.info()
+    local filename = environment.files[1]
+    local pdffile  = loadpdffile(filename)
+    if pdffile then
+        local catalog  = pdffile.Catalog
+        local info     = pdffile.Info
+        local pages    = pdffile.pages
+        local nofpages = pages.n -- no # yet. will be in 5.2
+
+        report("filename > %s",filename)
+        report("pdf version > %s",catalog.Version)
+        report("number of pages > %s",nofpages)
+        report("title > %s",info.Title)
+        report("creator > %s",info.Creator)
+        report("producer > %s",info.Producer)
+        report("creation date > %s",info.CreationDate)
+        report("modification date > %s",info.ModDate)
+
+        local width, height, start
+        for i=1, nofpages do
+            local page = pages[i]
+            local bbox = page.CropBox or page.MediaBox
+            local w, h = bbox[4]-bbox[2],bbox[3]-bbox[1]
+            if w ~= width or h ~= height then
+                if start then
+                    report("cropbox > pages: %s-%s, width: %s, height: %s",start,i-1,width,height)
+                end
+                width, height, start = w, h, i
+            end
+        end
+        report("cropbox > pages: %s-%s, width: %s, height: %s",start,nofpages,width,height)
+    end
+end
+
+function scripts.pdf.metadata()
+    local filename = environment.files[1]
+    local pdffile  = loadpdffile(filename)
+    if pdffile then
+        local catalog  = pdffile.Catalog
+        local metadata = catalog.Metadata
+        if metadata then
+            report("metadata > \n\n%s\n",metadata())
+        else
+            report("no metadata")
+        end
+    end
+end
+
+if environment.argument("info") then
+    scripts.pdf.info()
+elseif environment.argument("metadata") then
+    scripts.pdf.metadata()
+else
+    application.help()
+end
+
+-- a variant on an experiment by hartmut
+
+--~ function downloadlinks(filename)
+--~     local document = lpdf.epdf.load(filename)
+--~     if document then
+--~         local pages = document.pages
+--~         for p = 1,#pages do
+--~             local annotations = pages[p].Annots
+--~             if annotations then
+--~                 for a=1,#annotations do
+--~                     local annotation = annotations[a]
+--~                     local uri = annotation.Subtype == "Link" and annotation.A and annotation.A.URI
+--~                     if uri and string.find(uri,"^http") then
+--~                         os.execute("wget " .. uri)
+--~                     end
+--~                 end
+--~             end
+--~         end
+--~     end
+--~ end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 29665417e..158d11ecd 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -1350,7 +1350,7 @@ local utflinesplitter = utfbom^-1 * tsplitat(newline)
 patterns.utflinesplitter = utflinesplitter
 
 function string.utfsplitlines(str)
-    return match(utflinesplitter,str)
+    return match(utflinesplitter,str or "")
 end
 
 
@@ -3902,6 +3902,40 @@ end
 
 
 
+local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local one  = P(1)
+local two  = C(1) * C(1)
+local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
+
+local pattern = P("\254\255") * Cs( (
+                    four   / function(a,b,c,d)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(a,b)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+              + P("\255\254") * Cs( (
+                    four   / function(b,a,d,c)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(b,a)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+
+function string.toutf(s)
+    return lpegmatch(pattern,s) or s -- todo: utf32
+end
+
+
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 29665417e..158d11ecd 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -1350,7 +1350,7 @@ local utflinesplitter = utfbom^-1 * tsplitat(newline)
 patterns.utflinesplitter = utflinesplitter
 
 function string.utfsplitlines(str)
-    return match(utflinesplitter,str)
+    return match(utflinesplitter,str or "")
 end
 
 
@@ -3902,6 +3902,40 @@ end
 
 
 
+local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local one  = P(1)
+local two  = C(1) * C(1)
+local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
+
+local pattern = P("\254\255") * Cs( (
+                    four   / function(a,b,c,d)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(a,b)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+              + P("\255\254") * Cs( (
+                    four   / function(b,a,d,c)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(b,a)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+
+function string.toutf(s)
+    return lpegmatch(pattern,s) or s -- todo: utf32
+end
+
+
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 29665417e..158d11ecd 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -1350,7 +1350,7 @@ local utflinesplitter = utfbom^-1 * tsplitat(newline)
 patterns.utflinesplitter = utflinesplitter
 
 function string.utfsplitlines(str)
-    return match(utflinesplitter,str)
+    return match(utflinesplitter,str or "")
 end
 
 
@@ -3902,6 +3902,40 @@ end
 
 
 
+local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local one  = P(1)
+local two  = C(1) * C(1)
+local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
+
+local pattern = P("\254\255") * Cs( (
+                    four   / function(a,b,c,d)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(a,b)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+              + P("\255\254") * Cs( (
+                    four   / function(b,a,d,c)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(b,a)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+
+function string.toutf(s)
+    return lpegmatch(pattern,s) or s -- todo: utf32
+end
+
+
 end -- of closure
 
 do -- create closure to overcome 200 locals limit
diff --git a/tex/context/base/back-pdf.mkiv b/tex/context/base/back-pdf.mkiv
index 1737e9866..469ead0db 100644
--- a/tex/context/base/back-pdf.mkiv
+++ b/tex/context/base/back-pdf.mkiv
@@ -13,6 +13,7 @@
 
 \writestatus{loading}{ConTeXt Backend Macros / PDF}
 
+%registerctxluafile{lpdf-aux}{1.001} % common helpers
 \registerctxluafile{lpdf-ini}{1.001}
 \registerctxluafile{lpdf-nod}{1.001}
 \registerctxluafile{lpdf-col}{1.000}
@@ -99,7 +100,7 @@
 \def\pdfbackendsetpattern       #1#2{\ctxlua{lpdf.adddocumentpattern   ("#1",lpdf.verbose(\!!bs#2\!!es))}}
 \def\pdfbackendsetshade         #1#2{\ctxlua{lpdf.adddocumentshade     ("#1",lpdf.verbose(\!!bs#2\!!es))}}
 
-\def\pdfbackendcurrentresources     {\ctxlua{lpdf.collectedresources()}}
+\def\pdfbackendcurrentresources     {\cldcontext{lpdf.collectedresources()}}
 
 \def\pdfcolor                     #1{\ctxlua{lpdf.pdfcolor(\thecolorattribute{#1})}} \let\PDFcolor\pdfcolor
 
@@ -125,7 +126,7 @@
 
 \def\dostartrotation#1% grouped
   {\forcecolorhack % maybe use signal instead
-   \pdfliteral{q \ctxlua{lpdf.rotationcm(#1)}}}
+   \pdfliteral{q \cldcontext{lpdf.rotationcm(#1)}}}
 
 \def\dostoprotation
   {\pdfliteral{Q}}
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index 3c18bc96d..17dcba9f9 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2011.07.20 14:10}
+\newcontextversion{2011.07.21 18:09}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 1d44c51c3..83c7d3d20 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2011.07.20 14:10}
+\newcontextversion{2011.07.21 18:09}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index cd8d1c540..312e192ad 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2011.07.20 14:10}
+\edef\contextversion{2011.07.21 18:09}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index ee369b8c5..90da5a12c 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2011.07.20 14:10}
+\edef\contextversion{2011.07.21 18:09}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/file-job.lua b/tex/context/base/file-job.lua
index 8cd7ffeb8..4c7811fc5 100644
--- a/tex/context/base/file-job.lua
+++ b/tex/context/base/file-job.lua
@@ -233,7 +233,8 @@ local function useanyfile(name,onlyonce)
     if s then
         s(file.removesuffix(name),onlyonce)
     else
-        resolvers.readfilename(name) -- might change
+        usetexfile(name,onlyonce) -- e.g. ctx file
+--~         resolvers.readfilename(name)
     end
 end
 
diff --git a/tex/context/base/file-res.lua b/tex/context/base/file-res.lua
index d0db95f59..24b0f1cfc 100644
--- a/tex/context/base/file-res.lua
+++ b/tex/context/base/file-res.lua
@@ -22,7 +22,7 @@ local found = { } -- can best be done in the resolver itself
 
 local function readfilename(specification,backtrack,treetoo)
     local name = specification.filename
-    local fnd = found[name]
+    local fnd = name and found[name]
     if not fnd then
         if isfile(name) then
             if trace_files then
@@ -66,6 +66,8 @@ local function readfilename(specification,backtrack,treetoo)
     return fnd or ""
 end
 
+--~ resolvers.readfilename = readfilename -- bonus use getreadfilename instead
+
 function finders.job(specification) return readfilename(specification,false,                 false) end -- current path, no backtracking
 function finders.loc(specification) return readfilename(specification,resolvers.maxreadlevel,false) end -- current path, backtracking
 function finders.sys(specification) return readfilename(specification,false,                 true ) end -- current path, obeys tex search
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index 4b1a378ec..de9d9e5e3 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -284,7 +284,7 @@ local utflinesplitter = utfbom^-1 * tsplitat(newline)
 patterns.utflinesplitter = utflinesplitter
 
 function string.utfsplitlines(str)
-    return match(utflinesplitter,str)
+    return match(utflinesplitter,str or "")
 end
 
 --~ lpeg.splitters = cache -- no longer public
diff --git a/tex/context/base/l-unicode.lua b/tex/context/base/l-unicode.lua
index f30c32b9a..6295ecb7d 100644
--- a/tex/context/base/l-unicode.lua
+++ b/tex/context/base/l-unicode.lua
@@ -426,3 +426,40 @@ end
 --~     end
 --~ end
 --~ print(os.clock()-t,collectgarbage("count")*1024-u)
+
+--~ local byte = string.byte
+--~ local utfchar = utf.char
+--~ local lpegmatch = lpeg.match, lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local one  = P(1)
+local two  = C(1) * C(1)
+local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
+
+local pattern = P("\254\255") * Cs( (
+                    four   / function(a,b,c,d)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(a,b)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+              + P("\255\254") * Cs( (
+                    four   / function(b,a,d,c)
+                                local ab = 0xFF * byte(a) + byte(b)
+                                local cd = 0xFF * byte(c) + byte(d)
+                                return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+                            end
+                  + two   / function(b,a)
+                                return utfchar(byte(a)*256 + byte(b))
+                            end
+                  + one
+                )^1 )
+
+function string.toutf(s)
+    return lpegmatch(pattern,s) or s -- todo: utf32
+end
diff --git a/tex/context/base/lpdf-epd.lua b/tex/context/base/lpdf-epd.lua
index 19c1f9bd8..3e917dba2 100644
--- a/tex/context/base/lpdf-epd.lua
+++ b/tex/context/base/lpdf-epd.lua
@@ -23,8 +23,9 @@ if not modules then modules = { } end modules ['lpdf-epd'] = {
 -- We cannot access all destinations in one run.
 
 local setmetatable, rawset, rawget, tostring, tonumber = setmetatable, rawset, rawget, tostring, tonumber
-local lower, match, char = string.lower, string.match, string.char
+local lower, match, char, find, sub = string.lower, string.match, string.char, string.find, string.sub
 local concat = table.concat
+local toutf = string.toutf
 
 function epdf.type(o)
     local t = lower(match(tostring(o),"[^ :]+"))
@@ -51,8 +52,10 @@ local function prepare(document,d,t,n,k)
                 --
             else
                 c = checked_access[v:getTypeName()](v,document,r)
-                document.cache[r] = c
-                document.xrefs[c] = r
+                if c then
+                    document.cache[r] = c
+                    document.xrefs[c] = r
+                end
             end
             t[d:getKey(i)] = c
         end
@@ -149,7 +152,7 @@ checked_access = {
         return v:getNum()
     end,
     string = function(v)
-        return v:getString()
+        return toutf(v:getString())
     end,
     boolean = function(v)
         return v:getBool()
@@ -241,7 +244,8 @@ local function getpages(document)
     local cata  = data:getCatalog()
     local xref  = data:getXRef()
     local pages = { }
-    for pagenumber=1,cata:getNumPages() do
+    local nofpages = cata:getNumPages()
+    for pagenumber=1,nofpages do
         local pagereference = cata:getPageRef(pagenumber).num
         local pagedata = some_dictionary(xref:fetch(pagereference,0):getDict(),document,pagereference)
         pagedata.number = pagenumber
@@ -249,6 +253,7 @@ local function getpages(document)
         xrefs[pagedata] = pagereference
         cache[pagereference] = pagedata
     end
+    pages.n = nofpages
     return pages
 end
 
@@ -281,14 +286,16 @@ function lpdf.epdf.load(filename)
                 data     = data,
             }
             local Catalog    = some_dictionary(data:getXRef():getCatalog():getDict(),document)
+            local Info       = some_dictionary(data:getXRef():getDocInfo():getDict(),document)
             document.Catalog = Catalog
+            document.Info    = Info
          -- document.catalog = Catalog
             -- a few handy helper tables
             document.pages         = delayed(document,"pages",        function() return getpages(document) end)
-            document.destinations  = delayed(document,"destinations", function() return getnames(document,Catalog.Names.Dests) end)
-            document.javascripts   = delayed(document,"javascripts",  function() return getnames(document,Catalog.Names.JS) end)
-            document.widgets       = delayed(document,"widgets",      function() return getnames(document,Catalog.Names.AcroForm) end)
-            document.embeddedfiles = delayed(document,"embeddedfiles",function() return getnames(document,Catalog.Names.EmbeddedFiles) end)
+            document.destinations  = delayed(document,"destinations", function() return getnames(document,Catalog.Names and Catalog.Names.Dests) end)
+            document.javascripts   = delayed(document,"javascripts",  function() return getnames(document,Catalog.Names and Catalog.Names.JS) end)
+            document.widgets       = delayed(document,"widgets",      function() return getnames(document,Catalog.Names and Catalog.Names.AcroForm) end)
+            document.embeddedfiles = delayed(document,"embeddedfiles",function() return getnames(document,Catalog.Names and Catalog.Names.EmbeddedFiles) end)
             document.layers        = delayed(document,"layers",       function() return getlayers(document) end)
         else
             document = false
diff --git a/tex/context/base/lpdf-ini.lua b/tex/context/base/lpdf-ini.lua
index cb40edf89..e07f9748a 100644
--- a/tex/context/base/lpdf-ini.lua
+++ b/tex/context/base/lpdf-ini.lua
@@ -9,17 +9,15 @@ if not modules then modules = { } end modules ['lpdf-ini'] = {
 local setmetatable, getmetatable, type, next, tostring, tonumber, rawset = setmetatable, getmetatable, type, next, tostring, tonumber, rawset
 local char, byte, format, gsub, concat, match, sub, gmatch = string.char, string.byte, string.format, string.gsub, table.concat, string.match, string.sub, string.gmatch
 local utfvalues = string.utfvalues
-local texset = tex.set
+local utfchar = utf.char
 local sind, cosd = math.sind, math.cosd
-local lpegmatch = lpeg.match
+local lpegmatch, P, C, R, S, Cc, Cs = lpeg.match, lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cc, lpeg.Cs
 
 local pdfreserveobject   = pdf.reserveobj
 local pdfimmediateobject = pdf.immediateobj
 local pdfdeferredobject  = pdf.obj
 local pdfreferenceobject = pdf.refobj
 
-local pdfobject          = pdf.obj
-
 local trace_finalizers = false  trackers.register("backend.finalizers", function(v) trace_finalizers = v end)
 local trace_resources  = false  trackers.register("backend.resources",  function(v) trace_resources  = v end)
 local trace_objects    = false  trackers.register("backend.objects",    function(v) trace_objects    = v end)
@@ -28,7 +26,7 @@ local trace_detail     = false  trackers.register("backend.detail",     function
 local report_objects    = logs.reporter("backend","objects")
 local report_finalizing = logs.reporter("backend","finalizing")
 
-local backends, context = backends, context
+local backends = backends
 
 backends.pdf = backends.pdf or {
     comment        = "backend for directly generating pdf output",
@@ -41,7 +39,7 @@ backends.pdf = backends.pdf or {
 lpdf       = lpdf or { }
 local lpdf = lpdf
 
-local function tosixteen(str)
+local function tosixteen(str) -- an lpeg might be faster
     if not str or str == "" then
         return "<feff>" -- not () as we want an indication that it's unicode
     else
@@ -60,7 +58,7 @@ local function tosixteen(str)
     end
 end
 
-lpdf.tosixteen = tosixteen
+lpdf.tosixteen   = tosixteen
 
 -- lpeg is some 5 times faster than gsub (in test) on escaping
 
@@ -72,7 +70,7 @@ lpdf.tosixteen = tosixteen
 --     ["("] = "\\(", [")"] = "\\)",
 -- }
 --
--- local escaped = lpeg.Cs(lpeg.Cc("(") * (lpeg.S("\\/#<>[]()")/escapes + lpeg.P(1))^0 * lpeg.Cc(")"))
+-- local escaped = Cs(Cc("(") * (S("\\/#<>[]()")/escapes + P(1))^0 * Cc(")"))
 --
 -- local function toeight(str)
 --     if not str or str == "" then
@@ -287,7 +285,7 @@ for s in gmatch(forbidden,".") do
     replacements[s] = format("#%02x",byte(s))
 end
 
-local escaped = lpeg.Cs(lpeg.Cc("/") * (lpeg.S(forbidden)/replacements + lpeg.P(1))^0)
+local escaped = Cs(Cc("/") * (S(forbidden)/replacements + P(1))^0)
 
 local function pdfconstant(str,default)
     str = str or default or ""
@@ -376,7 +374,7 @@ end
 -- lpdf.immediateobject = pdfimmediateobject
 -- lpdf.deferredobject  = pdfdeferredobject
 -- lpdf.object          = pdfdeferredobject
--- lpdf.referenceobject    = pdfreferenceobject
+-- lpdf.referenceobject = pdfreferenceobject
 
 lpdf.pagereference      = pdf.pageref or tex.pdfpageref
 lpdf.registerannotation = pdf.registerannot
@@ -421,7 +419,7 @@ function lpdf.flushstreamobject(data,dict,compressed) -- default compressed
         report_objects("flushing stream object of %s bytes",#data)
     end
     local dtype = type(dict)
-    return pdfobject {
+    return pdfdeferredobject {
         immediate     = true,
         compresslevel = compressed == false and 0 or nil,
         type          = "stream",
@@ -435,7 +433,7 @@ function lpdf.flushstreamfileobject(filename,dict,compressed) -- default compres
         report_objects("flushing stream file object '%s'",filename)
     end
     local dtype = type(dict)
-    return pdfobject {
+    return pdfdeferredobject {
         immediate     = true,
         compresslevel = compressed == false and 0 or nil,
         type          = "stream",
@@ -663,17 +661,6 @@ local function flushcolorspaces() if next(d_colorspaces) then trace_flush("color
 local function flushpatterns   () if next(d_patterns   ) then trace_flush("patterns")    pdfimmediateobject(r_patterns,   tostring(d_patterns   )) end end
 local function flushshades     () if next(d_shades     ) then trace_flush("shades")      pdfimmediateobject(r_shades,     tostring(d_shades     )) end end
 
---~ local collected = pdfdictionary {
---~     ExtGState  = p_extgstates,
---~     ColorSpace = p_colorspaces,
---~     Pattern    = p_patterns,
---~     Shading    = p_shades,
---~ } ; collected = collected()
-
---~ function lpdf.collectedresources()
---~     context(collected)
---~ end
-
 function lpdf.collectedresources()
     local ExtGState  = next(d_extgstates ) and p_extgstates
     local ColorSpace = next(d_colorspaces) and p_colorspaces
@@ -687,7 +674,9 @@ function lpdf.collectedresources()
             Shading    = Shading,
          -- ProcSet    = pdfarray { pdfconstant("PDF") },
         }
-        context(collected())
+        return collected()
+    else
+        return ""
     end
 end
 
@@ -714,7 +703,7 @@ registerpagefinalizer(checkshades,3,"shades")
 
 function lpdf.rotationcm(a)
     local s, c = sind(a), cosd(a)
-    context("%s %s %s %s 0 0 cm",c,s,-s,c)
+    return format("%s %s %s %s 0 0 cm",c,s,-s,c)
 end
 
 -- ! -> universaltime
diff --git a/tex/context/base/phys-dim.lua b/tex/context/base/phys-dim.lua
index bc65fcff2..41f98b5fb 100644
--- a/tex/context/base/phys-dim.lua
+++ b/tex/context/base/phys-dim.lua
@@ -241,6 +241,9 @@ local short_units_to_long = {
     u  = "Hour",
     h  = "Hour",
     s  = "Second",
+    g  = "Gram",
+    n  = "Newton",
+    v  = "Volt",
 
     Litre = "Liter",
     Metre = "Meter",
@@ -306,9 +309,9 @@ local s_suffix   = appendlpeg(keys(short_suffixes))
 
 -- square centi meter per square kilo seconds
 
-local l_suffix       = Cs(somespace * l_suffix)
-local s_suffix       = Cs(somespace * s_suffix) + Cc("")
-local l_operator     = Cs(somespace * l_operator)
+local l_suffix   = Cs(somespace * l_suffix)
+local s_suffix   = Cs(somespace * s_suffix) + Cc("")
+local l_operator = Cs(somespace * l_operator)
 
 local combination = P { "start",
     l_prefix = Cs(somespace * l_prefix) + Cc(""),
@@ -321,6 +324,8 @@ local combination = P { "start",
              + V("s_prefix") * V("l_unit"),
 }
 
+--~ inspect(s_prefix)
+--~ inspect(s_unit)
 -- square kilo meter
 -- square km
 
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index def98ab1d..a948649ea 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 897c36fd7..8358053b6 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/strc-ren.mkiv b/tex/context/base/strc-ren.mkiv
index be6758a21..3106d7cb7 100644
--- a/tex/context/base/strc-ren.mkiv
+++ b/tex/context/base/strc-ren.mkiv
@@ -25,9 +25,11 @@
 
 \newevery \everyheadstart \relax
 
-\unexpanded\def\placeheadmargintexts
-  {\the\everyheadstart
-   \doif{\headparameter\c!margintext}\v!yes\placemargincontent}
+% \unexpanded\def\placeheadmargintexts
+%   {\the\everyheadstart
+%    \doif{\headparameter\c!margintext}\v!yes\placemargincontent}
+
+\let\placeheadmargintexts\relax
 
 \unexpanded\def\setupheadcomponentfont#1#2%
   {\dontconvertfont
diff --git a/tex/context/base/typo-mar.lua b/tex/context/base/typo-mar.lua
index 9f28c4dbc..e356bd7d0 100644
--- a/tex/context/base/typo-mar.lua
+++ b/tex/context/base/typo-mar.lua
@@ -19,7 +19,8 @@ local setmetatable, next = setmetatable, next
 
 local attributes, nodes, node, variables = attributes, nodes, node, variables
 
-local trace_margindata = false  trackers.register("typesetters.margindata", function(v) trace_margindata = v end)
+local trace_margindata  = false  trackers.register("typesetters.margindata",       function(v) trace_margindata  = v end)
+local trace_marginstack = false  trackers.register("typesetters.margindata.stack", function(v) trace_marginstack = v end)
 
 local report_margindata = logs.reporter("typesetters","margindata")
 
@@ -145,6 +146,17 @@ local defaults = {
 
 local enablelocal, enableglobal -- forward reference (delayed initialization)
 
+local function showstore(store,banner)
+    if #store == 0 then
+        report_margindata("%s: nothing stored",banner)
+    else
+        for i=1,#store do
+            local si =store[i]
+            report_margindata("%s: stored at %s: %s => %s",banner,i,si.name or "no name",nodes.toutf(si.box.list))
+        end
+    end
+end
+
 function margins.save(t)
     setmetatable(t,defaults)
     local inline   = t.inline
@@ -175,6 +187,9 @@ function margins.save(t)
     nofsaved = nofsaved + 1
     nofstored = nofstored + 1
     local name = t.name
+    if trace_marginstack then
+        showstore(store,"before ")
+    end
     if name and name ~= "" then
         for i=#store,1,-1 do
             local si = store[i]
@@ -183,35 +198,43 @@ function margins.save(t)
                 free_node_list(s.box)
             end
         end
+        if trace_marginstack then
+            showstore(store,"between")
+        end
     end
-    -- better make a new table and make t entry in t
-    t.box                 = copy_node_list(texbox[t.number])
-    t.n                   = nofsaved
-    -- used later (we will clean up this natural mess later)
-    -- nice is to make a special status table mechanism
-    local leftmargindistance  = texdimen.naturalleftmargindistance
-    local rightmargindistance = texdimen.naturalrightmargindistance
-    t.strutdepth          = texbox.strutbox.depth
-    t.strutheight         = texbox.strutbox.height
-    t.leftskip            = tex.leftskip.width
-    t.rightskip           = tex.rightskip.width
-    t.leftmargindistance  = leftmargindistance
-    t.rightmargindistance = rightmargindistance
-    t.leftedgedistance    = texdimen.naturalleftedgedistance
-                          + texdimen.leftmarginwidth
-                          + leftmargindistance
-    t.rightedgedistance   = texdimen.naturalrightedgedistance
-                          + texdimen.rightmarginwidth
-                          + rightmargindistance
-    t.lineheight          = texdimen.lineheight
-    --
- -- t.realpageno          = texcount.realpageno
-    if inline then
-        context(new_usernumber(inline_mark,nofsaved))
-        store[nofsaved] = t -- no insert
-        nofinlined = nofinlined + 1
-    else
-        insert(store,t)
+    if t.number then
+        -- better make a new table and make t entry in t
+        t.box                 = copy_node_list(texbox[t.number])
+        t.n                   = nofsaved
+        -- used later (we will clean up this natural mess later)
+        -- nice is to make a special status table mechanism
+        local leftmargindistance  = texdimen.naturalleftmargindistance
+        local rightmargindistance = texdimen.naturalrightmargindistance
+        t.strutdepth          = texbox.strutbox.depth
+        t.strutheight         = texbox.strutbox.height
+        t.leftskip            = tex.leftskip.width
+        t.rightskip           = tex.rightskip.width
+        t.leftmargindistance  = leftmargindistance
+        t.rightmargindistance = rightmargindistance
+        t.leftedgedistance    = texdimen.naturalleftedgedistance
+                              + texdimen.leftmarginwidth
+                              + leftmargindistance
+        t.rightedgedistance   = texdimen.naturalrightedgedistance
+                              + texdimen.rightmarginwidth
+                              + rightmargindistance
+        t.lineheight          = texdimen.lineheight
+        --
+     -- t.realpageno          = texcount.realpageno
+        if inline then
+            context(new_usernumber(inline_mark,nofsaved))
+            store[nofsaved] = t -- no insert
+            nofinlined = nofinlined + 1
+        else
+            insert(store,t)
+        end
+    end
+    if trace_marginstack then
+        showstore(store,"after  ")
     end
     if trace_margindata then
         report_margindata("saved: %s, location: %s, scope: %s, inline: %s",nofsaved,location,scope,tostring(inline))
diff --git a/tex/context/base/typo-mar.mkiv b/tex/context/base/typo-mar.mkiv
index 0a8e1b1d2..8093f17c9 100644
--- a/tex/context/base/typo-mar.mkiv
+++ b/tex/context/base/typo-mar.mkiv
@@ -162,72 +162,85 @@
   {\iffirstargument
      \setupmargindata[\currentmargindata][#dataparameters]%
    \fi
-   \edef\currentmargindatastrut{\margindataparameter\c!strut}%
-   \the\everymargindatacontent
-   \dostarttagged\t!margintext\currentmargindata
-   \ifcsname\??mf\currentmargindata\s!parent\endcsname
-     \setbox\nextbox\hbox\bgroup
-       \the\everymargindatacontent
-       \dosetmargindataattributes\c!style\c!color
-       \localframedwithsettings[\??mf\currentmargindata][\c!location=\v!normal,#textparameters]\bgroup
-         \ifx\currentmargindatastrut\empty \else
-           \dosetupstrut[\currentmargindatastrut]%
-         \fi
-         \begstrut\margindataparameter\c!command{#content}\endstrut
-       \egroup
-     \egroup
-     \edef\currentmarginfirstheight{\number\dimexpr\framedfirstheight}%
-   \else
-     \edef\currentmargindatawidth{\margindataparameter\c!width}%
-     \ifx\currentmargindatawidth\empty
+   \doifelsenothing{#content}\donefalse\donetrue
+   \ifdone
+     \edef\currentmargindatastrut{\margindataparameter\c!strut}%
+     \the\everymargindatacontent
+     \dostarttagged\t!margintext\currentmargindata
+     \ifcsname\??mf\currentmargindata\s!parent\endcsname
        \setbox\nextbox\hbox\bgroup
          \the\everymargindatacontent
          \dosetmargindataattributes\c!style\c!color
-         \ifx\currentmargindatastrut\empty \else
-           \dosetupstrut[\currentmargindatastrut]%
-         \fi
-         \begstrut\margindataparameter\c!command{#content}\endstrut
+         \localframedwithsettings[\??mf\currentmargindata][\c!location=\v!normal,#textparameters]\bgroup
+           \ifx\currentmargindatastrut\empty \else
+             \dosetupstrut[\currentmargindatastrut]%
+           \fi
+           \begstrut\margindataparameter\c!command{#content}\endstrut
+         \egroup
        \egroup
-       \let\currentmarginfirstheight\empty
+       \edef\currentmarginfirstheight{\number\dimexpr\framedfirstheight}%
      \else
-       \dosetraggedcommand{\margindataparameter\c!align}%
-       \setbox\nextbox\hbox \bgroup\vtop \bgroup % hbox is needed
-         \the\everymargindatacontent
-         \dosetmargindataattributes\c!style\c!color
-         \hsize\currentmargindatawidth
-         \raggedcommand
-         \ifx\currentmargindatastrut\empty \else
-           \dosetupstrut[\currentmargindatastrut]%
-         \fi
-         \begstrut\margindataparameter\c!command{#content}\endstrut
-       \egroup \egroup
-       \edef\currentmarginfirstheight{true}%
+       \edef\currentmargindatawidth{\margindataparameter\c!width}%
+       \ifx\currentmargindatawidth\empty
+         \setbox\nextbox\hbox\bgroup
+           \the\everymargindatacontent
+           \dosetmargindataattributes\c!style\c!color
+           \ifx\currentmargindatastrut\empty \else
+             \dosetupstrut[\currentmargindatastrut]%
+           \fi
+           \begstrut\margindataparameter\c!command{#content}\endstrut
+         \egroup
+         \let\currentmarginfirstheight\empty
+       \else
+         \dosetraggedcommand{\margindataparameter\c!align}%
+         \setbox\nextbox\hbox \bgroup\vtop \bgroup % hbox is needed
+           \the\everymargindatacontent
+           \dosetmargindataattributes\c!style\c!color
+           \hsize\currentmargindatawidth
+           \raggedcommand
+           \ifx\currentmargindatastrut\empty \else
+             \dosetupstrut[\currentmargindatastrut]%
+           \fi
+           \begstrut\margindataparameter\c!command{#content}\endstrut
+         \egroup \egroup
+         \edef\currentmarginfirstheight{true}%
+       \fi
      \fi
+     \dostoptagged
    \fi
-   \dostoptagged
-   \ctxlua{typesetters.margins.save{
-     \c!location = "\margindataparameter\c!location",
-     \c!method   = "\margindataparameter\c!method",
-     \c!category = "\margindataparameter\c!category",
-     \c!name     = "\margindataparameter\c!name",
-     \c!margin   = "\margindataparameter\c!margin", % local normal margin edge
-     \c!distance = \number\dimexpr\margindataparameter\c!distance,
-     \c!hoffset  = \number\dimexpr\margindataparameter\c!hoffset,
-     \c!voffset  = \number\dimexpr\margindataparameter\c!voffset,
-     \c!dy       = \number\dimexpr\margindataparameter\c!dy,
-   \ifx\currentmarginfirstheight\empty \else
-     baseline    = \currentmarginfirstheight,
-   \fi
-     threshold   = \number\dimexpr\margindataparameter\c!threshold, % will change
-   \ifhmode
-     inline      = true,
+   \ifdone
+     \ctxlua{typesetters.margins.save{
+       \c!location = "\margindataparameter\c!location",
+       \c!method   = "\margindataparameter\c!method",
+       \c!category = "\margindataparameter\c!category",
+       \c!name     = "\margindataparameter\c!name",
+       \c!margin   = "\margindataparameter\c!margin", % local normal margin edge
+       \c!distance = \number\dimexpr\margindataparameter\c!distance,
+       \c!hoffset  = \number\dimexpr\margindataparameter\c!hoffset,
+       \c!voffset  = \number\dimexpr\margindataparameter\c!voffset,
+       \c!dy       = \number\dimexpr\margindataparameter\c!dy,
+     \ifx\currentmarginfirstheight\empty \else
+       baseline    = \currentmarginfirstheight,
+     \fi
+       threshold   = \number\dimexpr\margindataparameter\c!threshold, % will change
+     \ifhmode
+       inline      = true,
+     \fi
+       \c!scope    = "\margindataparameter\c!scope",
+       \c!align    = "\margindataparameter\c!align",
+       \c!line     = "\margindataparameter\c!line",
+       \c!stack    = "\margindataparameter\c!stack",
+       \c!number   = \number\nextbox
+     }}%
+   \else
+     \ctxlua{typesetters.margins.save{
+       \c!location = "\margindataparameter\c!location",
+       \c!method   = "\margindataparameter\c!method",
+       \c!category = "\margindataparameter\c!category",
+       \c!name     = "\margindataparameter\c!name",
+       \c!scope    = "\margindataparameter\c!scope",
+     }}%
    \fi
-     \c!scope    = "\margindataparameter\c!scope",
-     \c!align    = "\margindataparameter\c!align",
-     \c!line     = "\margindataparameter\c!line",
-     \c!stack    = "\margindataparameter\c!stack",
-     \c!number   = \number\nextbox
-   }}%
    \endgroup}
 
 %D Downward compatible hack:
diff --git a/tex/context/base/x-ct.lua b/tex/context/base/x-ct.lua
index adaa0204f..1d0e63dae 100644
--- a/tex/context/base/x-ct.lua
+++ b/tex/context/base/x-ct.lua
@@ -11,7 +11,7 @@ if not modules then modules = { } end modules ['x-ct'] = {
 local xmlsprint, xmlcprint, xmlfilter, xmlcollected = xml.sprint, xml.cprint, xml.filter, xml.collected
 local format, concat, rep, find = string.format, table.concat, string.rep, string.find
 
-lxml.context = { }
+moduledata.ct = moduledata.ct or { }
 
 local halignments = {
     left       = 'l',
@@ -70,7 +70,7 @@ end
 
 local defaulttemplate = "|l|p|"
 
-function lxml.context.tabulate(root,namespace)
+function moduledata.ct.tabulate(root,namespace)
     if not root then
         return
     else
@@ -111,7 +111,7 @@ function lxml.context.tabulate(root,namespace)
 
 end
 
-function lxml.context.combination(root,namespace)
+function moduledata.ct.combination(root,namespace)
 
     if not root then
         return
@@ -143,7 +143,7 @@ function lxml.context.combination(root,namespace)
      -- )
         context("{")
         xmlfilter(e,contentspec)
-        context("}{")or
+        context("}{")
         xmlfilter(e,captionspec)
         context("}")
     end
diff --git a/tex/context/base/x-ct.mkiv b/tex/context/base/x-ct.mkiv
index ef6eb4ea4..6a6352f03 100644
--- a/tex/context/base/x-ct.mkiv
+++ b/tex/context/base/x-ct.mkiv
@@ -18,8 +18,8 @@
 \registerctxluafile{x-ct}{}
 
 \startxmlsetups xml:context:process
-    \xmlsetfunction {\xmldocument} {context:tabulate}    {lxml.context.tabulate}
-    \xmlsetfunction {\xmldocument} {context:combination} {lxml.context.combination}
+    \xmlsetfunction {\xmldocument} {context:tabulate}    {moduledata.ct.tabulate}
+    \xmlsetfunction {\xmldocument} {context:combination} {moduledata.ct.combination}
 \stopxmlsetups
 
 \xmlregistersetup{xml:context:process}
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 5d19f5649..939a141ce 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 07/20/11 14:10:55
+-- merge date  : 07/21/11 18:09:15
 
 do -- begin closure to overcome local limits and interference
 
@@ -1376,7 +1376,7 @@ local utflinesplitter = utfbom^-1 * tsplitat(newline)
 patterns.utflinesplitter = utflinesplitter
 
 function string.utfsplitlines(str)
-    return match(utflinesplitter,str)
+    return match(utflinesplitter,str or "")
 end
 
 --~ lpeg.splitters = cache -- no longer public