beta 2012.08.16 22:20

29 files changed, 1400 insertions, 485 deletions

diff --git a/scripts/context/lua/mtx-pdf.lua b/scripts/context/lua/mtx-pdf.lua
index 5654b8bc4..f37ee006a 100644
--- a/scripts/context/lua/mtx-pdf.lua
+++ b/scripts/context/lua/mtx-pdf.lua
@@ -6,14 +6,21 @@ if not modules then modules = { } end modules ['mtx-pdf'] = {
     license   = "see context related readme files"
 }
 
+local tonumber = tonumber
+local format, gmatch = string.format, string.gmatch
+local utfchar = utf.char
+local concat = table.concat
+local setmetatableindex, sortedhash, sortedkeys = table.setmetatableindex, table.sortedhash, table.sortedkeys
+
 local helpinfo = [[
 --info                show some info about the given file
 --metadata            show metadata xml blob
+--fonts               show used fonts (--detail)
 ]]
 
 local application = logs.application {
     name     = "mtx-pdf",
-    banner   = "ConTeXt PDF Helpers 0.01",
+    banner   = "ConTeXt PDF Helpers 0.10",
     helpinfo = helpinfo,
 }
 
@@ -39,9 +46,8 @@ local function loadpdffile(filename)
     end
 end
 
-function scripts.pdf.info()
-    local filename = environment.files[1]
-    local pdffile  = loadpdffile(filename)
+function scripts.pdf.info(filename)
+    local pdffile = loadpdffile(filename)
     if pdffile then
         local catalog  = pdffile.Catalog
         local info     = pdffile.Info
@@ -73,9 +79,8 @@ function scripts.pdf.info()
     end
 end
 
-function scripts.pdf.metadata()
-    local filename = environment.files[1]
-    local pdffile  = loadpdffile(filename)
+function scripts.pdf.metadata(filename)
+    local pdffile = loadpdffile(filename)
     if pdffile then
         local catalog  = pdffile.Catalog
         local metadata = catalog.Metadata
@@ -87,10 +92,127 @@ function scripts.pdf.metadata()
     end
 end
 
-if environment.argument("info") then
-    scripts.pdf.info()
+local function getfonts(pdffile)
+    local usedfonts = { }
+    for i=1,pdffile.pages.n do
+        local page = pdffile.pages[i]
+        local fontlist = page.Resources.Font
+        for k, v in next, lpdf.epdf.expand(fontlist) do
+            usedfonts[k] = lpdf.epdf.expand(v)
+        end
+    end
+    return usedfonts
+end
+
+local function getunicodes(font)
+    local cid = font.ToUnicode
+    if cid then
+        cid = cid()
+        local counts = { }
+     -- for s in gmatch(cid,"begincodespacerange%s*(.-)%s*endcodespacerange") do
+     --     for a, b in gmatch(s,"<([^>]+)>%s+<([^>]+)>") do
+     --         print(a,b)
+     --     end
+     -- end
+        setmetatableindex(counts, function(t,k) t[k] = 0 return 0 end)
+        for s in gmatch(cid,"beginbfrange%s*(.-)%s*endbfrange") do
+            for first, last, offset in gmatch(s,"<([^>]+)>%s+<([^>]+)>%s+<([^>]+)>") do
+                first  = tonumber(first,16)
+                last   = tonumber(last,16)
+                offset = tonumber(offset,16)
+                offset = offset - first
+                for i=first,last do
+                    local c = i + offset
+                    counts[c] = counts[c] + 1
+                end
+            end
+        end
+        for s in gmatch(cid,"beginbfchar%s*(.-)%s*endbfchar") do
+            for old, new in gmatch(s,"<([^>]+)>%s+<([^>]+)>") do
+                for n in gmatch(new,"....") do
+                    local c = tonumber(n,16)
+                    counts[c] = counts[c] + 1
+                end
+            end
+        end
+        return counts
+    end
+end
+
+function scripts.pdf.fonts(filename)
+    local pdffile = loadpdffile(filename)
+    if pdffile then
+        local usedfonts = getfonts(pdffile)
+        local found     = { }
+        for k, v in table.sortedhash(usedfonts) do
+            local counts = getunicodes(v)
+            local codes = { }
+            local chars = { }
+            local freqs = { }
+            if counts then
+                codes = sortedkeys(counts)
+                for i=1,#codes do
+                    local k = codes[i]
+                    local c = utfchar(k)
+                    chars[i] = c
+                    freqs[i] = format("U+%05X  %s  %s",k,counts[k] > 1 and "+" or " ", c)
+                end
+                for i=1,#codes do
+                    codes[i] = format("U+%05X",codes[i])
+                end
+            end
+            found[k] = {
+                basefont = v.BaseFont or "no basefont",
+                encoding = v.Encoding or "no encoding",
+                subtype  = v.Subtype or "no subtype",
+                unicode  = v.ToUnicode and "unicode" or "no unicode",
+                chars    = chars,
+                codes    = codes,
+                freqs    = freqs,
+            }
+        end
+
+        if environment.argument("detail") then
+            for k, v in sortedhash(found) do
+                report("id         : %s",k)
+                report("basefont   : %s",v.basefont)
+                report("encoding   : %s",v.encoding)
+                report("subtype    : %s",v.subtype)
+                report("unicode    : %s",v.unicode)
+                report("characters : %s", concat(v.chars," "))
+                report("codepoints : %s", concat(v.codes," "))
+                report("")
+            end
+        else
+            local results = { { "id", "basefont", "encoding", "subtype", "unicode", "characters" } }
+            for k, v in sortedhash(found) do
+                results[#results+1] = { k, v.basefont, v.encoding, v.subtype, v.unicode, concat(v.chars," ") }
+            end
+            utilities.formatters.formatcolumns(results)
+            report(results[1])
+            report("")
+            for i=2,#results do
+                report(results[i])
+            end
+            report("")
+        end
+    end
+end
+
+-- scripts.pdf.info("e:/tmp/oeps.pdf")
+-- scripts.pdf.metadata("e:/tmp/oeps.pdf")
+-- scripts.pdf.fonts("e:/tmp/oeps.pdf")
+
+local filename = environment.files[1] or ""
+
+if filename == "" then
+    application.help()
+elseif environment.argument("info") then
+    scripts.pdf.info(filename)
 elseif environment.argument("metadata") then
-    scripts.pdf.metadata()
+    scripts.pdf.metadata(filename)
+elseif environment.argument("fonts") then
+    scripts.pdf.fonts(filename)
 else
     application.help()
 end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 2090ec584..1eb0f5816 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -247,12 +247,16 @@ function table.strip(tab)
 end
 
 function table.keys(t)
-    local keys, k = { }, 0
-    for key, _ in next, t do
-        k = k + 1
-        keys[k] = key
+    if t then
+        local keys, k = { }, 0
+        for key, _ in next, t do
+            k = k + 1
+            keys[k] = key
+        end
+        return keys
+    else
+        return { }
     end
-    return keys
 end
 
 local function compare(a,b)
@@ -265,41 +269,49 @@ local function compare(a,b)
 end
 
 local function sortedkeys(tab)
-    local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
-    for key,_ in next, tab do
-        s = s + 1
-        srt[s] = key
-        if category == 3 then
-            -- no further check
-        else
-            local tkey = type(key)
-            if tkey == "string" then
-                category = (category == 2 and 3) or 1
-            elseif tkey == "number" then
-                category = (category == 1 and 3) or 2
+    if tab then
+        local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
+        for key,_ in next, tab do
+            s = s + 1
+            srt[s] = key
+            if category == 3 then
+                -- no further check
             else
-                category = 3
+                local tkey = type(key)
+                if tkey == "string" then
+                    category = (category == 2 and 3) or 1
+                elseif tkey == "number" then
+                    category = (category == 1 and 3) or 2
+                else
+                    category = 3
+                end
             end
         end
-    end
-    if category == 0 or category == 3 then
-        sort(srt,compare)
+        if category == 0 or category == 3 then
+            sort(srt,compare)
+        else
+            sort(srt)
+        end
+        return srt
     else
-        sort(srt)
+        return { }
     end
-    return srt
 end
 
 local function sortedhashkeys(tab) -- fast one
-    local srt, s = { }, 0
-    for key,_ in next, tab do
-        if key then
-            s= s + 1
-            srt[s] = key
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if key then
+                s= s + 1
+                srt[s] = key
+            end
         end
+        sort(srt)
+        return srt
+    else
+        return { }
     end
-    sort(srt)
-    return srt
 end
 
 table.sortedkeys     = sortedkeys
@@ -324,7 +336,7 @@ end
 table.sortedhash  = sortedhash
 table.sortedpairs = sortedhash
 
-function table.append(t, list)
+function table.append(t,list)
     local n = #t
     for i=1,#list do
         n = n + 1
@@ -1186,7 +1198,7 @@ local report = texio and texio.write_nl or print
 -- function lpeg.Cmt  (l) local p = lpcmt (l) report("LPEG Cmt =")  lpprint(l) return p end
 -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end
 
-local type = type
+local type, next = type, next
 local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format
 
 -- Beware, we predefine a bunch of patterns here and one reason for doing so
@@ -1247,6 +1259,10 @@ patterns.utf8char      = utf8char
 patterns.validutf8     = validutf8char
 patterns.validutf8char = validutf8char
 
+local eol              = S("\n\r")
+local spacer           = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local whitespace       = eol + spacer
+
 patterns.digit         = digit
 patterns.sign          = sign
 patterns.cardinal      = sign^0 * digit^1
@@ -1266,16 +1282,16 @@ patterns.letter        = patterns.lowercase + patterns.uppercase
 patterns.space         = space
 patterns.tab           = P("\t")
 patterns.spaceortab    = patterns.space + patterns.tab
-patterns.eol           = S("\n\r")
-patterns.spacer        = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+patterns.eol           = eol
+patterns.spacer        = spacer
+patterns.whitespace    = whitespace
 patterns.newline       = newline
 patterns.emptyline     = newline^1
-patterns.nonspacer     = 1 - patterns.spacer
-patterns.whitespace    = patterns.eol + patterns.spacer
-patterns.nonwhitespace = 1 - patterns.whitespace
+patterns.nonspacer     = 1 - spacer
+patterns.nonwhitespace = 1 - whitespace
 patterns.equal         = P("=")
 patterns.comma         = P(",")
-patterns.commaspacer   = P(",") * patterns.spacer^0
+patterns.commaspacer   = P(",") * spacer^0
 patterns.period        = P(".")
 patterns.colon         = P(":")
 patterns.semicolon     = P(";")
@@ -1491,8 +1507,8 @@ end
 function lpeg.replacer(one,two)
     if type(one) == "table" then
         local no = #one
+        local p
         if no > 0 then
-            local p
             for i=1,no do
                 local o = one[i]
                 local pp = P(o[1]) / o[2]
@@ -1502,8 +1518,17 @@ function lpeg.replacer(one,two)
                     p = pp
                 end
             end
-            return Cs((p + 1)^0)
+        else
+            for k, v in next, one do
+                local pp = P(k) / v
+                if p then
+                    p = p + pp
+                else
+                    p = pp
+                end
+            end
         end
+        return Cs((p + 1)^0)
     else
         two = two or ""
         return Cs((P(one)/two + 1)^0)
@@ -1875,6 +1900,14 @@ function string.tformat(fmt,...)
     return format(lpegmatch(replacer,fmt),...)
 end
 
+-- strips leading and trailing spaces and collapsed all other spaces
+
+local pattern = Cs(whitespace^0/"" * ((whitespace^1 * P(-1) / "") + (whitespace^1/" ") + P(1))^0)
+
+function string.collapsespaces(str)
+    return lpegmatch(pattern,str)
+end
+
 
 end -- of closure
 
@@ -5699,7 +5732,7 @@ statistics.elapsedtime    = elapsedtime
 statistics.elapsedindeed  = elapsedindeed
 statistics.elapsedseconds = elapsedseconds
 
--- general function
+-- general function .. we might split this module
 
 function statistics.register(tag,fnc)
     if statistics.enable and type(fnc) == "function" then
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 2090ec584..1eb0f5816 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -247,12 +247,16 @@ function table.strip(tab)
 end
 
 function table.keys(t)
-    local keys, k = { }, 0
-    for key, _ in next, t do
-        k = k + 1
-        keys[k] = key
+    if t then
+        local keys, k = { }, 0
+        for key, _ in next, t do
+            k = k + 1
+            keys[k] = key
+        end
+        return keys
+    else
+        return { }
     end
-    return keys
 end
 
 local function compare(a,b)
@@ -265,41 +269,49 @@ local function compare(a,b)
 end
 
 local function sortedkeys(tab)
-    local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
-    for key,_ in next, tab do
-        s = s + 1
-        srt[s] = key
-        if category == 3 then
-            -- no further check
-        else
-            local tkey = type(key)
-            if tkey == "string" then
-                category = (category == 2 and 3) or 1
-            elseif tkey == "number" then
-                category = (category == 1 and 3) or 2
+    if tab then
+        local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
+        for key,_ in next, tab do
+            s = s + 1
+            srt[s] = key
+            if category == 3 then
+                -- no further check
             else
-                category = 3
+                local tkey = type(key)
+                if tkey == "string" then
+                    category = (category == 2 and 3) or 1
+                elseif tkey == "number" then
+                    category = (category == 1 and 3) or 2
+                else
+                    category = 3
+                end
             end
         end
-    end
-    if category == 0 or category == 3 then
-        sort(srt,compare)
+        if category == 0 or category == 3 then
+            sort(srt,compare)
+        else
+            sort(srt)
+        end
+        return srt
     else
-        sort(srt)
+        return { }
     end
-    return srt
 end
 
 local function sortedhashkeys(tab) -- fast one
-    local srt, s = { }, 0
-    for key,_ in next, tab do
-        if key then
-            s= s + 1
-            srt[s] = key
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if key then
+                s= s + 1
+                srt[s] = key
+            end
         end
+        sort(srt)
+        return srt
+    else
+        return { }
     end
-    sort(srt)
-    return srt
 end
 
 table.sortedkeys     = sortedkeys
@@ -324,7 +336,7 @@ end
 table.sortedhash  = sortedhash
 table.sortedpairs = sortedhash
 
-function table.append(t, list)
+function table.append(t,list)
     local n = #t
     for i=1,#list do
         n = n + 1
@@ -1186,7 +1198,7 @@ local report = texio and texio.write_nl or print
 -- function lpeg.Cmt  (l) local p = lpcmt (l) report("LPEG Cmt =")  lpprint(l) return p end
 -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end
 
-local type = type
+local type, next = type, next
 local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format
 
 -- Beware, we predefine a bunch of patterns here and one reason for doing so
@@ -1247,6 +1259,10 @@ patterns.utf8char      = utf8char
 patterns.validutf8     = validutf8char
 patterns.validutf8char = validutf8char
 
+local eol              = S("\n\r")
+local spacer           = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local whitespace       = eol + spacer
+
 patterns.digit         = digit
 patterns.sign          = sign
 patterns.cardinal      = sign^0 * digit^1
@@ -1266,16 +1282,16 @@ patterns.letter        = patterns.lowercase + patterns.uppercase
 patterns.space         = space
 patterns.tab           = P("\t")
 patterns.spaceortab    = patterns.space + patterns.tab
-patterns.eol           = S("\n\r")
-patterns.spacer        = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+patterns.eol           = eol
+patterns.spacer        = spacer
+patterns.whitespace    = whitespace
 patterns.newline       = newline
 patterns.emptyline     = newline^1
-patterns.nonspacer     = 1 - patterns.spacer
-patterns.whitespace    = patterns.eol + patterns.spacer
-patterns.nonwhitespace = 1 - patterns.whitespace
+patterns.nonspacer     = 1 - spacer
+patterns.nonwhitespace = 1 - whitespace
 patterns.equal         = P("=")
 patterns.comma         = P(",")
-patterns.commaspacer   = P(",") * patterns.spacer^0
+patterns.commaspacer   = P(",") * spacer^0
 patterns.period        = P(".")
 patterns.colon         = P(":")
 patterns.semicolon     = P(";")
@@ -1491,8 +1507,8 @@ end
 function lpeg.replacer(one,two)
     if type(one) == "table" then
         local no = #one
+        local p
         if no > 0 then
-            local p
             for i=1,no do
                 local o = one[i]
                 local pp = P(o[1]) / o[2]
@@ -1502,8 +1518,17 @@ function lpeg.replacer(one,two)
                     p = pp
                 end
             end
-            return Cs((p + 1)^0)
+        else
+            for k, v in next, one do
+                local pp = P(k) / v
+                if p then
+                    p = p + pp
+                else
+                    p = pp
+                end
+            end
         end
+        return Cs((p + 1)^0)
     else
         two = two or ""
         return Cs((P(one)/two + 1)^0)
@@ -1875,6 +1900,14 @@ function string.tformat(fmt,...)
     return format(lpegmatch(replacer,fmt),...)
 end
 
+-- strips leading and trailing spaces and collapsed all other spaces
+
+local pattern = Cs(whitespace^0/"" * ((whitespace^1 * P(-1) / "") + (whitespace^1/" ") + P(1))^0)
+
+function string.collapsespaces(str)
+    return lpegmatch(pattern,str)
+end
+
 
 end -- of closure
 
@@ -5699,7 +5732,7 @@ statistics.elapsedtime    = elapsedtime
 statistics.elapsedindeed  = elapsedindeed
 statistics.elapsedseconds = elapsedseconds
 
--- general function
+-- general function .. we might split this module
 
 function statistics.register(tag,fnc)
     if statistics.enable and type(fnc) == "function" then
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 2090ec584..1eb0f5816 100755
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -247,12 +247,16 @@ function table.strip(tab)
 end
 
 function table.keys(t)
-    local keys, k = { }, 0
-    for key, _ in next, t do
-        k = k + 1
-        keys[k] = key
+    if t then
+        local keys, k = { }, 0
+        for key, _ in next, t do
+            k = k + 1
+            keys[k] = key
+        end
+        return keys
+    else
+        return { }
     end
-    return keys
 end
 
 local function compare(a,b)
@@ -265,41 +269,49 @@ local function compare(a,b)
 end
 
 local function sortedkeys(tab)
-    local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
-    for key,_ in next, tab do
-        s = s + 1
-        srt[s] = key
-        if category == 3 then
-            -- no further check
-        else
-            local tkey = type(key)
-            if tkey == "string" then
-                category = (category == 2 and 3) or 1
-            elseif tkey == "number" then
-                category = (category == 1 and 3) or 2
+    if tab then
+        local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
+        for key,_ in next, tab do
+            s = s + 1
+            srt[s] = key
+            if category == 3 then
+                -- no further check
             else
-                category = 3
+                local tkey = type(key)
+                if tkey == "string" then
+                    category = (category == 2 and 3) or 1
+                elseif tkey == "number" then
+                    category = (category == 1 and 3) or 2
+                else
+                    category = 3
+                end
             end
         end
-    end
-    if category == 0 or category == 3 then
-        sort(srt,compare)
+        if category == 0 or category == 3 then
+            sort(srt,compare)
+        else
+            sort(srt)
+        end
+        return srt
     else
-        sort(srt)
+        return { }
     end
-    return srt
 end
 
 local function sortedhashkeys(tab) -- fast one
-    local srt, s = { }, 0
-    for key,_ in next, tab do
-        if key then
-            s= s + 1
-            srt[s] = key
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if key then
+                s= s + 1
+                srt[s] = key
+            end
         end
+        sort(srt)
+        return srt
+    else
+        return { }
     end
-    sort(srt)
-    return srt
 end
 
 table.sortedkeys     = sortedkeys
@@ -324,7 +336,7 @@ end
 table.sortedhash  = sortedhash
 table.sortedpairs = sortedhash
 
-function table.append(t, list)
+function table.append(t,list)
     local n = #t
     for i=1,#list do
         n = n + 1
@@ -1186,7 +1198,7 @@ local report = texio and texio.write_nl or print
 -- function lpeg.Cmt  (l) local p = lpcmt (l) report("LPEG Cmt =")  lpprint(l) return p end
 -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end
 
-local type = type
+local type, next = type, next
 local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format
 
 -- Beware, we predefine a bunch of patterns here and one reason for doing so
@@ -1247,6 +1259,10 @@ patterns.utf8char      = utf8char
 patterns.validutf8     = validutf8char
 patterns.validutf8char = validutf8char
 
+local eol              = S("\n\r")
+local spacer           = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local whitespace       = eol + spacer
+
 patterns.digit         = digit
 patterns.sign          = sign
 patterns.cardinal      = sign^0 * digit^1
@@ -1266,16 +1282,16 @@ patterns.letter        = patterns.lowercase + patterns.uppercase
 patterns.space         = space
 patterns.tab           = P("\t")
 patterns.spaceortab    = patterns.space + patterns.tab
-patterns.eol           = S("\n\r")
-patterns.spacer        = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+patterns.eol           = eol
+patterns.spacer        = spacer
+patterns.whitespace    = whitespace
 patterns.newline       = newline
 patterns.emptyline     = newline^1
-patterns.nonspacer     = 1 - patterns.spacer
-patterns.whitespace    = patterns.eol + patterns.spacer
-patterns.nonwhitespace = 1 - patterns.whitespace
+patterns.nonspacer     = 1 - spacer
+patterns.nonwhitespace = 1 - whitespace
 patterns.equal         = P("=")
 patterns.comma         = P(",")
-patterns.commaspacer   = P(",") * patterns.spacer^0
+patterns.commaspacer   = P(",") * spacer^0
 patterns.period        = P(".")
 patterns.colon         = P(":")
 patterns.semicolon     = P(";")
@@ -1491,8 +1507,8 @@ end
 function lpeg.replacer(one,two)
     if type(one) == "table" then
         local no = #one
+        local p
         if no > 0 then
-            local p
             for i=1,no do
                 local o = one[i]
                 local pp = P(o[1]) / o[2]
@@ -1502,8 +1518,17 @@ function lpeg.replacer(one,two)
                     p = pp
                 end
             end
-            return Cs((p + 1)^0)
+        else
+            for k, v in next, one do
+                local pp = P(k) / v
+                if p then
+                    p = p + pp
+                else
+                    p = pp
+                end
+            end
         end
+        return Cs((p + 1)^0)
     else
         two = two or ""
         return Cs((P(one)/two + 1)^0)
@@ -1875,6 +1900,14 @@ function string.tformat(fmt,...)
     return format(lpegmatch(replacer,fmt),...)
 end
 
+-- strips leading and trailing spaces and collapsed all other spaces
+
+local pattern = Cs(whitespace^0/"" * ((whitespace^1 * P(-1) / "") + (whitespace^1/" ") + P(1))^0)
+
+function string.collapsespaces(str)
+    return lpegmatch(pattern,str)
+end
+
 
 end -- of closure
 
@@ -5699,7 +5732,7 @@ statistics.elapsedtime    = elapsedtime
 statistics.elapsedindeed  = elapsedindeed
 statistics.elapsedseconds = elapsedseconds
 
--- general function
+-- general function .. we might split this module
 
 function statistics.register(tag,fnc)
     if statistics.enable and type(fnc) == "function" then
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index b68b34047..3c34ace37 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.08.14 10:44}
+\newcontextversion{2012.08.16 22:20}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index a5b59432f..d2c59c9ad 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.08.14 10:44}
+\newcontextversion{2012.08.16 22:20}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
@@ -20,6 +20,37 @@
 
 \writestatus\m!system{beware: some patches loaded from cont-new.mkiv}
 
+\let\strc_formulas_handle_numbering_indeed    \strc_formulas_handle_numbering
+\let\strc_formulas_handle_sub_numbering_indeed\strc_formulas_handle_sub_numbering
+\let\strc_formulas_handle_sub_number_indeed   \strc_formulas_handle_sub_number
+
+\def\strc_formulas_handle_numbering
+  {\iftrialtypesetting
+     \strc_counters_save\v!formula
+     \strc_formulas_handle_numbering_indeed
+     \strc_counters_restore\v!formula
+   \else
+     \strc_formulas_handle_numbering_indeed
+   \fi}
+
+\def\strc_formulas_handle_sub_numbering
+  {\iftrialtypesetting
+     \strc_counters_save\v!formula
+     \strc_formulas_handle_sub_numbering
+     \strc_counters_restore\v!formula
+   \else
+     \strc_formulas_handle_sub_numbering
+   \fi}
+
+\def\strc_formulas_handle_sub_number % sub formulas
+  {\iftrialtypesetting
+     \strc_counters_save\v!formula
+     \strc_formulas_handle_sub_number_indeed
+     \strc_counters_restore\v!formula
+   \else
+     \strc_formulas_handle_sub_number_indeed
+   \fi}
+
 %D Maybe:
 
 \unexpanded\def\tightvbox{\dowithnextbox{\dp\nextbox\zeropoint\box\nextbox}\vbox}
diff --git a/tex/context/base/context-base.lmx b/tex/context/base/context-base.lmx
index 09817463b..2b093c3e1 100644
--- a/tex/context/base/context-base.lmx
+++ b/tex/context/base/context-base.lmx
@@ -19,7 +19,9 @@
         <?lua end ?>
         <style type="text/css">
             <?lmx-include context.css ?>
-        </style>
+            <?lua if v('cssdata') then ?>
+<?lua pv('cssdata') ?>
+            <?lua end ?>        </style>
         <?lua if v('styles') then for k, v in ipairs(v('styles')) do ?>
             <link rel="StyleSheet" href="<?lua p(v) ?>" type="text/css" />
         <?lua end end ?>
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 689c9383b..e687c41b1 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index 6c4bbc6cf..ca343017a 100644
--- a/tex/context/base/context-version.png
+++ b/tex/context/base/context-version.png
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 82281400e..5e210159a 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.08.14 10:44}
+\edef\contextversion{2012.08.16 22:20}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 22d28841f..f33938f01 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.08.14 10:44}
+\edef\contextversion{2012.08.16 22:20}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/core-env.lua b/tex/context/base/core-env.lua
index 96da8cc9f..8b9ae172b 100644
--- a/tex/context/base/core-env.lua
+++ b/tex/context/base/core-env.lua
@@ -52,7 +52,7 @@ setmetatableindex(tex.systemmodes, function(t,k)
     if m then
         return m()
     else
-        local n = "mode*" .. k
+        local n = "mode>*" .. k
         if csname_id(n) == undefined then
             return false
         else
diff --git a/tex/context/base/font-ctx.lua b/tex/context/base/font-ctx.lua
index 1badc090d..4961c4ae8 100644
--- a/tex/context/base/font-ctx.lua
+++ b/tex/context/base/font-ctx.lua
@@ -1601,3 +1601,18 @@ commands.definefontfeature = fonts.specifiers.presetcontext
 function commands.featurelist(...)
     context(fonts.specifiers.contexttostring(...))
 end
+
+-- a fontkern plug:
+
+local copy_node = node.copy
+local kern      = nodes.pool.register(nodes.pool.kern())
+
+node.set_attribute(kern,attributes.private('fontkern'),1) -- we can have several, attributes are shared
+
+nodes.injections.installnewkern(function(k)
+    local c = copy_node(kern)
+    c.kern = k
+    return c
+end)
+
+directives.register("nodes.injections.fontkern", function(v) kern.subtype = v and 0 or 1 end)
diff --git a/tex/context/base/font-otn.lua b/tex/context/base/font-otn.lua
index 9e233a7a7..ef842b099 100644
--- a/tex/context/base/font-otn.lua
+++ b/tex/context/base/font-otn.lua
@@ -185,6 +185,7 @@ local glyph_code         = nodecodes.glyph
 local glue_code          = nodecodes.glue
 local disc_code          = nodecodes.disc
 local whatsit_code       = nodecodes.whatsit
+local user_code          = nodecodes.user
 
 local dir_code           = whatcodes.dir
 local localpar_code      = whatcodes.localpar
@@ -1722,7 +1723,7 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                                 break
                             end
                             prev = prev.prev
-                        elseif seq[n][32] then -- somehat special, as zapfino can have many preceding spaces
+                        elseif seq[n][32] then -- somewhat special, as zapfino can have many preceding spaces
                             n = n -1
                         else
                             match = false
diff --git a/tex/context/base/font-sol.lua b/tex/context/base/font-sol.lua
index d77a990b5..1c341129c 100644
--- a/tex/context/base/font-sol.lua
+++ b/tex/context/base/font-sol.lua
@@ -63,17 +63,22 @@ local hpack_nodes        = node.hpack
 local insert_node_before = node.insert_before
 local insert_node_after  = node.insert_after
 local repack_hlist       = nodes.repackhlist
+local nodes_to_utf       = nodes.listtoutf
 
 local setnodecolor       = nodes.tracers.colors.set
 
 local nodecodes          = nodes.nodecodes
 local whatsitcodes       = nodes.whatsitcodes
+local kerncodes          = nodes.kerncodes
 
 local glyph_code         = nodecodes.glyph
 local disc_code          = nodecodes.disc
+local kern_code          = nodecodes.kern
 local hlist_code         = nodecodes.hlist
 local whatsit_code       = nodecodes.whatsit
 
+local fontkern_code      = kerncodes.fontkern
+
 local localpar_code      = whatsitcodes.localpar
 local dir_code           = whatsitcodes.dir
 local userdefined_code   = whatsitcodes.userdefined
@@ -303,6 +308,15 @@ local nofwords, noftries, nofadapted, nofkept, nofparagraphs = 0, 0, 0, 0, 0
 local splitter_one = usernodeids["splitters.one"]
 local splitter_two = usernodeids["splitters.two"]
 
+local a_word       = attributes.private('word')
+local a_fontkern   = attributes.private('fontkern')
+
+local encapsulate  = false
+
+directives.register("builders.paragraphs.solutions.splitters.encapsulate", function(v)
+    encapsulate = v
+end)
+
 function splitters.split(head)
     -- quite fast
     local current, done, rlmode, start, stop, attribute = head, false, false, nil, nil, 0
@@ -312,10 +326,22 @@ function splitters.split(head)
         local last = stop.next
         local list = last and copy_nodelist(start,last) or copy_nodelist(start)
         local n = #cache + 1
-        local user_one = new_usernumber(splitter_one,n)
-        local user_two = new_usernumber(splitter_two,n)
-        head, start = insert_node_before(head,start,user_one)
-        insert_node_after(head,stop,user_two)
+        if encapsulate then
+            local user_one = new_usernumber(splitter_one,n)
+            local user_two = new_usernumber(splitter_two,n)
+            head, start = insert_node_before(head,start,user_one)
+            insert_node_after(head,stop,user_two)
+        else
+            local current = start
+            while true do
+                set_attribute(current,a_word,n)
+                if current == stop then
+                    break
+                else
+                    current = current.next
+                end
+            end
+        end
         if rlmode == "TRT" or rlmode == "+TRT" then
             local dirnode = new_textdir("+TRT")
             list.prev = dirnode
@@ -330,7 +356,7 @@ function splitters.split(head)
         }
         if trace_split then
             report_splitters("cached %4i: font: %s, attribute: %s, word: %s, direction: %s",
-                n, font, attribute, nodes.listtoutf(list,true), rlmode and "r2l" or "l2r")
+                n, font, attribute, nodes_to_utf(list,true), rlmode and "r2l" or "l2r")
         end
         cache[n] = c
         local solution = solutions[attribute]
@@ -389,20 +415,93 @@ function splitters.split(head)
     return head, done
 end
 
-local function collect_words(list)
+local function collect_words(list) -- can be made faster for attributes
     local words, w, word = { }, 0, nil
-    for current in traverse_ids(whatsit_code,list) do
-        if current.subtype == userdefined_code then -- hm
-            local user_id = current.user_id
-            if user_id == splitter_one then
-                word = { current.value, current, current }
+    if encapsulate then
+        for current in traverse_ids(whatsit_code,list) do
+            if current.subtype == userdefined_code then -- hm
+                local user_id = current.user_id
+                if user_id == splitter_one then
+                    word = { current.value, current, current }
+                    w = w + 1
+                    words[w] = word
+                elseif user_id == splitter_two then
+                    if word then
+                        word[3] = current
+                    else
+                        -- something is wrong
+                    end
+                end
+            end
+        end
+    else
+        local current, first, last, index = list, nil, nil, nil
+        while current do
+            -- todo: disc and kern
+            local id = current.id
+            if id == glyph_code then
+                local a = has_attribute(current,a_word)
+                if a then
+                    if a == index then
+                        -- same word
+                        last = current
+                    elseif index then
+                        w = w + 1
+                        words[w] = { index, first, last }
+                        first = current
+                        last  = current
+                        index = a
+                    elseif first then
+                        last  = current
+                        index = a
+                    else
+                        first = current
+                        last  = current
+                        index = a
+                    end
+                elseif index then
+                    if first then
+                        w = w + 1
+                        words[w] = { index, first, last }
+                    end
+                    index = nil
+                    first = nil
+                elseif trace_split then
+                    report_splitters("skipped: %s",utfchar(current.char))
+                end
+            elseif id == kern_code and (current.subtype == fontkern_code or has_attribute(current,a_fontkern)) then
+                if first then
+                    last = current
+                else
+                    first = current
+                    last = current
+                end
+            elseif index then
                 w = w + 1
-                words[w] = word
-            elseif user_id == splitter_two then
-                if word then
-                    word[3] = current
+                words[w] = { index, first, last }
+                index = nil
+                first = nil
+                if id == disc_node then
+                    if trace_split then
+                        report_splitters("skipped disc node")
+                    end
+                end
+            end
+            current = current.next
+        end
+        if index then
+            w = w + 1
+            words[w] = { index, first, last }
+        end
+        if trace_split then
+            for i=1,#words do
+                local w = words[i]
+                local n, f, l = w[1], w[2], w[3]
+                local c = cache[n]
+                if c then
+                    report_splitters("found %4i: word: %s, cached: %s",n,nodes_to_utf(f,true,true,l),nodes_to_utf(c.original,true))
                 else
-                    -- something is wrong
+                    report_splitters("found %4i: word: %s, not in cache",n,nodes_to_utf(f,true,true,l))
                 end
             end
         end
@@ -417,9 +516,14 @@ local function doit(word,list,best,width,badness,line,set,listdir)
     local n = word[1]
     local found = cache[n]
     if found then
-
-        local h = word[2].next -- head of current word
-        local t = word[3].prev -- tail of current word
+        local h, t
+        if encapsulate then
+            h = word[2].next -- head of current word
+            t = word[3].prev -- tail of current word
+        else
+            h = word[2]
+            t = word[3]
+        end
 
         if splitwords then
             -- there are no lines crossed in a word
@@ -485,8 +589,6 @@ local function doit(word,list,best,width,badness,line,set,listdir)
                     report_solutions("fatal error, no dynamics for font %s",font)
                 end
                 first = inject_kerns(first)
---                 local h = word[2].next -- head of current word
---                 local t = word[3].prev -- tail of current word
                 if first.id == whatsit_code then
                     local temp = first
                     first = first.next
diff --git a/tex/context/base/grph-inc.lua b/tex/context/base/grph-inc.lua
index 8672ca2be..cb1fdbd8b 100644
--- a/tex/context/base/grph-inc.lua
+++ b/tex/context/base/grph-inc.lua
@@ -1171,6 +1171,8 @@ local function makeoptions(options)
     return (to == "table" and concat(options," ")) or (to == "string" and options) or ""
 end
 
+-- programs.makeoptions = makeoptions
+
 local function runprogram(template,binary,...)
     local command = format(template,binary,...)
     local binary = match(binary,"[%S]+") -- to be sure
@@ -1453,7 +1455,7 @@ function figures.applyratio(width,height,w,h) -- width and height are strings an
     end
 end
 
--- example of a simple plugin:
+-- example of simple plugins:
 --
 -- figures.converters.png = {
 --     png = function(oldname,newname,resolution)
@@ -1463,6 +1465,12 @@ end
 --     end,
 -- }
 
+-- figures.converters.bmp = {
+--     pdf = function(oldname,newname)
+--         os.execute(string.format("gm convert %s %s",oldname,newname))
+--     end
+-- }
+
 
 --     local fig = figures.push { name = pdffile }
 --     figures.identify()
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index 70c5f14e1..652772d79 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -66,7 +66,7 @@ local report = texio and texio.write_nl or print
 -- function lpeg.Cmt  (l) local p = lpcmt (l) report("LPEG Cmt =")  lpprint(l) return p end
 -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end
 
-local type = type
+local type, next = type, next
 local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format
 
 -- Beware, we predefine a bunch of patterns here and one reason for doing so
@@ -127,6 +127,10 @@ patterns.utf8char      = utf8char
 patterns.validutf8     = validutf8char
 patterns.validutf8char = validutf8char
 
+local eol              = S("\n\r")
+local spacer           = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local whitespace       = eol + spacer
+
 patterns.digit         = digit
 patterns.sign          = sign
 patterns.cardinal      = sign^0 * digit^1
@@ -146,16 +150,16 @@ patterns.letter        = patterns.lowercase + patterns.uppercase
 patterns.space         = space
 patterns.tab           = P("\t")
 patterns.spaceortab    = patterns.space + patterns.tab
-patterns.eol           = S("\n\r")
-patterns.spacer        = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+patterns.eol           = eol
+patterns.spacer        = spacer
+patterns.whitespace    = whitespace
 patterns.newline       = newline
 patterns.emptyline     = newline^1
-patterns.nonspacer     = 1 - patterns.spacer
-patterns.whitespace    = patterns.eol + patterns.spacer
-patterns.nonwhitespace = 1 - patterns.whitespace
+patterns.nonspacer     = 1 - spacer
+patterns.nonwhitespace = 1 - whitespace
 patterns.equal         = P("=")
 patterns.comma         = P(",")
-patterns.commaspacer   = P(",") * patterns.spacer^0
+patterns.commaspacer   = P(",") * spacer^0
 patterns.period        = P(".")
 patterns.colon         = P(":")
 patterns.semicolon     = P(";")
@@ -387,8 +391,8 @@ end
 function lpeg.replacer(one,two)
     if type(one) == "table" then
         local no = #one
+        local p
         if no > 0 then
-            local p
             for i=1,no do
                 local o = one[i]
                 local pp = P(o[1]) / o[2]
@@ -398,8 +402,17 @@ function lpeg.replacer(one,two)
                     p = pp
                 end
             end
-            return Cs((p + 1)^0)
+        else
+            for k, v in next, one do
+                local pp = P(k) / v
+                if p then
+                    p = p + pp
+                else
+                    p = pp
+                end
+            end
         end
+        return Cs((p + 1)^0)
     else
         two = two or ""
         return Cs((P(one)/two + 1)^0)
@@ -793,3 +806,11 @@ local replacer = lpeg.replacer("@","%%") -- Watch the escaped % in lpeg!
 function string.tformat(fmt,...)
     return format(lpegmatch(replacer,fmt),...)
 end
+
+-- strips leading and trailing spaces and collapsed all other spaces
+
+local pattern = Cs(whitespace^0/"" * ((whitespace^1 * P(-1) / "") + (whitespace^1/" ") + P(1))^0)
+
+function string.collapsespaces(str)
+    return lpegmatch(pattern,str)
+end
diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua
index 337ce054a..cc69af0ef 100644
--- a/tex/context/base/l-table.lua
+++ b/tex/context/base/l-table.lua
@@ -76,12 +76,16 @@ function table.strip(tab)
 end
 
 function table.keys(t)
-    local keys, k = { }, 0
-    for key, _ in next, t do
-        k = k + 1
-        keys[k] = key
+    if t then
+        local keys, k = { }, 0
+        for key, _ in next, t do
+            k = k + 1
+            keys[k] = key
+        end
+        return keys
+    else
+        return { }
     end
-    return keys
 end
 
 local function compare(a,b)
@@ -94,41 +98,49 @@ local function compare(a,b)
 end
 
 local function sortedkeys(tab)
-    local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
-    for key,_ in next, tab do
-        s = s + 1
-        srt[s] = key
-        if category == 3 then
-            -- no further check
-        else
-            local tkey = type(key)
-            if tkey == "string" then
-                category = (category == 2 and 3) or 1
-            elseif tkey == "number" then
-                category = (category == 1 and 3) or 2
+    if tab then
+        local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
+        for key,_ in next, tab do
+            s = s + 1
+            srt[s] = key
+            if category == 3 then
+                -- no further check
             else
-                category = 3
+                local tkey = type(key)
+                if tkey == "string" then
+                    category = (category == 2 and 3) or 1
+                elseif tkey == "number" then
+                    category = (category == 1 and 3) or 2
+                else
+                    category = 3
+                end
             end
         end
-    end
-    if category == 0 or category == 3 then
-        sort(srt,compare)
+        if category == 0 or category == 3 then
+            sort(srt,compare)
+        else
+            sort(srt)
+        end
+        return srt
     else
-        sort(srt)
+        return { }
     end
-    return srt
 end
 
 local function sortedhashkeys(tab) -- fast one
-    local srt, s = { }, 0
-    for key,_ in next, tab do
-        if key then
-            s= s + 1
-            srt[s] = key
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if key then
+                s= s + 1
+                srt[s] = key
+            end
         end
+        sort(srt)
+        return srt
+    else
+        return { }
     end
-    sort(srt)
-    return srt
 end
 
 table.sortedkeys     = sortedkeys
@@ -153,7 +165,7 @@ end
 table.sortedhash  = sortedhash
 table.sortedpairs = sortedhash
 
-function table.append(t, list)
+function table.append(t,list)
     local n = #t
     for i=1,#list do
         n = n + 1
diff --git a/tex/context/base/lpdf-epd.lua b/tex/context/base/lpdf-epd.lua
index 76d258cef..1c4b4b5c5 100644
--- a/tex/context/base/lpdf-epd.lua
+++ b/tex/context/base/lpdf-epd.lua
@@ -179,14 +179,17 @@ checked_access = {
     ref = function(v)
         return v:getRef()
     end,
+    null = function()
+        return nil
+    end,
 }
 
---~ checked_access.real    = epdf.real
---~ checked_access.integer = epdf.integer
---~ checked_access.string  = epdf.string
---~ checked_access.boolean = epdf.boolean
---~ checked_access.name    = epdf.name
---~ checked_access.ref     = epdf.ref
+-- checked_access.real    = epdf.real
+-- checked_access.integer = epdf.integer
+-- checked_access.string  = epdf.string
+-- checked_access.boolean = epdf.boolean
+-- checked_access.name    = epdf.name
+-- checked_access.ref     = epdf.ref
 
 local function getnames(document,n,target) -- direct
     if n then
@@ -323,6 +326,15 @@ function lpdf.epdf.load(filename)
     return document
 end
 
+-- for k, v in next, expand(t) do
+
+function lpdf.epdf.expand(t)
+    if type(t) == "table" then
+        local dummy = t.dummy
+    end
+    return t
+end
+
 -- helpers
 
 -- function lpdf.epdf.getdestinationpage(document,name)
diff --git a/tex/context/base/node-inj.lua b/tex/context/base/node-inj.lua
index bcc7b08c5..2cbcc8b88 100644
--- a/tex/context/base/node-inj.lua
+++ b/tex/context/base/node-inj.lua
@@ -34,7 +34,6 @@ local traverse_id        = node.traverse_id
 local unset_attribute    = node.unset_attribute
 local has_attribute      = node.has_attribute
 local set_attribute      = node.set_attribute
-local copy_node          = node.copy
 local insert_node_before = node.insert_before
 local insert_node_after  = node.insert_after
 
@@ -46,27 +45,16 @@ local curscurs = attributes.private('curscurs')
 local cursdone = attributes.private('cursdone')
 local kernpair = attributes.private('kernpair')
 local ligacomp = attributes.private('ligacomp')
-local fontkern = attributes.private('fontkern')
-
-if context then
-
-    local kern = nodes.pool.register(newkern())
-
-    set_attribute(kern,fontkern,1) -- we can have several, attributes are shared
-
-    newkern = function(k)
-        local c = copy_node(kern)
-        c.kern = k
-        return c
-    end
-
-end
 
 -- This injector has been tested by Idris Samawi Hamid (several arabic fonts as well as
 -- the rather demanding Husayni font), Khaled Hosny (latin and arabic) and Kaj Eigner
 -- (arabic, hebrew and thai) and myself (whatever font I come across). I'm pretty sure
 -- that this code is not 100% okay but examples are needed to figure things out.
 
+function injections.installnewkern(nk)
+    newkern = nk or newkern
+end
+
 local cursives = { }
 local marks    = { }
 local kerns    = { }
diff --git a/tex/context/base/node-tra.lua b/tex/context/base/node-tra.lua
index 34d51529c..4ff6c1033 100644
--- a/tex/context/base/node-tra.lua
+++ b/tex/context/base/node-tra.lua
@@ -539,7 +539,7 @@ end
 
 nodes.showsimplelist = function(h,depth) showsimplelist(h,depth,0) end
 
-local function listtoutf(h,joiner,textonly)
+local function listtoutf(h,joiner,textonly,last)
     local joiner = (joiner == true and utfchar(0x200C)) or joiner -- zwnj
     local w = { }
     while h do
@@ -563,7 +563,11 @@ local function listtoutf(h,joiner,textonly)
         else
             w[#w+1] = "[-]"
         end
-        h = h.next
+        if h == last then
+            break
+        else
+            h = h.next
+        end
     end
     return concat(w)
 end
diff --git a/tex/context/base/page-txt.mkvi b/tex/context/base/page-txt.mkvi
index c6ef366d3..dc445e0e1 100644
--- a/tex/context/base/page-txt.mkvi
+++ b/tex/context/base/page-txt.mkvi
@@ -699,7 +699,7 @@
 \let\m_page_layouts_page_number_location_h\relax
 \let\m_page_layouts_page_number_location_x\relax
 
-\def\page_layouts_place_page_number % also elsewhere .. beware, not \unexpanded else
+\def\page_layouts_place_page_number   % also elsewhere .. beware, not \unexpanded else
   {\strc_pagenumbers_place_location}  % test below fails
 
 \def\page_layouts_reset_page_number_location
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index a7a181ece..3bf7b15d5 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 3e6336e26..307ed299a 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/trac-inf.lua b/tex/context/base/trac-inf.lua
index cbc9d13f8..4f5f6ed98 100644
--- a/tex/context/base/trac-inf.lua
+++ b/tex/context/base/trac-inf.lua
@@ -96,7 +96,7 @@ statistics.elapsedtime    = elapsedtime
 statistics.elapsedindeed  = elapsedindeed
 statistics.elapsedseconds = elapsedseconds
 
--- general function
+-- general function .. we might split this module
 
 function statistics.register(tag,fnc)
     if statistics.enable and type(fnc) == "function" then
diff --git a/tex/context/base/trac-lmx.lua b/tex/context/base/trac-lmx.lua
index 452d03002..4db9fde42 100644
--- a/tex/context/base/trac-lmx.lua
+++ b/tex/context/base/trac-lmx.lua
@@ -6,42 +6,59 @@ if not modules then modules = { } end modules ['trac-lmx'] = {
     license   = "see context related readme files"
 }
 
--- todo: use lpeg instead (although not really needed)
-
-local gsub, format, concat, byte = string.gsub, string.format, table.concat, string.byte
-
-local allocate = utilities.storage.allocate
-
-lmx                = lmx or { }
-local lmx          = lmx
-
-lmx.variables      = allocate()
-local lmxvariables = lmx.variables
-
-local escapes = allocate {
-    ['&'] = '&amp;',
-    ['<'] = '&lt;',
-    ['>'] = '&gt;',
-    ['"'] = '&quot;'
+local type, tostring, rawget, loadstring, pcall = type, tostring, rawget, loadstring, pcall
+local format, sub, gsub = string.format, string.sub, string.gsub
+local concat = table.concat
+local P, Cc, Cs, C, Carg, lpegmatch = lpeg.P, lpeg.Cc, lpeg.Cs, lpeg.C, lpeg.Carg, lpeg.match
+
+local allocate          = utilities.storage.allocate
+local setmetatableindex = table.setmetatableindex
+
+----- trace_templates   = false  trackers  .register("lmx.templates",      function(v) trace_templates = v end)
+local trace_variables   = false  trackers  .register("lmx.variables",      function(v) trace_variables = v end)
+
+local cache_templates   = true   directives.register("lmx.cache.templates",function(v) cache_templates = v end)
+local cache_files       = true   directives.register("lmx.cache.files",    function(v) cache_files     = v end)
+
+local report_lmx        = logs.reporter("lmx")
+local report_error      = logs.reporter("lmx","error")
+
+lmx                     = lmx or { }
+local lmx               = lmx
+
+local lmxvariables = {
+    ['title-default']           = 'ConTeXt LMX File',
+    ['color-background-green']  = '#4F6F6F',
+    ['color-background-blue']   = '#6F6F8F',
+    ['color-background-yellow'] = '#8F8F6F',
+    ['color-background-purple'] = '#8F6F8F',
+    ['color-background-body']   = '#808080',
+    ['color-background-main']   = '#3F3F3F',
 }
 
--- variables
-
-lmxvariables['title-default']           = 'ConTeXt LMX File'
-lmxvariables['title']                   = lmx.variables['title-default']
-lmxvariables['color-background-green']  = '#4F6F6F'
-lmxvariables['color-background-blue']   = '#6F6F8F'
-lmxvariables['color-background-yellow'] = '#8F8F6F'
-lmxvariables['color-background-purple'] = '#8F6F8F'
-lmxvariables['color-background-body']   = '#808080'
-lmxvariables['color-background-main']   = '#3F3F3F'
-lmxvariables['color-background-one']    = lmxvariables['color-background-green']
-lmxvariables['color-background-two']    = lmxvariables['color-background-blue']
+local lmxinherited = {
+    ['title']                   = 'title-default',
+    ['color-background-one']    = 'color-background-green',
+    ['color-background-two']    = 'color-background-blue',
+    ['color-background-three']  = 'color-background-one',
+    ['color-background-four']   = 'color-background-two',
+}
 
-lmxvariables['color-background-three']  = function() return lmxvariables['color-background-one'] end
-lmxvariables['color-background-four']   = function() return lmxvariables['color-background-two'] end
+lmx.variables = lmxvariables
+lmx.inherited = lmxinherited
+
+setmetatableindex(lmxvariables,function(t,k)
+    k = lmxinherited[k]
+    while k do
+        local v = rawget(lmxvariables,k)
+        if v then
+            return v
+        end
+        k = lmxinherited[k]
+    end
+end)
 
-function lmx.set(key, value)
+function lmx.set(key,value)
     lmxvariables[key] = value
 end
 
@@ -49,9 +66,16 @@ function lmx.get(key)
     return lmxvariables[key] or ""
 end
 
+lmx.report = report_lmx
+
 -- helpers
 
-local variables, result = { } -- we assume no nesting
+-- the variables table is an empty one that gets linked to a defaults table
+-- that gets passed with a creation (first time only) and that itself links
+-- to one that gets passed to the converter
+
+local variables = { }  -- we assume no nesting
+local result    = { }  -- we assume no nesting
 
 local function do_print(one,two,...)
     if two then
@@ -61,40 +85,117 @@ local function do_print(one,two,...)
     end
 end
 
-local function do_escape(str)
-    str = tostring(str)
-    str = gsub(str,'&','&amp;')
-    str = gsub(str,'[<>"]',escapes)
-    return str
+-- Although it does not make much sense for most elements, we provide a mechanism
+-- to print wrapped content, something that is more efficient when we are constructing
+-- tables.
+
+local html = { }
+lmx.html   = html
+
+function html.td(str)
+    if type(str) == "table" then
+        for i=1,#str do -- spoils t !
+            str[i] = format("<td>%s</td>",str[i])
+        end
+        result[#result+1] = concat(str)
+    else
+        result[#result+1] = format("<td>%s</td>",str or "")
+    end
 end
 
-local function do_urlescaped(str)
-    return (gsub(str,"[^%a%d]",format("%%0x",byte("%1"))))
+function html.th(str)
+    if type(str) == "table" then
+        for i=1,#str do -- spoils t !
+            str[i] = format("<th>%s</th>",str[i])
+        end
+        result[#result+1] = concat(str)
+    else
+        result[#result+1] = format("<th>%s</th>",str or "")
+    end
 end
 
-local function do_type(str)
-    if str then do_print("<tt>" .. do_escape(str) .. "</tt>") end
+setmetatableindex(html,function(t,k)
+    local f = format("<%s>%%s</%s>",k,k)
+    local v = function(str) result[#result+1] = format(f,str or "") end
+    t[k] = v
+    return v
+end)
+
+-- Loading templates:
+
+local function loadedfile(name)
+    name = (resolvers and resolvers.findfile and resolvers.findfile(name)) or name
+    return io.loaddata(name)
+end
+
+lmx.loadedfile = loadedfile
+
+-- A few helpers (the next one could end up in l-lpeg):
+
+local pattern = lpeg.replacer {
+    ["&"] = "&amp;",
+    [">"] = "&gt;",
+    ["<"] = "&lt;",
+    ['"'] = "&quot;",
+}
+
+local function do_escape(str)
+    return lpegmatch(pattern,str) or str
 end
 
 local function do_variable(str)
-    local value = variables[str] or lmxvariables[str] -- or format("<!-- unset lmx instance variable: %s -->",str or "?")
-    if type(value) == "function" then
+    local value = variables[str]
+    if not trace_variables then
+        -- nothing
+    elseif type(value) == "string" then
+        if #value > 80 then
+            report_lmx("variable %q => %s ...",str,string.collapsespaces(sub(value,1,80)))
+        else
+            report_lmx("variable %q => %s",str,string.collapsespaces(value))
+        end
+    elseif type(value) == "nil" then
+        report_lmx("variable %q => <!-- unset -->",str)
+    else
+        report_lmx("variable %q => %q",str,tostring(value))
+    end
+    if type(value) == "function" then -- obsolete ... will go away
         return value(str)
     else
         return value
     end
 end
 
-function lmx.loadedfile(name)
-    name = (resolvers and resolvers.findfile and resolvers.findfile(name)) or name
-    return io.loaddata(name)
+local function do_type(str)
+    if str and str ~= "" then
+        result[#result+1] = format("<tt>%s</tt>",do_escape(str))
+    end
+end
+
+local function do_fprint(...)
+    if str and str ~= "" then
+        result[#result+1] = format(...)
+    end
+end
+
+local function do_print_variable(str)
+    local str = do_variable(str) -- variables[str]
+    if str and str ~= "" then
+        result[#result+1] = str
+    end
 end
 
-local function do_include(filename)
-    local stylepath = do_variable('includepath') -- todo: store paths of loaded files
-    local data = lmx.loadedfile(filename)
+local function do_type_variable(str)
+    local str = do_variable(str) -- variables[str]
+    if str and str ~= "" then
+        result[#result+1] = format("<tt>%s</tt>",do_escape(str))
+    end
+end
+
+local function do_include(filename) -- todo: store paths of loaded files
+    local stylepath = do_variable('includepath') -- variables[str]
+    local data = loadedfile(filename)
     if (not data or data == "") and stylepath and stylepath ~= "" then
-        data = lmx.loadedfile(file.join(stylepath,filename))
+        data = loadedfile(file.join(stylepath,filename))
     end
     if not data or data == "" then
         data = format("<!-- unknown lmx include file: %s -->",filename)
@@ -102,132 +203,348 @@ local function do_include(filename)
     return data
 end
 
+-- Flushers:
+
 lmx.print     = do_print
 lmx.type      = do_type
+lmx.fprint    = do_fprint
+
 lmx.escape    = do_escape
-lmx.urlescape = do_escape
+lmx.urlescape = url.escape
 lmx.variable  = do_variable
 lmx.include   = do_include
 
-function lmx.pv(str)
-    do_print(do_variable(str) or "")
+lmx.inject    = do_print
+lmx.finject   = do_fprint
+
+lmx.pv        = do_print_variable
+lmx.tv        = do_type_variable
+
+-- The next functions set up the closure.
+
+function lmx.initialize(d,v)
+    if not v then
+        setmetatableindex(d,lmxvariables)
+        if variables ~= d then
+            setmetatableindex(variables,d)
+            report_lmx("variables => given defaults => lmx variables")
+        elseif trace_variables then
+            report_lmx("variables == given defaults => lmx variables")
+        end
+    elseif d ~= v then
+        setmetatableindex(v,d)
+        if d ~= lmxvariables then
+            setmetatableindex(d,lmxvariables)
+            if variables ~= v then
+                setmetatableindex(variables,v)
+                if trace_variables then
+                    report_lmx("variables => given variables => given defaults => lmx variables")
+                end
+            elseif trace_variables then
+                report_lmx("variables == given variables => given defaults => lmx variables")
+            end
+        else
+            if variables ~= v then
+                setmetatableindex(variables,v)
+                if trace_variables then
+                    report_lmx("variabes => given variables => given defaults")
+                end
+            elseif trace_variables then
+                report_lmx("variables == given variables => given defaults")
+            end
+        end
+    else
+        setmetatableindex(v,lmxvariables)
+        if variables ~= v then
+            setmetatableindex(variables,v)
+            report_lmx("variables => given variables => lmx variables")
+        elseif trace_variables then
+            report_lmx("variables == given variables => lmx variables")
+        end
+    end
+    result = { }
+end
+
+function lmx.finalized()
+    local collapsed = concat(result)
+    result = { } -- free memory
+    return collapsed
+end
+
+function lmx.getvariables()
+    return variables
 end
 
-function lmx.tv(str)
-    lmx.type(do_variable(str) or "")
+function lmx.reset()
+    -- obsolete
 end
 
+-- Creation:
+
 local template = [[
-    local definitions = { }
-    local p, v, e, t, pv, tv = lmx.print, lmx.variable, lmx.escape, lmx.type, lmx.pv, lmx.tv
-    %s
+return function(defaults,variables)
+
+-- initialize
+
+lmx.initialize(defaults,variables)
+
+-- interface
+
+local definitions = { }
+local variables   = lmx.getvariables()
+local html        = lmx.html
+local inject      = lmx.print
+local finject     = lmx.fprint
+local escape      = lmx.escape
+local verbose     = lmx.type
+
+-- shortcuts (sort of obsolete as there is no gain)
+
+local p  = lmx.print
+local f  = lmx.fprint
+local v  = lmx.variable
+local e  = lmx.escape
+local t  = lmx.type
+local pv = lmx.pv
+local tv = lmx.tv
+
+-- generator
+
+%s
+
+-- finalize
+
+return lmx.finalized()
+
+end
 ]]
 
+local function savedefinition(definitions,tag,content)
+    definitions[tag] = content
+    return ""
+end
+
+local function getdefinition(definitions,tag)
+    return definitions[tag] or ""
+end
+
+local whitespace    = lpeg.patterns.whitespace^0
+
+local beginembedxml = P("<?")
+local endembedxml   = P("?>")
+
+local beginembedcss = P("/*")
+local endembedcss   = P("*/")
+
+local gobbledend    = (whitespace * endembedxml) / ""
+local argument      = (1-gobbledend)^0
+
+local beginluaxml   = (beginembedxml * P("lua")) / ""
+local endluaxml     = endembedxml / ""
+
+local luacodexml    = beginluaxml
+                    * (1-endluaxml)^1
+                    * endluaxml
+
+local beginluacss   = (beginembedcss * P("lua")) / ""
+local endluacss     = endembedcss / ""
+
+local luacodecss    = beginluacss
+                    * (1-endluacss)^1
+                    * endluacss
+
+local othercode     = Cc(" p[==[")
+                    * (1-beginluaxml-beginluacss)^1
+                    * Cc("]==] ")
+
+local include       = ((beginembedxml * P("lmx-include") * whitespace) / "")
+                    * (argument / lmx.include)
+                    * gobbledend
+
+local define_b      = ((beginembedxml * P("lmx-define-begin") * whitespace) / "")
+                    * argument
+                    * gobbledend
+
+local define_e      = ((beginembedxml * P("lmx-define-end") * whitespace) / "")
+                    * argument
+                    * gobbledend
+
+local define_c      = C((1-define_e)^0)
+
+local define        = (Carg(1) * C(define_b) * define_c * define_e) / savedefinition
+
+local resolve       = ((beginembedxml * P("lmx-resolve") * whitespace) / "")
+                    * ((Carg(1) * C(argument)) / getdefinition)
+                    * gobbledend
+
+local pattern_1     = Cs((include + P(1))^0)
+local pattern_2     = Cs((define  + resolve + P(1))^0)
+local pattern_3     = Cs((luacodexml + luacodecss + othercode)^0)
+
 local cache = { }
 
-local trace = false
+local function lmxerror(str)
+    report_error(str)
+    return html.tt(str)
+end
+
+local function wrapper(converter,defaults,variables)
+    local outcome, message = pcall(converter,defaults,variables)
+    if not outcome then
+        return lmxerror(format("error in conversion: %s",message))
+    else
+        return message
+    end
+end
 
-function lmx.new(data,variables)
+function lmxnew(data,defaults)
     data = data or ""
     local known = cache[data]
     if not known then
-        local definitions = { }
-        data = gsub(data,"<%?lmx%-include%s+(.-)%s-%?>", function(filename)
-            return lmx.include(filename)
-        end)
-        local definitions =  { }
-        data = gsub(data,"<%?lmx%-define%-begin%s+(%S-)%s-%?>(.-)<%?lmx%-define%-end%s-%?>", function(tag,content)
-            definitions[tag] = content
-            return ""
-        end)
-        data = gsub(data,"<%?lmx%-resolve%s+(%S-)%s-%?>", function(tag)
-            return definitions[tag] or ""
-        end)
-        data = gsub(data .. "<?lua ?>","(.-)<%?lua%s+(.-)%s*%?>", function(txt,lua)
-            txt = gsub(txt,"%c+","\n")
-            return format("p(%q)%s ",txt,lua) -- nb! space
-        end)
-        data = format(template,data)
+        defaults = defaults or { }
+        data = lpegmatch(pattern_1,data)
+        data = lpegmatch(pattern_2,data,1,{})
+        data = lpegmatch(pattern_3,data)
+        local converted = loadstring(format(template,data))
+        if converted then
+            converted = converted()
+        end
+        local converter
+        if converted then
+            converter = function(variables)
+                return wrapper(converted,defaults,variables)
+            end
+        else
+            converter = function() lmxerror("error in template") end
+        end
         known = {
-            data = trace and data,
-            variables = variables or { },
-            converter = loadstring(data),
+            data      = defaults.trace and data or "",
+            variables = defaults,
+            converter = converter,
         }
+        if cache_templates then
+            cache[data] = known
+        end
     elseif variables then
         known.variables = variables
     end
     return known, known.variables
 end
 
-function lmx.reset(self)
-    self.variables = { }
-end
-
-function lmx.result(self)
-    if trace then
-        return self.data
+local function lmxresult(self,variables)
+    if self then
+        local converter = self.converter
+        if converter then
+            return converter(variables)
+        else
+            return lmxerror("invalid converter")
+        end
     else
-        variables, result = self.variables, { }
-        self.converter()
-        return concat(result)
+        return lmxerror("invalid specification")
     end
 end
 
--- file converter
+lmx.new    = lmxnew
+lmx.result = lmxresult
 
-local loaded = { }
+local loadedfiles = { }
 
-function lmx.convert(templatefile,resultfile,variables)
-    local data = loaded[templatefile]
-    if not data then
-        data = lmx.new(lmx.loadedfile(templatefile),variables)
-        loaded[template] = data
-    elseif variables then
-        data.variables = variables
+function lmx.convertstring(templatestring,variables)
+    return lmxresult(lmxnew(templatestring),variables)
+end
+
+function lmx.convertfile(templatefile,variables)
+    local converter = loadedfiles[templatefile]
+    if not converter then
+        converter = lmxnew(loadedfile(templatefile))
+        loadedfiles[templatefile] = converter
+    end
+    return lmxresult(converter,variables)
+end
+
+function lmxconvert(templatefile,resultfile,variables) -- or (templatefile,variables)
+    if not variables and type(resultfile) == "table" then
+        variables = resultfile
     end
-    local result = lmx.result(data)
+    local converter = loadedfiles[templatefile]
+    if not converter then
+        converter = lmxnew(loadedfile(templatefile))
+        if cache_files then
+            loadedfiles[templatefile] = converter
+        end
+    end
+    local result = lmxresult(converter,variables)
     if resultfile then
         io.savedata(resultfile,result)
     else
-        return lmx.result(data,result)
+        return result
     end
 end
 
--- these can be overloaded; we assume that the os handles filename associations
+lmx.convert = lmxconvert
 
-lmx.lmxfile = function(filename) return filename end -- beware, these can be set!
-lmx.htmfile = function(filename) return filename end -- beware, these can be set!
+-- these can be overloaded
 
-if os.type == "windows" then
-    lmx.popupfile = function(filename) os.execute("start " .. filename) end
-else
-    lmx.popupfile = function(filename) os.execute(filename) end
-end
+lmx.lmxfile   = string.itself
+lmx.htmfile   = string.itself
+lmx.popupfile = os.launch
 
-function lmx.make(name,variables)
+function lmxmake(name,variables)
     local lmxfile = lmx.lmxfile(name)
     local htmfile = lmx.htmfile(name)
     if lmxfile == htmfile then
-        htmfile = gsub(lmxfile, "%.%a+$", "html")
+        htmfile = file.replacesuffix(lmxfile,"html")
     end
-    lmx.convert(lmxfile,htmfile,variables)
+    lmxconvert(lmxfile,htmfile,variables)
     return htmfile
 end
 
+lmxmake = lmx.make
+
 function lmx.show(name,variables)
-    local htmfile = lmx.make(name,variables)
+    local htmfile = lmxmake(name,variables)
     lmx.popupfile(htmfile)
     return htmfile
 end
 
--- test
-
---~ print(lmx.result(lmx.new(io.loaddata("t:/sources/context-timing.lmx"))))
-
--- command line
+-- Command line (will become mtx-lmx):
 
 if arg then
     if     arg[1] == "--show"    then if arg[2] then lmx.show   (arg[2])                        end
     elseif arg[1] == "--convert" then if arg[2] then lmx.convert(arg[2], arg[3] or "temp.html") end
     end
 end
+
+-- Test 1:
+
+-- inspect(lmx.result(lmx.new(io.loaddata("t:/sources/context-timing.lmx"))))
+
+-- Test 2:
+
+-- local str = [[
+--     <?lmx-include somefile.css ?>
+--     <test>
+--         <?lmx-define-begin whatever?>some content a<?lmx-define-end ?>
+--         <?lmx-define-begin somemore?>some content b<?lmx-define-end ?>
+--         <more>
+--             <?lmx-resolve whatever ?>
+--             <?lua
+--                 for i=1,10 do end
+--             ?>
+--             <?lmx-resolve somemore ?>
+--         </more>
+--         <td><?lua p(100) ?></td>
+--         <td><?lua p(variables.a) ?></td>
+--         <td><?lua p(variables.b) ?></td>
+--         <td><?lua p(variables.c) ?></td>
+--         <td><?lua pv('title-default') ?></td>
+--     </test>
+-- ]]
+--
+-- local defaults = { trace = true, a = 3, b = 3 }
+-- local result = lmx.new(str,defaults)
+-- inspect(result.data)
+-- inspect(result.converter(defaults))
+-- inspect(result.converter { a = 1 })
+-- inspect(lmx.result(result, { b = 2 }))
+-- inspect(lmx.result(result, { a = 20000, b = 40000 }))
diff --git a/tex/context/base/trac-vis.lua b/tex/context/base/trac-vis.lua
index 81994c38f..9349a2aac 100644
--- a/tex/context/base/trac-vis.lua
+++ b/tex/context/base/trac-vis.lua
@@ -41,6 +41,7 @@ local vlist_code          = nodecodes.vlist
 local glue_code           = nodecodes.glue
 local penalty_code        = nodecodes.penalty
 local whatsit_code        = nodecodes.whatsit
+local user_code           = nodecodes.user
 
 local kerncodes           = nodes.kerncodes
 local font_kern_code      = kerncodes.fontkern
@@ -121,6 +122,7 @@ local trace_penalty
 local trace_fontkern
 local trace_strut
 local trace_whatsit
+local trace_user
 
 local report_visualize = logs.reporter("visualize")
 
@@ -139,18 +141,19 @@ local modes = {
     simplehbox = 1024 + 1,
     simplevbox = 1024 + 2,
     simplevtop = 1024 + 4,
+    user       = 2048,
 }
 
 -- local modes_makeup = { "hbox", "vbox", "vtop", "kern", "glue", "penalty" }
 -- local modes_boxes  = { "hbox", "vbox", "vtop" }
--- local modes_all    = { "hbox", "vbox", "vtop", "kern", "glue", "penalty", "fontkern", "whatsit", "glyph" }
+-- local modes_all    = { "hbox", "vbox", "vtop", "kern", "glue", "penalty", "fontkern", "whatsit", "glyph", "user" }
 
 local modes_makeup = { "hbox", "vbox", "kern", "glue", "penalty" }
 local modes_boxes  = { "hbox", "vbox"  }
-local modes_all    = { "hbox", "vbox", "kern", "glue", "penalty", "fontkern", "whatsit", "glyph" }
+local modes_all    = { "hbox", "vbox", "kern", "glue", "penalty", "fontkern", "whatsit", "glyph", "user" }
 
 local usedfont, exheight, emwidth
-local l_penalty, l_glue, l_kern, l_fontkern, l_hbox, l_vbox, l_vtop, l_strut, l_whatsit, l_glyph
+local l_penalty, l_glue, l_kern, l_fontkern, l_hbox, l_vbox, l_vtop, l_strut, l_whatsit, l_glyph, l_user
 
 local enabled = false
 local layers  = { }
@@ -226,6 +229,7 @@ local function setvisual(n,a,what) -- this will become more efficient when we ha
         l_strut    = layers.strut
         l_whatsit  = layers.whatsit
         l_glyph    = layers.glyph
+        l_user     = layers.user
         nodes.tasks.enableaction("shipouts","nodes.visualizers.handler")
         report_visualize("enabled")
         enabled = true
@@ -354,6 +358,20 @@ local function whatsit(head,current)
     return head, current
 end
 
+local function user(head,current)
+    local what = current.subtype
+    local info = w_cache[what]
+    if info then
+        -- print("hit user")
+    else
+        local info = sometext(format("U:%s",what),usedfont)
+        set_attribute(info,a_layer,l_user)
+        w_cache[what] = info
+    end
+    head, current = insert_after(head,current,copy_list(info))
+    return head, current
+end
+
 local b_cache = { }
 
 local function ruledbox(head,current,vertical,layer,what,simple)
@@ -640,6 +658,7 @@ local function visualize(head,vertical)
     local trace_whatsit  = false
     local trace_glyph    = false
     local trace_simple   = false
+    local trace_user     = false
     local current = head
     local prev_trace_fontkern = nil
     local attr = unsetvalue
@@ -660,6 +679,7 @@ local function visualize(head,vertical)
                 trace_whatsit  = false
                 trace_glyph    = false
                 trace_simple   = false
+                trace_user     = false
             else -- dead slow:
                 trace_hbox     = hasbit(a,   1)
                 trace_vbox     = hasbit(a,   2)
@@ -672,6 +692,7 @@ local function visualize(head,vertical)
                 trace_whatsit  = hasbit(a, 256)
                 trace_glyph    = hasbit(a, 512)
                 trace_simple   = hasbit(a,1024)
+                trace_user     = hasbit(a,2048)
             end
             attr = a
         end
@@ -745,6 +766,10 @@ local function visualize(head,vertical)
             if trace_whatsit then
                 head, current = whatsit(head,current)
             end
+        elseif id == user_code then
+            if trace_whatsit then
+                head, current = user(head,current)
+            end
         end
         current = current.next
     end
diff --git a/tex/context/base/util-sql.lua b/tex/context/base/util-sql.lua
index 1f7842a83..1eabe3c31 100644
--- a/tex/context/base/util-sql.lua
+++ b/tex/context/base/util-sql.lua
@@ -6,116 +6,179 @@ if not modules then modules = { } end modules ['util-sql'] = {
     license   = "see context related readme files"
 }
 
+-- Of course we could use a library but we don't want another depedency and
+-- there is a bit of flux in these libraries. Also, we want the data back in
+-- a way that we like.
+
+-- buffer template
+
 local format = string.format
-local P, V, Ct, Cs, Cc, Cg, Cf, patterns, lpegmatch = lpeg.P, lpeg.V, lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.patterns, lpeg.match
+local rawset, setmetatable = rawset, setmetatable
+local P, V, C, Ct, Cc, Cg, Cf, patterns, lpegmatch = lpeg.P, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.patterns, lpeg.match
+
+local osclock = os.clock or os.time
 
-local trace_sql  = false  trackers.register("sql.trace",function(v) trace_sql = v end)
+local trace_sql    = false  trackers.register("sql.trace",function(v) trace_sql = v end)
 local report_state = logs.reporter("sql")
 
-utilities.sql = utilities.sql or { }
-local sql     = utilities.sql
+utilities.sql      = utilities.sql or { }
+local sql          = utilities.sql
 
-local inifile = ""
+local separator    = P("\t")
+local newline      = patterns.newline
+local entry        = C((1-separator-newline)^1) -- C 10% faster than C
+local empty        = Cc("")
 
--- todo:
+local getfirst     = Ct( entry * (separator * (entry+empty))^0) + newline
+local skipfirst    = (1-newline)^1 * newline
 
-if os.platform == "mswin" then
-    inifile = "C:\\Program Files\\MySQL\\MySQL Server 5.5\\ld-test.ini"
-else
-    inifile = "/etc/mysql/ld-test.ini"
-end
+local defaults     = { __index =
+    {
+        resultfile     = "result.dat",
+        templatefile   = "template.sql",
+        queryfile      = "query.sql",
+        variables      = { },
+        username       = "default",
+        password       = "default",
+        host           = "localhost",
+        port           = 3306,
+        database       = "default",
+    },
+}
 
-local separator = P("\t")
-local newline   = patterns.newline
-local entry     = Cs((1-separator-newline)^1)
-local empty     = Cc("")
+local engine  = "mysql"
 
-local getfirst  = Ct( entry * (separator * (entry+empty))^0) + newline
-local skipfirst = (1-newline)^1 * newline
+local runners = { -- --defaults-extra-file="%inifile"
+    mysql = [[mysql --user="%username%" --password="%password%" --host="%host%" --port=%port% --database="%database%" < "%queryfile%" > "%resultfile%"]]
+}
 
--- -- faster but less flexible:
---
--- local splitter  = Ct ( (getfirst)^1 )
---
--- local function splitdata(data)
---     return lpegmatch(splitter,data) or { }
--- end
+-- Experiments with an p/action demonstrated that there is not much gain. We could do a runtime
+-- capture but creating all the small tables is not faster and it doesn't work well anyway.
 
 local function splitdata(data)
     if data == "" then
         if trace_sql then
             report_state("no data")
         end
-        return { }
+        return { }, { }
     end
-    local t = lpegmatch(getfirst,data) or { }
-    if #t == 0 then
+    local keys = lpegmatch(getfirst,data) or { }
+    if #keys == 0 then
         if trace_sql then
             report_state("no banner")
         end
-        return { }
+        return { }, { }
     end
     -- quite generic, could be a helper
     local p = nil
-    for i=1,#t do
-        local ti = t[i]
+    local n = #keys
+--     for i=1,n do
+--         local key = keys[i]
+--         if trace_sql then
+--             report_state("field %s has name %q",i,key)
+--         end
+--         local s = Cg(Cc(key) * entry)
+--         if p then
+--             p = p * s
+--         else
+--             p = s
+--         end
+--         if i < n then
+--             p = p * separator
+--         end
+--     end
+    for i=1,n do
+        local key = keys[i]
         if trace_sql then
-            report_state("field %s has name %q",i,ti)
+            report_state("field %s has name %q",i,key)
         end
-        local s = Cg(Cc(ti) * entry)
+        local s = Cg(Cc(key) * entry)
         if p then
-            p = p * s
+            p = p * separator * s
         else
             p = s
         end
-        if i < #t then
-            p = p * separator
-        end
     end
     p = Cf(Ct("") * p,rawset) * newline^0
-    local d = lpegmatch(skipfirst * Ct(p^0),data)
-    return d or { }
+    local entries = lpegmatch(skipfirst * Ct(p^0),data)
+    return entries or { }, keys
 end
 
-local function preparedata(sqlfile,templatefile,mapping)
-    local query = utilities.templates.load(templatefile,mapping)
-    io.savedata(sqlfile,query)
+-- I will add a bit more checking.
+
+local function validspecification(specification)
+    local presets = specification.presets
+    if type(presets) == "string" then
+        presets = dofile(presets)
+    end
+    if type(presets) == "table" then
+        setmetatable(presets,defaults)
+        setmetatable(specification,{ __index = presets })
+    else
+        setmetatable(specification,defaults)
+    end
+    local templatefile = specification.templatefile
+    local queryfile    = specification.queryfile  or file.nameonly(templatefile) .. "-temp.sql"
+    local resultfile   = specification.resultfile or file.nameonly(templatefile) .. "-temp.dat"
+    specification.queryfile  = queryfile
+    specification.resultfile = resultfile
+    if trace_sql then
+        report_state("template file: %q",templatefile)
+        report_state("query file: %q",queryfile)
+        report_state("result file: %q",resultfile)
+    end
+    return true
 end
 
-local function fetchdata(sqlfile,datfile)
-    local command
-    if inifile ~= "" then
-        command = format([[mysql --defaults-extra-file="%s" < %s > %s]],inifile,sqlfile,datfile)
+local function dataprepared(specification)
+    local query = false
+    if specification.template then
+        query = utilities.templates.replace(specification.template,specification.variables)
+    elseif specification.templatefile then
+        query = utilities.templates.load(specification.templatefile,specification.variables)
+    end
+    if query then
+        io.savedata(specification.queryfile,query)
+        return true
     else
-        command = format([[[mysql < %s > %s]],sqlfile,datfile)
+        -- maybe push an error
+        os.remove(specification.queryfile)
     end
+end
+
+local function datafetched(specification)
+    local command = utilities.templates.replace(runners[engine],specification)
     if trace_sql then
-        local t = os.clock()
+        local t = osclock()
+        report_state("command: %s",command)
         os.execute(command)
-        report_state("fetchtime: %.3f sec",os.clock()-t) -- not okay under linux
+        report_state("fetchtime: %.3f sec",osclock()-t) -- not okay under linux
     else
         os.execute(command)
     end
+    return true
 end
 
-local function loaddata(datfile)
+local function dataloaded(specification)
     if trace_sql then
-        local t = os.clock()
-        local data = io.loaddata(datfile) or ""
+        local t = osclock()
+        local data = io.loaddata(specification.resultfile) or ""
         report_state("datasize: %.3f MB",#data/1024/1024)
-        report_state("loadtime: %.3f sec",os.clock()-t)
+        report_state("loadtime: %.3f sec",osclock()-t)
         return data
     else
-        return io.loaddata(datfile) or ""
+        return io.loaddata(specification.resultfile) or ""
     end
 end
 
-local function convertdata(data)
+local function dataconverted(data)
     if trace_sql then
-        local t = os.clock()
-        data = splitdata(data)
-        report_state("converttime: %.3f",os.clock()-t)
-        report_state("entries: %s ",#data) -- #data-1 if indexed
+        local t = osclock()
+        local data, keys = splitdata(data)
+        report_state("converttime: %.3f",osclock()-t)
+        report_state("keys: %s ",#keys)
+        report_state("entries: %s ",#data)
+        return data, keys
     else
         return splitdata(data)
     end
@@ -123,37 +186,108 @@ end
 
 -- todo: new, etc
 
-function sql.fetch(templatefile,mapping)
-    local sqlfile = file.nameonly(templatefile) .. "-temp.sql"
-    local datfile = file.nameonly(templatefile) .. "-temp.dat"
-    preparedata(sqlfile,templatefile,mapping)
-    fetchdata(sqlfile,datfile)
-    local data = loaddata(datfile)
-    data = convertdata(data)
-    return data
+function sql.fetch(specification)
+    if trace_sql then
+        report_state("fetching")
+    end
+    if not validspecification(specification) then
+        report("error in specification")
+        return
+    end
+    if not dataprepared(specification) then
+        report("error in preparation")
+        return
+    end
+    if not datafetched(specification) then
+        report("error in fetching")
+        return
+    end
+    local data = dataloaded(specification)
+    if not data then
+        report("error in loading")
+        return
+    end
+    local data, keys = dataconverted(data)
+    if not data then
+        report("error in converting")
+        return
+    end
+    return data, keys
 end
 
-function sql.reuse(templatefile)
-    local datfile = file.nameonly(templatefile) .. "-temp.dat"
-    local data = loaddata(datfile)
-    data = convertdata(data)
-    return data
+function sql.reuse(specification)
+    if trace_sql then
+        report_state("reusing")
+    end
+    if not validspecification(specification) then
+        report("error in specification")
+        return
+    end
+    local data = dataloaded(specification)
+    if not data then
+        report("error in loading")
+        return
+    end
+    local data, keys = dataconverted(data)
+    if not data then
+        report("error in converting")
+        return
+    end
+    return data, keys
 end
 
--- tex specific
+sql.splitdata = splitdata
+
+-- -- --
+
+-- local data = utilities.sql.prepare {
+--     templatefile = "ld-003.sql",
+--     variables    = { },
+--     host         = "...",
+--     username     = "...",
+--     password     = "...",
+--     database     = "...",
+-- }
+
+-- local presets = {
+--     host     = "...",
+--     username = "...",
+--     password = "...",
+--     database = "...",
+-- }
+--
+-- local data = utilities.sql.prepare {
+--     templatefile = "ld-003.sql",
+--     variables    = { },
+--     presets      = presets,
+-- }
+
+-- local data = utilities.sql.prepare {
+--     templatefile = "ld-003.sql",
+--     variables    = { },
+--     presets      = dofile(...),
+-- }
+
+-- local data = utilities.sql.prepare {
+--     templatefile = "ld-003.sql",
+--     variables    = { },
+--     presets      = "...",
+-- }
+
+-- -- --
 
-if tex then
+if tex and tex.systemmodes then
 
-    function sql.prepare(sqlfile,mapping)
+    function sql.prepare(specification)
         if tex.systemmodes["first"] then
-            return utilities.sql.fetch(sqlfile,mapping)
+            return sql.fetch(specification)
         else
-            return utilities.sql.reuse(sqlfile)
+            return sql.reuse(specification)
         end
     end
 
 else
 
-    sql.prepare = utilities.sql.fetch
+    sql.prepare = sql.fetch
 
 end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 6f624c38f..78606488a 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 08/14/12 10:44:56
+-- merge date  : 08/16/12 22:20:23
 
 do -- begin closure to overcome local limits and interference
 
@@ -214,12 +214,16 @@ function table.strip(tab)
 end
 
 function table.keys(t)
-    local keys, k = { }, 0
-    for key, _ in next, t do
-        k = k + 1
-        keys[k] = key
+    if t then
+        local keys, k = { }, 0
+        for key, _ in next, t do
+            k = k + 1
+            keys[k] = key
+        end
+        return keys
+    else
+        return { }
     end
-    return keys
 end
 
 local function compare(a,b)
@@ -232,41 +236,49 @@ local function compare(a,b)
 end
 
 local function sortedkeys(tab)
-    local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
-    for key,_ in next, tab do
-        s = s + 1
-        srt[s] = key
-        if category == 3 then
-            -- no further check
-        else
-            local tkey = type(key)
-            if tkey == "string" then
-                category = (category == 2 and 3) or 1
-            elseif tkey == "number" then
-                category = (category == 1 and 3) or 2
+    if tab then
+        local srt, category, s = { }, 0, 0 -- 0=unknown 1=string, 2=number 3=mixed
+        for key,_ in next, tab do
+            s = s + 1
+            srt[s] = key
+            if category == 3 then
+                -- no further check
             else
-                category = 3
+                local tkey = type(key)
+                if tkey == "string" then
+                    category = (category == 2 and 3) or 1
+                elseif tkey == "number" then
+                    category = (category == 1 and 3) or 2
+                else
+                    category = 3
+                end
             end
         end
-    end
-    if category == 0 or category == 3 then
-        sort(srt,compare)
+        if category == 0 or category == 3 then
+            sort(srt,compare)
+        else
+            sort(srt)
+        end
+        return srt
     else
-        sort(srt)
+        return { }
     end
-    return srt
 end
 
 local function sortedhashkeys(tab) -- fast one
-    local srt, s = { }, 0
-    for key,_ in next, tab do
-        if key then
-            s= s + 1
-            srt[s] = key
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if key then
+                s= s + 1
+                srt[s] = key
+            end
         end
+        sort(srt)
+        return srt
+    else
+        return { }
     end
-    sort(srt)
-    return srt
 end
 
 table.sortedkeys     = sortedkeys
@@ -291,7 +303,7 @@ end
 table.sortedhash  = sortedhash
 table.sortedpairs = sortedhash
 
-function table.append(t, list)
+function table.append(t,list)
     local n = #t
     for i=1,#list do
         n = n + 1
@@ -1172,7 +1184,7 @@ local report = texio and texio.write_nl or print
 -- function lpeg.Cmt  (l) local p = lpcmt (l) report("LPEG Cmt =")  lpprint(l) return p end
 -- function lpeg.Carg (l) local p = lpcarg(l) report("LPEG Carg =") lpprint(l) return p end
 
-local type = type
+local type, next = type, next
 local byte, char, gmatch, format = string.byte, string.char, string.gmatch, string.format
 
 -- Beware, we predefine a bunch of patterns here and one reason for doing so
@@ -1233,6 +1245,10 @@ patterns.utf8char      = utf8char
 patterns.validutf8     = validutf8char
 patterns.validutf8char = validutf8char
 
+local eol              = S("\n\r")
+local spacer           = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+local whitespace       = eol + spacer
+
 patterns.digit         = digit
 patterns.sign          = sign
 patterns.cardinal      = sign^0 * digit^1
@@ -1252,16 +1268,16 @@ patterns.letter        = patterns.lowercase + patterns.uppercase
 patterns.space         = space
 patterns.tab           = P("\t")
 patterns.spaceortab    = patterns.space + patterns.tab
-patterns.eol           = S("\n\r")
-patterns.spacer        = S(" \t\f\v")  -- + char(0xc2, 0xa0) if we want utf (cf mail roberto)
+patterns.eol           = eol
+patterns.spacer        = spacer
+patterns.whitespace    = whitespace
 patterns.newline       = newline
 patterns.emptyline     = newline^1
-patterns.nonspacer     = 1 - patterns.spacer
-patterns.whitespace    = patterns.eol + patterns.spacer
-patterns.nonwhitespace = 1 - patterns.whitespace
+patterns.nonspacer     = 1 - spacer
+patterns.nonwhitespace = 1 - whitespace
 patterns.equal         = P("=")
 patterns.comma         = P(",")
-patterns.commaspacer   = P(",") * patterns.spacer^0
+patterns.commaspacer   = P(",") * spacer^0
 patterns.period        = P(".")
 patterns.colon         = P(":")
 patterns.semicolon     = P(";")
@@ -1493,8 +1509,8 @@ end
 function lpeg.replacer(one,two)
     if type(one) == "table" then
         local no = #one
+        local p
         if no > 0 then
-            local p
             for i=1,no do
                 local o = one[i]
                 local pp = P(o[1]) / o[2]
@@ -1504,8 +1520,17 @@ function lpeg.replacer(one,two)
                     p = pp
                 end
             end
-            return Cs((p + 1)^0)
+        else
+            for k, v in next, one do
+                local pp = P(k) / v
+                if p then
+                    p = p + pp
+                else
+                    p = pp
+                end
+            end
         end
+        return Cs((p + 1)^0)
     else
         two = two or ""
         return Cs((P(one)/two + 1)^0)
@@ -1900,6 +1925,14 @@ function string.tformat(fmt,...)
     return format(lpegmatch(replacer,fmt),...)
 end
 
+-- strips leading and trailing spaces and collapsed all other spaces
+
+local pattern = Cs(whitespace^0/"" * ((whitespace^1 * P(-1) / "") + (whitespace^1/" ") + P(1))^0)
+
+function string.collapsespaces(str)
+    return lpegmatch(pattern,str)
+end
+
 end -- closure
 
 do -- begin closure to overcome local limits and interference
@@ -8232,7 +8265,6 @@ local traverse_id        = node.traverse_id
 local unset_attribute    = node.unset_attribute
 local has_attribute      = node.has_attribute
 local set_attribute      = node.set_attribute
-local copy_node          = node.copy
 local insert_node_before = node.insert_before
 local insert_node_after  = node.insert_after
 
@@ -8244,27 +8276,16 @@ local curscurs = attributes.private('curscurs')
 local cursdone = attributes.private('cursdone')
 local kernpair = attributes.private('kernpair')
 local ligacomp = attributes.private('ligacomp')
-local fontkern = attributes.private('fontkern')
-
-if context then
-
-    local kern = nodes.pool.register(newkern())
-
-    set_attribute(kern,fontkern,1) -- we can have several, attributes are shared
-
-    newkern = function(k)
-        local c = copy_node(kern)
-        c.kern = k
-        return c
-    end
-
-end
 
 -- This injector has been tested by Idris Samawi Hamid (several arabic fonts as well as
 -- the rather demanding Husayni font), Khaled Hosny (latin and arabic) and Kaj Eigner
 -- (arabic, hebrew and thai) and myself (whatever font I come across). I'm pretty sure
 -- that this code is not 100% okay but examples are needed to figure things out.
 
+function injections.installnewkern(nk)
+    newkern = nk or newkern
+end
+
 local cursives = { }
 local marks    = { }
 local kerns    = { }
@@ -8883,6 +8904,7 @@ local glyph_code         = nodecodes.glyph
 local glue_code          = nodecodes.glue
 local disc_code          = nodecodes.disc
 local whatsit_code       = nodecodes.whatsit
+local user_code          = nodecodes.user
 
 local dir_code           = whatcodes.dir
 local localpar_code      = whatcodes.localpar
@@ -10420,7 +10442,7 @@ local function normal_handle_contextchain(start,kind,chainname,contexts,sequence
                                 break
                             end
                             prev = prev.prev
-                        elseif seq[n][32] then -- somehat special, as zapfino can have many preceding spaces
+                        elseif seq[n][32] then -- somewhat special, as zapfino can have many preceding spaces
                             n = n -1
                         else
                             match = false