beta 2013.02.13 18:06

27 files changed, 467 insertions, 177 deletions

diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index f35d7d2a9..277967ef3 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -635,22 +635,13 @@ end
 <p>Setting the lccodes is also done in a loop over the data table.</p>
 --ldx]]--
 
---~ function tex.setsfcode (index,sf) ... end
---~ function tex.setlccode (index,lc,[uc]) ... end -- optional third value, safes call
---~ function tex.setuccode (index,uc,[lc]) ... end
---~ function tex.setcatcode(index,cc) ... end
-
--- we need a function ...
-
---~ tex.lccode
---~ tex.uccode
---~ tex.sfcode
---~ tex.catcode
+local sfmode = "unset" -- unset, traditional, normal
 
 function characters.setcodes()
     if trace_defining then
         report_defining("defining lc and uc codes")
     end
+    local traditional = sfstate == "traditional" or sfstate == "unset"
     for code, chr in next, data do
         local cc = chr.category
         if is_letter[cc] then
@@ -674,7 +665,7 @@ function characters.setcodes()
                 end
                 texsetcatcode(code,11)   -- letter
                 texsetlccode(code,lc,uc)
-                if cc == "lu" then
+                if traditional and cc == "lu" then
                     texsetsfcode(code,999)
                 end
             end
@@ -682,8 +673,34 @@ function characters.setcodes()
             texsetlccode(code,code,code) -- for hyphenation
         end
     end
+    if traditional then
+        sfstate = "traditional"
+    end
+end
+
+-- If this is something that is not documentwide and used a lot, then we
+-- need a more clever approach (trivial but not now).
+
+local function setuppersfcodes(v,n)
+    if sfstate ~= "unset" then
+        report_defining("setting uppercase sf codes to %s",n)
+        for code, chr in next, data do
+            if chr.category == "lu" then
+                texsetsfcode(code,n)
+            end
+        end
+    end
+    sfstate = v
 end
 
+directives.register("characters.spaceafteruppercase",function(v)
+    if v == "traditional" then
+        setuppersfcodes(v,999)
+    elseif v == "normal" then
+        setuppersfcodes(v,1000)
+    end
+end)
+
 --[[ldx--
 <p>Next comes a whole series of helper methods. These are (will be) part
 of the official <l n='api'/>.</p>
diff --git a/tex/context/base/cldf-ini.lua b/tex/context/base/cldf-ini.lua
index c943dcb51..e3b7ed82a 100644
--- a/tex/context/base/cldf-ini.lua
+++ b/tex/context/base/cldf-ini.lua
@@ -20,6 +20,9 @@ if not modules then modules = { } end modules ['cldf-ini'] = {
 --
 -- tex.print == line with endlinechar appended
 
+-- todo: context("%bold{total: }%s",total)
+-- todo: context.documentvariable("title")
+
 local tex = tex
 
 context       = context or { }
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index 6fbd38de4..02c90c61b 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.02.05 22:32}
+\newcontextversion{2013.02.13 18:06}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index e5938b11a..c9781dc1b 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.02.05 22:32}
+\newcontextversion{2013.02.13 18:06}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 81b9897d6..efe94e504 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index ef320e323..1aaafe23f 100644
--- a/tex/context/base/context-version.png
+++ b/tex/context/base/context-version.png
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 355dd2c8a..26687ca61 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.02.05 22:32}
+\edef\contextversion{2013.02.13 18:06}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 3a5f88110..9782937a5 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.02.05 22:32}
+\edef\contextversion{2013.02.13 18:06}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/file-mod.lua b/tex/context/base/file-mod.lua
index 4a520e998..cfc60cfb7 100644
--- a/tex/context/base/file-mod.lua
+++ b/tex/context/base/file-mod.lua
@@ -28,6 +28,7 @@ commands             = commands or { }
 local commands       = commands
 
 local findbyscheme   = resolvers.finders.byscheme -- use different one
+local iterator       = utilities.parsers.iterator
 
 -- modules can have a specific suffix or can specify one
 
@@ -168,3 +169,9 @@ function commands.doifolderversionelse(one,two) -- one >= two
     two = (two[1] or 0) * 10000 + (two[2] or 0) * 100 + (two[3] or 0)
     commands.doifelse(one>=two)
 end
+
+function commands.useluamodule(list)
+    for filename in iterator(list) do
+        environment.loadluafile(filename)
+    end
+end
diff --git a/tex/context/base/file-mod.mkvi b/tex/context/base/file-mod.mkvi
index 33d100402..92b8a8959 100644
--- a/tex/context/base/file-mod.mkvi
+++ b/tex/context/base/file-mod.mkvi
@@ -31,10 +31,11 @@
 %D their dedicated manuals. We use \type {\next} so that we
 %D can \type {\end} in modules.
 
-\def\usemodules
+\unexpanded\def\usemodules
   {\dotripleempty\strc_modules_use}
 
-\let\usemodule\usemodules
+\let\usemodule   \usemodules
+\let\usetexmodule\usemodules
 
 % \def\strc_modules_use[#category][#name][#parameters]%
 %   {\pushmacro\currentmodule
@@ -247,4 +248,8 @@
 % \setupexternalresources
 %   [url=]
 
+% new:
+
+\unexpanded\def\useluamodule[#1]{\ctxcommand{useluamodule("#1")}}
+
 \protect \endinput
diff --git a/tex/context/base/font-aux.lua b/tex/context/base/font-aux.lua
index 5e938ceb9..2a605d224 100644
--- a/tex/context/base/font-aux.lua
+++ b/tex/context/base/font-aux.lua
@@ -7,7 +7,7 @@ if not modules then modules = { } end modules ['font-aux'] = {
 }
 
 local tonumber, type = tonumber, type
-local wrap, yield = coroutine.wrap, coroutine.yield
+----- wrap, yield = coroutine.wrap, coroutine.yield
 
 local fonts, font = fonts, font
 
@@ -18,9 +18,11 @@ local currentfont = font.current
 local identifiers = fonts.hashes.identifiers
 local sortedkeys  = table.sortedkeys
 
--- for unicode, character   in fonts.iterators.characters  () do print(k,v) end
--- for unicode, description in fonts.iterators.descriptions() do print(k,v) end
--- for index,   glyph       in fonts.iterators.glyphs      () do print(k,v) end
+-- for unicode, character   in fonts.iterators.characters  () do print(unicode) end
+-- for unicode, description in fonts.iterators.descriptions() do print(unicode) end
+-- for index,   glyph       in fonts.iterators.glyphs      () do print(index  ) end
+
+local function dummy() end
 
 local function checkeddata(data) -- beware, nullfont is the fallback in identifiers
     local t = type(data)
@@ -32,22 +34,93 @@ local function checkeddata(data) -- beware, nullfont is the fallback in identifi
     return identifiers[data] -- has nullfont as fallback
 end
 
+local function getindices(data)
+    data = checkeddata(data)
+    local indices = { }
+    local characters = data.characters
+    if characters then
+        for unicode, character in next, characters do
+            indices[character.index or unicode] = unicode
+        end
+    end
+    return indices
+end
+
+-- function iterators.characters(data)
+--     data = checkeddata(data)
+--     local characters = data.characters
+--     if characters then
+--         local collected = sortedkeys(characters)
+--         return wrap(function()
+--             for c=1,#collected do
+--                 local cc = collected[c]
+--                 local dc = characters[cc]
+--                 if dc then
+--                     yield(cc,dc)
+--                 end
+--             end
+--         end)
+--     else
+--         return wrap(function() end)
+--     end
+-- end
+
+-- function iterators.descriptions(data)
+--     data = checkeddata(data)
+--     local characters = data.characters
+--     local descriptions = data.descriptions
+--     if characters and descriptions then
+--         local collected = sortedkeys(characters)
+--         return wrap(function()
+--             for c=1,#collected do
+--                 local cc = collected[c]
+--                 local dc = descriptions[cc]
+--                 if dc then
+--                     yield(cc,dc)
+--                 end
+--             end
+--         end)
+--     else
+--         return wrap(function() end)
+--     end
+-- end
+
+-- function iterators.glyphs(data)
+--     data = checkeddata(data)
+--     local descriptions = data.descriptions
+--     if descriptions then
+--         local indices = getindices(data)
+--         local collected = sortedkeys(indices)
+--         return wrap(function()
+--             for c=1,#collected do
+--                 local cc = collected[c]
+--                 local dc = descriptions[indices[cc]]
+--                 if dc then
+--                     yield(cc,dc)
+--                 end
+--             end
+--         end)
+--     else
+--         return wrap(function() end)
+--     end
+-- end
+
 function iterators.characters(data)
     data = checkeddata(data)
     local characters = data.characters
     if characters then
-        local collected  = sortedkeys(characters)
-        return wrap(function()
-            for c=1,#collected do
-                local cc = collected[c]
+        local collected = sortedkeys(characters)
+        local n, i = #collected, 0
+        return function()
+            i = i + 1
+            if i <= n then
+                local cc = collected[i]
                 local dc = characters[cc]
-                if dc then
-                    yield(cc,dc)
-                end
+                return cc, dc or { }
             end
-        end)
+        end
     else
-        return wrap(function() end)
+        return dummy
     end
 end
 
@@ -57,30 +130,18 @@ function iterators.descriptions(data)
     local descriptions = data.descriptions
     if characters and descriptions then
         local collected = sortedkeys(characters)
-        return wrap(function()
-            for c=1,#collected do
-                local cc = collected[c]
+        local n, i = #collected, 0
+        return function()
+            i = i + 1
+            if i <= n then
+                local cc = collected[i]
                 local dc = descriptions[cc]
-                if dc then
-                    yield(cc,dc)
-                end
+                return cc, dc or { }
             end
-        end)
-    else
-        return wrap(function() end)
-    end
-end
-
-local function getindices(data)
-    data = checkeddata(data)
-    local indices = { }
-    local characters = data.characters
-    if characters then
-        for unicode, character in next, characters do
-            indices[character.index or unicode] = unicode
         end
+    else
+        return dummy
     end
-    return indices
 end
 
 function iterators.glyphs(data)
@@ -89,16 +150,16 @@ function iterators.glyphs(data)
     if descriptions then
         local indices = getindices(data)
         local collected = sortedkeys(indices)
-        return wrap(function()
-            for c=1,#collected do
-                local cc = collected[c]
+        local n, i = #collected, 0
+        return function()
+            i = i + 1
+            if i <= n then
+                local cc = collected[i]
                 local dc = descriptions[indices[cc]]
-                if dc then
-                    yield(cc,dc)
-                end
+                return cc, dc or { }
             end
-        end)
+        end
     else
-        return wrap(function() end)
+        return dummy
     end
 end
diff --git a/tex/context/base/grph-inc.lua b/tex/context/base/grph-inc.lua
index 4006dfdbe..ef03f24e0 100644
--- a/tex/context/base/grph-inc.lua
+++ b/tex/context/base/grph-inc.lua
@@ -659,6 +659,10 @@ end
 
 local resolve_too = false -- true
 
+local internalschemes = {
+    file = true,
+}
+
 local function locate(request) -- name, format, cache
     -- not resolvers.cleanpath(request.name) as it fails on a!b.pdf and b~c.pdf
     -- todo: more restricted cleanpath
@@ -680,14 +684,14 @@ local function locate(request) -- name, format, cache
     local hashed = url.hashed(askedname)
     if not hashed then
         -- go on
-    elseif hashed.scheme == "file" then
+    elseif internalschemes[hashed.scheme] then
         local path = hashed.path
         if path and path ~= "" then
             askedname = path
         end
     else
         local foundname = resolvers.findbinfile(askedname)
-        if not lfs.isfile(foundname) then -- foundname can be dummy
+        if not foundname or not lfs.isfile(foundname) then -- foundname can be dummy
             if trace_figures then
                 report_inclusion("strategy: unresolved url: %s",askedname)
             end
@@ -1161,9 +1165,11 @@ includers.mov = includers.nongeneric
 
 -- -- -- mps -- -- --
 
+internalschemes.mprun = true
+
 local function internal(askedname)
-    local spec, mprun, mpnum = match(lower(askedname),"mprun(:?)(.-)%.(%d+)")
-    if spec == ":" then
+    local spec, mprun, mpnum = match(lower(askedname),"mprun([:%.]?)(.-)%.(%d+)")
+    if spec ~= "" then
         return mprun, mpnum
     else
         return "", mpnum
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index d934df8e1..9a08647f9 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -117,8 +117,11 @@ patterns.utfoffset     = utfoffset
 local utf8char         = patterns.utf8one + patterns.utf8two + patterns.utf8three + patterns.utf8four
 local validutf8char    = utf8char^0 * endofstring * Cc(true) + Cc(false)
 
+local utf8character    = P(1) * R("\128\191")^0 -- unchecked but fast
+
 patterns.utf8          = utf8char
 patterns.utf8char      = utf8char
+patterns.utf8character = utf8character -- this one can be used in most cases so we might use that one
 patterns.validutf8     = validutf8char
 patterns.validutf8char = validutf8char
 
diff --git a/tex/context/base/l-os.lua b/tex/context/base/l-os.lua
index 1e7c91757..08d0473bc 100644
--- a/tex/context/base/l-os.lua
+++ b/tex/context/base/l-os.lua
@@ -193,21 +193,10 @@ os.resolvers = os.resolvers or { } -- will become private
 
 local resolvers = os.resolvers
 
-local osmt = getmetatable(os) or { __index = function(t,k)
-    local v = function()
-        print(format("function os.%s in namespace is undefined"))
-    end
-    t[k] = v
-    return v
-end } -- maybe nil
-
-local osix = osmt.__index
-
-osmt.__index = function(t,k)
-    return (resolvers[k] or osix)(t,k)
-end
-
-setmetatable(os,osmt)
+setmetatable(os, { __index = function(t,k)
+    local r = resolvers[k]
+    return r and r(t,k) or nil -- no memoize
+end })
 
 -- we can use HOSTTYPE on some platforms
 
diff --git a/tex/context/base/l-unicode.lua b/tex/context/base/l-unicode.lua
index de835a7d8..813ffd54b 100644
--- a/tex/context/base/l-unicode.lua
+++ b/tex/context/base/l-unicode.lua
@@ -27,7 +27,7 @@ utf.values     = utf.values     or string.utfvalues
 local type = type
 local char, byte, format, sub = string.char, string.byte, string.format, string.sub
 local concat = table.concat
-local P, C, R, Cs, Ct, Cmt, Cc, Carg = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Carg
+local P, C, R, Cs, Ct, Cmt, Cc, Carg, Cp = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Carg, lpeg.Cp
 local lpegmatch, patterns = lpeg.match, lpeg.patterns
 
 local bytepairs     = string.bytepairs
@@ -131,32 +131,31 @@ function utf.toentities(str)
     return lpegmatch(toentities,str)
 end
 
---~ local utfchr = { } -- 60K -> 2.638 M extra mem but currently not called that often (on latin)
---~
---~ setmetatable(utfchr, { __index = function(t,k) local v = utfchar(k) t[k] = v return v end } )
---~
---~ collectgarbage("collect")
---~ local u = collectgarbage("count")*1024
---~ local t = os.clock()
---~ for i=1,1000 do
---~     for i=1,600 do
---~         local a = utfchr[i]
---~     end
---~ end
---~ print(os.clock()-t,collectgarbage("count")*1024-u)
-
---~ collectgarbage("collect")
---~ local t = os.clock()
---~ for i=1,1000 do
---~     for i=1,600 do
---~         local a = utfchar(i)
---~     end
---~ end
---~ print(os.clock()-t,collectgarbage("count")*1024-u)
-
---~ local byte = string.byte
---~ local utfchar = utf.char
---~ local lpegmatch = lpeg.match, lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+-- local utfchr = { } -- 60K -> 2.638 M extra mem but currently not called that often (on latin)
+--
+-- setmetatable(utfchr, { __index = function(t,k) local v = utfchar(k) t[k] = v return v end } )
+--
+-- collectgarbage("collect")
+-- local u = collectgarbage("count")*1024
+-- local t = os.clock()
+-- for i=1,1000 do
+--     for i=1,600 do
+--         local a = utfchr[i]
+--     end
+-- end
+-- print(os.clock()-t,collectgarbage("count")*1024-u)
+
+-- collectgarbage("collect")
+-- local t = os.clock()
+-- for i=1,1000 do
+--     for i=1,600 do
+--         local a = utfchar(i)
+--     end
+-- end
+-- print(os.clock()-t,collectgarbage("count")*1024-u)
+
+-- local byte = string.byte
+-- local utfchar = utf.char
 
 local one  = P(1)
 local two  = C(1) * C(1)
@@ -870,25 +869,74 @@ if not utf.values then
     -- So, a logical next step is to check for the values variant. It over five times
     -- slower than the built-in string.utfvalues. I optimized it a bit for n=0,1.
 
-    local wrap, yield, gmatch = coroutine.wrap, coroutine.yield, string.gmatch
+    ----- wrap, yield, gmatch = coroutine.wrap, coroutine.yield, string.gmatch
+    local find =  string.find
 
     local dummy = function()
         -- we share this one
     end
 
+    -- function utf.values(str)
+    --     local n = #str
+    --     if n == 0 then
+    --         return wrap(dummy)
+    --     elseif n == 1 then
+    --         return wrap(function() yield(utfbyte(str)) end)
+    --     else
+    --         return wrap(function() for s in gmatch(str,".[\128-\191]*") do
+    --             yield(utfbyte(s))
+    --         end end)
+    --     end
+    -- end
+    --
+    -- faster:
+
     function utf.values(str)
         local n = #str
         if n == 0 then
-            return wrap(dummy)
+            return dummy
         elseif n == 1 then
-            return wrap(function() yield(utfbyte(str)) end)
+            return function() return utfbyte(str) end
         else
-            return wrap(function() for s in gmatch(str,".[\128-\191]*") do
-                yield(utfbyte(s))
-            end end)
+            local p = 1
+         -- local n = #str
+            return function()
+             -- if p <= n then -- slower than the last find
+                    local b, e = find(str,".[\128-\191]*",p)
+                    if b then
+                        p = e + 1
+                        return utfbyte(sub(str,b,e))
+                    end
+             -- end
+            end
         end
     end
 
+    -- slower:
+    --
+    -- local pattern = C(patterns.utf8character) * Cp()
+    -- ----- pattern = patterns.utf8character/utfbyte * Cp()
+    -- ----- pattern = patterns.utf8byte * Cp()
+    --
+    -- function utf.values(str) -- one of the cases where a find is faster than an lpeg
+    --     local n = #str
+    --     if n == 0 then
+    --         return dummy
+    --     elseif n == 1 then
+    --         return function() return utfbyte(str) end
+    --     else
+    --         local p = 1
+    --         return function()
+    --             local s, e = lpegmatch(pattern,str,p)
+    --             if e then
+    --                 p = e
+    --                 return utfbyte(s)
+    --              -- return s
+    --             end
+    --         end
+    --     end
+    -- end
+
     string.utfvalues = utf.values
 
 end
diff --git a/tex/context/base/luat-cnf.lua b/tex/context/base/luat-cnf.lua
index e574bf428..3672c603e 100644
--- a/tex/context/base/luat-cnf.lua
+++ b/tex/context/base/luat-cnf.lua
@@ -152,6 +152,13 @@ function texconfig.init()
     storage.noftables , storage.toftables  = init(0)
     storage.nofmodules, storage.tofmodules = init(%s)
 
+    if modules then
+        local loaded = package.loaded
+        for module, _ in next, modules do
+            loaded[module] = true
+        end
+    end
+
 end
 
 -- we provide a qualified path
diff --git a/tex/context/base/luat-env.lua b/tex/context/base/luat-env.lua
index efaaebf3e..058e161d5 100644
--- a/tex/context/base/luat-env.lua
+++ b/tex/context/base/luat-env.lua
@@ -11,6 +11,8 @@
 -- sense. Much of this evolved before bytecode arrays were available and so a lot of
 -- code has disappeared already.
 
+local rawset, rawget, loadfile, assert = rawset, rawget, loadfile, assert
+
 local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v end)
 
 local report_lua = logs.reporter("resolvers","lua")
@@ -122,6 +124,7 @@ function environment.loadluafile(filename, version)
         if trace_locating then
             report_lua("loading %s", fullname)
         end
+        -- maybe: package.loaded[file.nameonly(fullname)] = true
         chunk = loadfile(fullname) -- this way we don't need a file exists check
     end
     if chunk then
diff --git a/tex/context/base/lxml-lpt.lua b/tex/context/base/lxml-lpt.lua
index 875b6bb96..d73b87287 100644
--- a/tex/context/base/lxml-lpt.lua
+++ b/tex/context/base/lxml-lpt.lua
@@ -1346,34 +1346,89 @@ end
 </typing>
 --ldx]]--
 
-local wrap, yield = coroutine.wrap, coroutine.yield
+-- local wrap, yield = coroutine.wrap, coroutine.yield
+-- local dummy = function() end
+--
+-- function xml.elements(root,pattern,reverse) -- r, d, k
+--     local collected = applylpath(root,pattern)
+--     if collected then
+--         if reverse then
+--             return wrap(function() for c=#collected,1,-1 do
+--                 local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni)
+--             end end)
+--         else
+--             return wrap(function() for c=1,#collected    do
+--                 local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni)
+--             end end)
+--         end
+--     end
+--     return wrap(dummy)
+-- end
+--
+-- function xml.collected(root,pattern,reverse) -- e
+--     local collected = applylpath(root,pattern)
+--     if collected then
+--         if reverse then
+--             return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end)
+--         else
+--             return wrap(function() for c=1,#collected    do yield(collected[c]) end end)
+--         end
+--     end
+--     return wrap(dummy)
+-- end
+
+-- faster:
+
+local dummy = function() end
 
 function xml.elements(root,pattern,reverse) -- r, d, k
     local collected = applylpath(root,pattern)
-    if collected then
-        if reverse then
-            return wrap(function() for c=#collected,1,-1 do
-                local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni)
-            end end)
-        else
-            return wrap(function() for c=1,#collected    do
-                local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni)
-            end end)
+    if not collected then
+        return dummy
+    elseif reverse then
+        local c = #collected + 1
+        return function()
+            if c > 1 then
+                c = c - 1
+                local e = collected[c]
+                local r = e.__p__
+                return r, r.dt, e.ni
+            end
+        end
+    else
+        local n, c = #collected, 0
+        return function()
+            if c < n then
+                c = c + 1
+                local e = collected[c]
+                local r = e.__p__
+                return r, r.dt, e.ni
+            end
         end
     end
-    return wrap(function() end)
 end
 
 function xml.collected(root,pattern,reverse) -- e
     local collected = applylpath(root,pattern)
-    if collected then
-        if reverse then
-            return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end)
-        else
-            return wrap(function() for c=1,#collected    do yield(collected[c]) end end)
+    if not collected then
+        return dummy
+    elseif reverse then
+        local c = #collected + 1
+        return function()
+            if c > 1 then
+                c = c - 1
+                return collected[c]
+            end
+        end
+    else
+        local n, c = #collected, 0
+        return function()
+            if c < n then
+                c = c + 1
+                return collected[c]
+            end
         end
     end
-    return wrap(function() end)
 end
 
 -- handy
diff --git a/tex/context/base/mult-low.lua b/tex/context/base/mult-low.lua
index bdce5af7e..edb6c7376 100644
--- a/tex/context/base/mult-low.lua
+++ b/tex/context/base/mult-low.lua
@@ -96,7 +96,7 @@ return {
         "startproduct", "stopproduct", "product",
         "startproject", "stopproject", "project",
         "starttext", "stoptext", "startnotext", "stopnotext","startdocument", "stopdocument", "documentvariable",
-        "startmodule", "stopmodule", "usemodule",
+        "startmodule", "stopmodule", "usemodule", "usetexmodule", "useluamodule",
         --
         "startTEXpage", "stopTEXpage",
     --  "startMPpage", "stopMPpage", -- already catched by nested lexer
diff --git a/tex/context/base/s-pre-61.tex b/tex/context/base/s-pre-61.tex
index ea221fbcb..48b9d09a0 100644
--- a/tex/context/base/s-pre-61.tex
+++ b/tex/context/base/s-pre-61.tex
@@ -156,6 +156,7 @@
     \NormalizeFontHeight \SubTitleFont    {\setstrut\strut\quad} {1.5\headerheight} {SansBold}
     \NormalizeFontHeight \SubSubTitleFont {\setstrut\strut\quad} {1.0\headerheight} {SansBold}
 \to \everystarttext
+
 \appendtoks
     \NormalizeFontHeight \HeadFont        {\setstrut\strut\quad} {1.0\headerheight} {SansBold}
     \NormalizeFontHeight \TitleFont       {\setstrut\strut\quad} {2.0\headerheight} {SansBold}
diff --git a/tex/context/base/spac-hor.mkiv b/tex/context/base/spac-hor.mkiv
index 42661010b..2ce502fb5 100644
--- a/tex/context/base/spac-hor.mkiv
+++ b/tex/context/base/spac-hor.mkiv
@@ -348,6 +348,23 @@
 \def\spac_spacecodes_setup_nop
   {\updateraggedskips}
 
+%D Here's a tweak .. if needed one can configure it in the configuration
+%D so that initialization happens more efficient.
+%D
+%D \starttyping
+%D \startoverlay
+%D     {
+%D         \green
+%D         \enabledirectives[characters.spaceafteruppercase=normal]%
+%D         \vbox{\hsize 5em x. X\par x.\ X\par X. X\par X.\ X\par}
+%D     } {
+%D         \blue
+%D         \enabledirectives[characters.spaceafteruppercase=traditional]%
+%D         \vbox{\hsize 5em x. X\par x.\ X\par X. X\par X.\ X\par}
+%D     }
+%D \stopoverlay
+%D \stoptyping
+
 % This is not needed, as \updateraggedskips is taking care of it:
 
 \let\synchronizespacecodes\spac_spacecodes_setup_nop % \relax
diff --git a/tex/context/base/spac-ver.mkiv b/tex/context/base/spac-ver.mkiv
index f407a66d8..667b3d00c 100644
--- a/tex/context/base/spac-ver.mkiv
+++ b/tex/context/base/spac-ver.mkiv
@@ -305,7 +305,7 @@
    \prevdepth\d_spac_prevdepth}
 
 \def\spac_helpers_hglue_indeed
-  {\dontleavehmode % \leavevmode
+  {\dontleavehmode
    \c_spac_spacefactor\spacefactor
    \vrule\s!width\zeropoint
    \nobreak
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 148b8568f..4bf36f0f7 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 470c46144..66c3581c1 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/util-mrg.lua b/tex/context/base/util-mrg.lua
index 276531d79..f8742151c 100644
--- a/tex/context/base/util-mrg.lua
+++ b/tex/context/base/util-mrg.lua
@@ -51,6 +51,8 @@ local m_report = [[
 -- stripped bytes    : %s
 ]]
 
+local m_preloaded = [[package.loaded[%q] = package.loaded[%q] or true]]
+
 local function self_fake()
     return m_faked
 end
@@ -175,11 +177,13 @@ local function self_libs(libs,list)
             local fullname = foundpath .. "/" .. lib
             if lfs.isfile(fullname) then
                 utilities.report("merge: using library %s",fullname)
+                local preloaded = file.nameonly(lib)
                 local data = io.loaddata(fullname,true)
                 original = original + #data
                 local data, delta = self_compact(data)
                 right[#right+1] = lib
                 result[#result+1] = m_begin_closure
+                result[#result+1] = format(m_preloaded,preloaded,preloaded)
                 result[#result+1] = data
                 result[#result+1] = m_end_closure
                 stripped = stripped + delta
diff --git a/tex/context/base/util-prs.lua b/tex/context/base/util-prs.lua
index 6f16af666..69d70fbc9 100644
--- a/tex/context/base/util-prs.lua
+++ b/tex/context/base/util-prs.lua
@@ -8,14 +8,15 @@ if not modules then modules = { } end modules ['util-prs'] = {
 
 local lpeg, table, string = lpeg, table, string
 local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp
-local lpegmatch, patterns = lpeg.match, lpeg.patterns
+local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
 local concat, format, gmatch, find = table.concat, string.format, string.gmatch, string.find
 local tostring, type, next, rawset = tostring, type, next, rawset
 
 utilities         = utilities or {}
-utilities.parsers = utilities.parsers or { }
-local parsers     = utilities.parsers
-parsers.patterns  = parsers.patterns or { }
+local parsers     = utilities.parsers or { }
+utilities.parsers = parsers
+local patterns    = parsers.patterns or { }
+parsers.patterns  = patterns
 
 local setmetatableindex = table.setmetatableindex
 local sortedhash        = table.sortedhash
@@ -32,11 +33,11 @@ local lparent     = P("(")
 local rparent     = P(")")
 local period      = S(".")
 local punctuation = S(".,:;")
-local spacer      = patterns.spacer
-local whitespace  = patterns.whitespace
-local newline     = patterns.newline
-local anything    = patterns.anything
-local endofstring = patterns.endofstring
+local spacer      = lpegpatterns.spacer
+local whitespace  = lpegpatterns.whitespace
+local newline     = lpegpatterns.newline
+local anything    = lpegpatterns.anything
+local endofstring = lpegpatterns.endofstring
 
 local nobrace     = 1 - ( lbrace  + rbrace )
 local noparent    = 1 - ( lparent + rparent)
@@ -45,7 +46,7 @@ local noparent    = 1 - ( lparent + rparent)
 
 local escape, left, right = P("\\"), P('{'), P('}')
 
-patterns.balanced = P {
+lpegpatterns.balanced = P {
     [1] = ((escape * (left+right)) + (1 - (left+right)) + V(2))^0,
     [2] = left * V(1) * right
 }
@@ -56,11 +57,11 @@ local spaces        = space^0
 local argument      = Cs((lbrace/"") * ((nobrace + nestedbraces)^0) * (rbrace/""))
 local content       = (1-endofstring)^0
 
-patterns.nestedbraces  = nestedbraces  -- no capture
-patterns.nestedparents = nestedparents -- no capture
-patterns.nested        = nestedbraces  -- no capture
-patterns.argument      = argument      -- argument after e.g. =
-patterns.content       = content       -- rest after e.g =
+lpegpatterns.nestedbraces  = nestedbraces  -- no capture
+lpegpatterns.nestedparents = nestedparents -- no capture
+lpegpatterns.nested        = nestedbraces  -- no capture
+lpegpatterns.argument      = argument      -- argument after e.g. =
+lpegpatterns.content       = content       -- rest after e.g =
 
 local value     = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) + C((nestedbraces + (1-comma))^0)
 
@@ -87,9 +88,9 @@ local pattern_a_s = (pattern_a/set)^1
 local pattern_b_s = (pattern_b/set)^1
 local pattern_c_s = (pattern_c/set)^1
 
-parsers.patterns.settings_to_hash_a = pattern_a_s
-parsers.patterns.settings_to_hash_b = pattern_b_s
-parsers.patterns.settings_to_hash_c = pattern_c_s
+patterns.settings_to_hash_a = pattern_a_s
+patterns.settings_to_hash_b = pattern_b_s
+patterns.settings_to_hash_c = pattern_c_s
 
 function parsers.make_settings_to_hash_pattern(set,how)
     if how == "strict" then
@@ -138,7 +139,7 @@ local pattern   = spaces * Ct(value*(separator*value)^0)
 
 -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored
 
-parsers.patterns.settings_to_array = pattern
+patterns.settings_to_array = pattern
 
 -- we could use a weak table as cache
 
@@ -311,7 +312,7 @@ local pattern = Cs { "start",
     three    = V("thousand") * V("rest"),
 }
 
-patterns.splitthousands = pattern -- maybe better in the parsers namespace ?
+lpegpatterns.splitthousands = pattern -- maybe better in the parsers namespace ?
 
 function parsers.splitthousands(str)
     return lpegmatch(pattern,str) or str
@@ -321,14 +322,14 @@ end
 
 local optionalwhitespace = whitespace^0
 
-patterns.words      = Ct((Cs((1-punctuation-whitespace)^1) + anything)^1)
-patterns.sentences  = Ct((optionalwhitespace * Cs((1-period)^0 * period))^1)
-patterns.paragraphs = Ct((optionalwhitespace * Cs((whitespace^1*endofstring/"" + 1 - (spacer^0*newline*newline))^1))^1)
+lpegpatterns.words      = Ct((Cs((1-punctuation-whitespace)^1) + anything)^1)
+lpegpatterns.sentences  = Ct((optionalwhitespace * Cs((1-period)^0 * period))^1)
+lpegpatterns.paragraphs = Ct((optionalwhitespace * Cs((whitespace^1*endofstring/"" + 1 - (spacer^0*newline*newline))^1))^1)
 
 -- local str = " Word1 word2. \n Word3 word4. \n\n Word5 word6.\n "
--- inspect(lpegmatch(patterns.paragraphs,str))
--- inspect(lpegmatch(patterns.sentences,str))
--- inspect(lpegmatch(patterns.words,str))
+-- inspect(lpegmatch(lpegpatterns.paragraphs,str))
+-- inspect(lpegmatch(lpegpatterns.sentences,str))
+-- inspect(lpegmatch(lpegpatterns.words,str))
 
 -- handy for k="v" [, ] k="v"
 
@@ -342,7 +343,7 @@ local value     = dquote * C((1-dquote-escape*dquote)^0) * dquote
 
 local pattern   = Cf(Ct("") * Cg(key * equal * value) * separator^0,rawset)^0
 
-parsers.patterns.keq_to_hash_c = pattern
+patterns.keq_to_hash_c = pattern
 
 function parsers.keq_to_hash(str)
     if str and str ~= "" then
@@ -446,9 +447,9 @@ local function ranger(first,last,n,action)
     end
 end
 
-local cardinal    = patterns.cardinal / tonumber
-local spacers     = patterns.spacer^0
-local endofstring = patterns.endofstring
+local cardinal    = lpegpatterns.cardinal / tonumber
+local spacers     = lpegpatterns.spacer^0
+local endofstring = lpegpatterns.endofstring
 
 local stepper  = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + Cc(true) ) + Cc(false) )
                * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1
@@ -456,10 +457,65 @@ local stepper  = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(c
 local stepper  = spacers * ( C(cardinal) * ( spacers * S(":-") * spacers * ( C(cardinal) + (P("*") + endofstring) * Cc(true) ) + Cc(false) )
                * Carg(1) * Carg(2) / ranger * S(", ")^0 )^1 * endofstring -- we're sort of strict (could do without endofstring)
 
-function utilities.parsers.stepper(str,n,action)
+function parsers.stepper(str,n,action)
     if type(n) == "function" then
         lpegmatch(stepper,str,1,false,n or print)
     else
         lpegmatch(stepper,str,1,n,action or print)
     end
 end
+
+--
+
+local pattern = Cs((P("%")/"\\percent " + P("^") * Cc("{") * lpegpatterns.integer * Cc("}") + P(1))^0)
+
+patterns.unittotex = pattern
+
+function parsers.unittotex(str)
+    return lpegmatch(pattern,str)
+end
+
+local pattern = Cs((P("^") / "<sup>" * lpegpatterns.integer * Cc("</sup>") + P(1))^0)
+
+function parsers.unittoxml(str)
+    return lpegmatch(pattern,str)
+end
+
+-- print(utilities.parsers.unittotex("10^-32 %"),utilities.parsers.unittoxml("10^32 %"))
+
+local cache   = { }
+local spaces  = lpeg.patterns.space^0
+local dummy   = function() end
+
+table.setmetatableindex(cache,function(t,k)
+    local separator = P(k)
+    local value     = (1-separator)^0
+    local pattern   = spaces * C(value) * separator^0 * Cp()
+    t[k] = pattern
+    return pattern
+end)
+
+local commalistiterator = cache[","]
+
+function utilities.parsers.iterator(str,separator)
+    local n = #str
+    if n == 0 then
+        return dummy
+    else
+        local pattern = separator and cache[separator] or commalistiterator
+        local p = 1
+        return function()
+            if p <= n then
+                local s, e = lpegmatch(pattern,str,p)
+                if e then
+                    p = e
+                    return s
+                end
+            end
+        end
+    end
+end
+
+-- for s in utilities.parsers.iterator("a b c,b,c") do
+--     print(s)
+-- end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 1d56ffd16..6b4b8709a 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 02/05/13 22:32:22
+-- merge date  : 02/13/13 18:06:17
 
 do -- begin closure to overcome local limits and interference
 
@@ -53,8 +53,10 @@ patterns.utftype=utftype
 patterns.utfoffset=utfoffset
 local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four
 local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false)
+local utf8character=P(1)*R("\128\191")^0 
 patterns.utf8=utf8char
 patterns.utf8char=utf8char
+patterns.utf8character=utf8character 
 patterns.validutf8=validutf8char
 patterns.validutf8char=validutf8char
 local eol=S("\n\r")
@@ -7656,15 +7658,18 @@ local function copy_glyph(g)
     return copy_node(g)
   end
 end
-local function markstoligature(kind,lookupname,start,stop,char)
+local function markstoligature(kind,lookupname,head,start,stop,char)
   if start==stop and start.char==char then
-    return start
+    return head,start
   else
     local prev=start.prev
     local next=stop.next
     start.prev=nil
     stop.next=nil
     local base=copy_glyph(start)
+    if head==start then
+      head=base
+    end
     base.char=char
     base.subtype=ligature_code
     base.components=start
@@ -7676,7 +7681,7 @@ local function markstoligature(kind,lookupname,start,stop,char)
     end
     base.next=next
     base.prev=prev
-    return base
+    return head,base
   end
 end
 local function getcomponentindex(start)
@@ -7696,16 +7701,19 @@ local function getcomponentindex(start)
     return 0
   end
 end
-local function toligature(kind,lookupname,start,stop,char,markflag,discfound) 
+local function toligature(kind,lookupname,head,start,stop,char,markflag,discfound) 
   if start==stop and start.char==char then
     start.char=char
-    return start
+    return head,start
   end
   local prev=start.prev
   local next=stop.next
   start.prev=nil
   stop.next=nil
   local base=copy_glyph(start)
+  if start==head then
+    head=base
+  end
   base.char=char
   base.subtype=ligature_code
   base.components=start 
@@ -7752,7 +7760,7 @@ local function toligature(kind,lookupname,start,stop,char,markflag,discfound)
       start=start.next
     end
   end
-  return base
+  return head,base
 end
 function handlers.gsub_single(head,start,kind,lookupname,replacement)
   if trace_singles then
@@ -7866,10 +7874,10 @@ function handlers.gsub_ligature(head,start,kind,lookupname,ligature,sequence)
       if lig then
         if trace_ligatures then
           local stopchar=stop.char
-          start=markstoligature(kind,lookupname,start,stop,lig)
-          logprocess("%s: replacing %s upto %s by ligature %s",pref(kind,lookupname),gref(startchar),gref(stopchar),gref(start.char))
+          head,start=markstoligature(kind,lookupname,start,stop,lig)
+          logprocess("%s: replacing %s upto %s by ligature %s case 1",pref(kind,lookupname),gref(startchar),gref(stopchar),gref(start.char))
         else
-          start=markstoligature(kind,lookupname,start,stop,lig)
+          head,start=markstoligature(kind,lookupname,start,stop,lig)
         end
         return head,start,true
       else
@@ -7909,10 +7917,10 @@ function handlers.gsub_ligature(head,start,kind,lookupname,ligature,sequence)
       if lig then
         if trace_ligatures then
           local stopchar=stop.char
-          start=toligature(kind,lookupname,start,stop,lig,skipmark,discfound)
-          logprocess("%s: replacing %s upto %s by ligature %s",pref(kind,lookupname),gref(startchar),gref(stopchar),gref(start.char))
+          head,start=toligature(kind,lookupname,head,start,stop,lig,skipmark,discfound)
+          logprocess("%s: replacing %s upto %s by ligature %s case 2",pref(kind,lookupname),gref(startchar),gref(stopchar),gref(start.char))
         else
-          start=toligature(kind,lookupname,start,stop,lig,skipmark,discfound)
+          head,start=toligature(kind,lookupname,head,start,stop,lig,skipmark,discfound)
         end
         return head,start,true
       else
@@ -8428,12 +8436,12 @@ function chainprocs.gsub_ligature(head,start,stop,kind,chainname,currentcontext,
         end
         if trace_ligatures then
           if start==stop then
-            logprocess("%s: replacing character %s by ligature %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(l2))
+            logprocess("%s: replacing character %s by ligature %s case 3",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(l2))
           else
-            logprocess("%s: replacing character %s upto %s by ligature %s",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(stop.char),gref(l2))
+            logprocess("%s: replacing character %s upto %s by ligature %s case 4",cref(kind,chainname,chainlookupname,lookupname,chainindex),gref(startchar),gref(stop.char),gref(l2))
           end
         end
-        start=toligature(kind,lookupname,start,stop,l2,currentlookup.flags[1],discfound)
+        head,start=toligature(kind,lookupname,head,start,stop,l2,currentlookup.flags[1],discfound)
         return head,start,true,nofreplacements
       elseif trace_bugs then
         if start==stop then