beta 2012.09.21 20:58

23 files changed, 1840 insertions, 230 deletions

diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii
index 2e90a2dc9..d7edc921c 100644
--- a/tex/context/base/cont-new.mkii
+++ b/tex/context/base/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.09.16 23:18}
+\newcontextversion{2012.09.21 20:58}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index cc6342b05..2f90130c7 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2012.09.16 23:18}
+\newcontextversion{2012.09.21 20:58}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 7db72b855..e232cfeb6 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png
index 98253d101..048d04143 100644
--- a/tex/context/base/context-version.png
+++ b/tex/context/base/context-version.png
diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii
index 2582f1f9f..0aab24aad 100644
--- a/tex/context/base/context.mkii
+++ b/tex/context/base/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.09.16 23:18}
+\edef\contextversion{2012.09.21 20:58}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 52c4d897e..d19d90a93 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -25,7 +25,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2012.09.16 23:18}
+\edef\contextversion{2012.09.21 20:58}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/core-dat.lua b/tex/context/base/core-dat.lua
index 071a3fe0b..879ff6130 100644
--- a/tex/context/base/core-dat.lua
+++ b/tex/context/base/core-dat.lua
@@ -11,7 +11,7 @@ if not modules then modules = { } end modules ['core-dat'] = {
 replaces the twopass data mechanism.</p>
 --ldx]]--
 
-local tonumber = tonumber
+local tonumber, type = tonumber, type
 
 local context, commands = context, commands
 
@@ -69,7 +69,12 @@ local function setdata(settings)
     local tag  = settings.tag
     local data = settings.data
     local list = tobesaved[name]
-    data = settings_to_hash(data) or { }
+    if settings.convert and type(data) == "string" then
+        data = settings_to_hash(data)
+    end
+    if type(data) ~= "table" then
+        data = { data = settings.data }
+    end
     if not tag then
         tag = #list + 1
     else
@@ -109,25 +114,30 @@ end
 
 function datasets.getdata(name,tag,key,default)
     local t = collected[name]
-    if t then
+    if t == nil then
+        if trace_datasets then
+            report_dataset("unknown: name %s",name)
+        end
+    elseif type(t) ~= "table" then
+        return t
+    else
         t = t[tag] or t[tonumber(tag)]
-        if t then
-            if key then
-                return t[key] or default
-            else
-                return t
+        if not t then
+            if trace_datasets then
+                report_dataset("unknown: name %s, tag %s",name,tag)
             end
-        elseif trace_datasets then
-            report_dataset("unknown: name %s, tag %s",name,tag)
+        elseif key then
+            return t[key] or default
+        else
+            return t
         end
-    elseif trace_datasets then
-        report_dataset("unknown: name %s",name)
     end
     return default
 end
 
 function commands.setdataset(settings)
-    local name, tag, data = setdata(settings)
+    settings.convert = true
+    local name, tag = setdata(settings)
     if settings.delay ~= v_yes then
         --
     elseif type(tag) == "number" then
@@ -139,11 +149,25 @@ end
 
 function commands.datasetvariable(name,tag,key)
     local t = collected[name]
-    t = t and (t[tag] or t[tonumber(tag)])
-    if t then
-        local s = t[key]
-        if s then
-            context(s)
+    if t == nil then
+        if trace_datasets then
+            report_dataset("unknown: name %s (not passed to tex)",name)
+        end
+    elseif type(t) ~= "table" then
+        context(tostring(t))
+    else
+        t = t and (t[tag] or t[tonumber(tag)])
+        if not t then
+            if trace_datasets then
+                report_dataset("unknown: name %s with tag %s (not passed to tex)",name,tag)
+            end
+        elseif type(t) ~= "table" then
+            local s = t[key]
+            if type(s) ~= "table" then
+                context(tostring(s))
+            elseif trace_datasets then
+                report_dataset("table: name %s, tag %s (not passed to tex)",name,tag)
+            end
         end
     end
 end
diff --git a/tex/context/base/data-lua.lua b/tex/context/base/data-lua.lua
index ab762f3d4..fec5856ea 100644
--- a/tex/context/base/data-lua.lua
+++ b/tex/context/base/data-lua.lua
@@ -6,9 +6,15 @@ if not modules then modules = { } end modules ['data-lua'] = {
     license   = "see context related readme files"
 }
 
--- some loading stuff ... we might move this one to slot 2 depending
--- on the developments (the loaders must not trigger kpse); we could
--- of course use a more extensive lib path spec
+-- We overload the regular loader. We do so because we operate mostly in
+-- tds and use our own loader code. Alternatively we could use a more
+-- extensive definition of package.path and package.cpath but even then
+-- we're not done. Also, we now have better tracing.
+--
+-- -- local mylib = require("libtest")
+-- -- local mysql = require("luasql.mysql")
+
+local concat = table.concat
 
 local trace_libraries = false
 
@@ -18,160 +24,178 @@ trackers.register("resolvers.locating",  function(v) trace_libraries = v end)
 local report_libraries = logs.reporter("resolvers","libraries")
 
 local gsub, insert = string.gsub, table.insert
+local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match
 local unpack = unpack or table.unpack
+local is_readable = file.is_readable
 
 local resolvers, package = resolvers, package
 
--- local  libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' }
-local  libformats = { 'lua', 'tex' }
-local clibformats = { 'lib' }
+local  libsuffixes = { 'tex', 'lua' }
+local clibsuffixes = { 'lib' }
+local  libformats  = { 'TEXINPUTS', 'LUAINPUTS' }
+local clibformats  = { 'CLUAINPUTS' }
+
+local libpaths   = nil
+local clibpaths  = nil
+local libhash    = { }
+local clibhash   = { }
+local libextras  = { }
+local clibextras = { }
 
-local _path_, libpaths, _cpath_, clibpaths
+local pattern = Cs(P("!")^0 / "" * (P("/") * P(-1) / "/" + P("/")^1 / "/" + 1)^0)
+
+local function cleanpath(path) --hm, don't we have a helper for this?
+    return resolvers.resolve(lpegmatch(pattern,path))
+end
 
-function package.libpaths()
-    if not _path_ or package.path ~= _path_ then
-        _path_ = package.path
-        libpaths = file.splitpath(_path_,";")
+local function getlibpaths()
+    if not libpaths then
+        libpaths = { }
+        for i=1,#libformats do
+            local paths = resolvers.expandedpathlistfromvariable(libformats[i])
+            for i=1,#paths do
+                local path = cleanpath(paths[i])
+                if not libhash[path] then
+                    libpaths[#libpaths+1] = path
+                    libhash[path] = true
+                end
+            end
+        end
     end
     return libpaths
 end
 
-function package.clibpaths()
-    if not _cpath_ or package.cpath ~= _cpath_ then
-        _cpath_ = package.cpath
-        clibpaths = file.splitpath(_cpath_,";")
+local function getclibpaths()
+    if not clibpaths then
+        clibpaths = { }
+        for i=1,#clibformats do
+            local paths = resolvers.expandedpathlistfromvariable(clibformats[i])
+            for i=1,#paths do
+                local path = cleanpath(paths[i])
+                if not clibhash[path] then
+                    clibpaths[#clibpaths+1] = path
+                    clibhash[path] = true
+                end
+            end
+        end
     end
     return clibpaths
 end
 
-local function thepath(...)
-    local t = { ... } t[#t+1] = "?.lua"
-    local path = file.join(unpack(t))
-    if trace_libraries then
-        report_libraries("! appending '%s' to 'package.path'",path)
+package.libpaths  = getlibpaths
+package.clibpaths = getclibpaths
+
+function package.extralibpath(...)
+    local paths = { ... }
+    for i=1,#paths do
+        local path = cleanpath(paths[i])
+        if not libhash[path] then
+            if trace_libraries then
+                report_libraries("! extra lua path '%s'",path)
+            end
+            libextras[#libextras+1] = path
+            libpaths[#libpaths  +1] = path
+        end
     end
-    return path
 end
 
-local p_libpaths, a_libpaths = { }, { }
-
-function package.appendtolibpath(...)
-    insert(a_libpath,thepath(...))
+function package.extraclibpath(...)
+    local paths = { ... }
+    for i=1,#paths do
+        local path = cleanpath(paths[i])
+        if not clibhash[path] then
+            if trace_libraries then
+                report_libraries("! extra lib path '%s'",path)
+            end
+            clibextras[#clibextras+1] = path
+            clibpaths[#clibpaths  +1] = path
+        end
+    end
 end
 
-function package.prependtolibpath(...)
-    insert(p_libpaths,1,thepath(...))
+if not package.loaders[-2] then
+    -- use package-path and package-cpath
+    package.loaders[-2] = package.loaders[2]
 end
 
--- beware, we need to return a loadfile result !
+local function loadedaslib(resolved,rawname)
+    return package.loadlib(resolved,"luaopen_" .. gsub(rawname,"%.","_"))
+end
 
-local function loaded(libpaths,name,simple)
-    for i=1,#libpaths do -- package.path, might become option
-        local libpath = libpaths[i]
-        local resolved = gsub(libpath,"%?",simple)
-        if trace_libraries then -- more detail
-            report_libraries("! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved)
-        end
-        if file.is_readable(resolved) then
-            if trace_libraries then
-                report_libraries("! lib '%s' located via 'package.path': '%s'",name,resolved)
-            end
-            return loadfile(resolved)
-        end
+local function loadedbylua(name)
+    if trace_libraries then
+        report_libraries("! locating %q using normal loader",name)
     end
+    local resolved = package.loaders[-2](name)
 end
 
-package.loaders[2] = function(name) -- was [#package.loaders+1]
-    if file.suffix(name) == "" then
-        name = file.addsuffix(name,"lua") -- maybe a list
-        if trace_libraries then -- mode detail
-            report_libraries("! locating '%s' with forced suffix",name)
-        end
-    else
-        if trace_libraries then -- mode detail
-            report_libraries("! locating '%s'",name)
-        end
+local function loadedbyformat(name,rawname,suffixes,islib)
+    if trace_libraries then
+        report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes))
     end
-    for i=1,#libformats do
-        local format = libformats[i]
+    for i=1,#suffixes do -- so we use findfile and not a lookup loop
+        local format = suffixes[i]
         local resolved = resolvers.findfile(name,format) or ""
-        if trace_libraries then -- mode detail
-            report_libraries("! checking for '%s' using 'libformat path': '%s'",name,format)
+        if trace_libraries then
+            report_libraries("! checking for %q' using format %q",name,format)
         end
         if resolved ~= "" then
             if trace_libraries then
-                report_libraries("! lib '%s' located via environment: '%s'",name,resolved)
-            end
-            return loadfile(resolved)
-        end
-    end
-    -- libpaths
-    local libpaths, clibpaths = package.libpaths(), package.clibpaths()
-    local simple = gsub(name,"%.lua$","")
-    local simple = gsub(simple,"%.","/")
-    local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple)
-    if resolved then
-        return resolved
-    end
-    --
-    local libname = file.addsuffix(simple,os.libsuffix)
-    for i=1,#clibformats do
-        -- better have a dedicated loop
-        local format = clibformats[i]
-        local paths = resolvers.expandedpathlistfromvariable(format)
-        for p=1,#paths do
-            local path = paths[p]
-            local resolved = file.join(path,libname)
-            if trace_libraries then -- mode detail
-                report_libraries("! checking for '%s' using 'clibformat path': '%s'",libname,path)
+                report_libraries("! lib %q located on %q",name,resolved)
             end
-            if file.is_readable(resolved) then
-                if trace_libraries then
-                    report_libraries("! lib '%s' located via 'clibformat': '%s'",libname,resolved)
-                end
-                return package.loadlib(resolved,name)
+            if islib then
+                return loadedaslib(resolved,rawname)
+            else
+                return loadfile(resolved)
             end
         end
     end
-    for i=1,#clibpaths do -- package.path, might become option
-        local libpath = clibpaths[i]
-        local resolved = gsub(libpath,"?",simple)
-        if trace_libraries then -- more detail
-            report_libraries("! checking for '%s' on 'package.cpath': '%s'",simple,libpath)
+end
+
+local function loadedbypath(name,rawname,paths,islib,what)
+    if trace_libraries then
+        report_libraries("! locating %q as %q on %q paths",rawname,name,what)
+    end
+    for p=1,#paths do
+        local path = paths[p]
+        local resolved = file.join(path,name)
+        if trace_libraries then -- mode detail
+            report_libraries("! checking for %q using %q path %q",name,what,path)
         end
-        if file.is_readable(resolved) then
+        if is_readable(resolved) then
             if trace_libraries then
-                report_libraries("! lib '%s' located via 'package.cpath': '%s'",name,resolved)
+                report_libraries("! lib %q located on %q",name,resolved)
+            end
+            if islib then
+                return loadedaslib(resolved,rawname)
+            else
+                return loadfile(resolved)
             end
-            return package.loadlib(resolved,name)
-        end
-    end
-    -- just in case the distribution is messed up
-    if trace_loading then -- more detail
-        report_libraries("! checking for '%s' using 'luatexlibs': '%s'",name)
-    end
-    local resolved = resolvers.findfile(file.basename(name),'luatexlibs') or ""
-    if resolved ~= "" then
-        if trace_libraries then
-            report_libraries("! lib '%s' located by basename via environment: '%s'",name,resolved)
         end
-        return loadfile(resolved)
     end
+end
+
+local function notloaded(name)
     if trace_libraries then
-        report_libraries('? unable to locate lib: %s',name)
+        report_libraries("? unable to locate library %q",name)
     end
---  return "unable to locate " .. name
 end
 
-resolvers.loadlualib = require
-
--- -- -- --
-
-package.obsolete = package.obsolete or { }
-
-package.append_libpath           = appendtolibpath   -- will become obsolete
-package.prepend_libpath          = prependtolibpath  -- will become obsolete
+package.loaders[2] = function(name)
+    local thename = gsub(name,"%.","/")
+    local luaname = file.addsuffix(thename,"lua")
+    local libname = file.addsuffix(thename,os.libsuffix)
+    return
+        loadedbyformat(luaname,name,libsuffixes,   false)
+     or loadedbyformat(libname,name,clibsuffixes,  true)
+     or loadedbypath  (luaname,name,getlibpaths (),false,"lua")
+     or loadedbypath  (luaname,name,getclibpaths(),false,"lua")
+     or loadedbypath  (libname,name,getclibpaths(),true, "lib")
+     or loadedbylua   (name)
+     or notloaded     (name)
+end
 
-package.obsolete.append_libpath  = appendtolibpath   -- will become obsolete
-package.obsolete.prepend_libpath = prependtolibpath  -- will become obsolete
+-- package.loaders[3] = nil
+-- package.loaders[4] = nil
 
+resolvers.loadlualib = require
diff --git a/tex/context/base/data-pre.lua b/tex/context/base/data-pre.lua
index 4e82b6186..214477d87 100644
--- a/tex/context/base/data-pre.lua
+++ b/tex/context/base/data-pre.lua
@@ -121,6 +121,8 @@ function resolvers.resetresolve(str)
     resolved, abstract = { }, { }
 end
 
+-- todo: use an lpeg (see data-lua for !! / stripper)
+
 local function resolve(str) -- use schemes, this one is then for the commandline only
     if type(str) == "table" then
         local t = { }
diff --git a/tex/context/base/data-res.lua b/tex/context/base/data-res.lua
index e51b980ff..9236cbe02 100644
--- a/tex/context/base/data-res.lua
+++ b/tex/context/base/data-res.lua
@@ -805,7 +805,7 @@ function resolvers.expandedpathlist(str)
     end
 end
 
-function resolvers.expandedpathlistfromvariable(str) -- brrr
+function resolvers.expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ //
     str = lpegmatch(dollarstripper,str)
     local tmp = resolvers.variableofformatorsuffix(str)
     return resolvers.expandedpathlist(tmp ~= "" and tmp or str)
diff --git a/tex/context/base/l-os.lua b/tex/context/base/l-os.lua
index 700affdac..799f44957 100644
--- a/tex/context/base/l-os.lua
+++ b/tex/context/base/l-os.lua
@@ -22,18 +22,28 @@ if not modules then modules = { } end modules ['l-os'] = {
 -- os.name     : windows | msdos | linux | macosx | solaris | .. | generic (new)
 -- os.platform : extended os.name with architecture
 
+-- os.sleep() => socket.sleep()
+-- math.randomseed(tonumber(string.sub(string.reverse(tostring(math.floor(socket.gettime()*10000))),1,6)))
+
 -- maybe build io.flush in os.execute
 
 local os = os
-local date = os.date
+local date, time = os.date, os.time
 local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch
 local concat = table.concat
 local random, ceil, randomseed = math.random, math.ceil, math.randomseed
-local rawget, rawset, type, getmetatable, setmetatable, tonumber = rawget, rawset, type, getmetatable, setmetatable, tonumber
+local rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring = rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring
 
 -- The following code permits traversing the environment table, at least
 -- in luatex. Internally all environment names are uppercase.
 
+-- The randomseed in Lua is not that random, although this depends on the operating system as well
+-- as the binary (Luatex is normally okay). But to be sure we set the seed anyway.
+
+math.initialseed = tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6))
+
+randomseed(math.initialseed)
+
 if not os.__getenv__ then
 
     os.__getenv__ = os.getenv
@@ -375,10 +385,41 @@ end
 local timeformat = format("%%s%s",os.timezone(true))
 local dateformat = "!%Y-%m-%d %H:%M:%S"
 
-function os.fulltime(t)
+function os.fulltime(t,default)
+    t = tonumber(t) or 0
+    if t > 0 then
+        -- valid time
+    elseif default then
+        return default
+    else
+        t = nil
+    end
     return format(timeformat,date(dateformat,t))
 end
 
+local dateformat = "%Y-%m-%d %H:%M:%S"
+
+function os.localtime(t,default)
+    t = tonumber(t) or 0
+    if t > 0 then
+        -- valid time
+    elseif default then
+        return default
+    else
+        t = nil
+    end
+    return date(dateformat,t)
+end
+
+function os.converttime(t,default)
+    local t = tonumber(t)
+    if t and t > 0 then
+        return date(dateformat,t)
+    else
+        return default or "-"
+    end
+end
+
 local memory = { }
 
 local function which(filename)
diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua
index 413d31eb5..2b3319e45 100644
--- a/tex/context/base/l-table.lua
+++ b/tex/context/base/l-table.lua
@@ -913,23 +913,27 @@ function table.reversed(t)
     end
 end
 
-function table.sequenced(t,sep,simple) -- hash only
-    local s, n = { }, 0
-    for k, v in sortedhash(t) do
-        if simple then
-            if v == true then
-                n = n + 1
-                s[n] = k
-            elseif v and v~= "" then
+function table.sequenced(t,sep) -- hash only
+    if t then
+        local s, n = { }, 0
+        for k, v in sortedhash(t) do
+            if simple then
+                if v == true then
+                    n = n + 1
+                    s[n] = k
+                elseif v and v~= "" then
+                    n = n + 1
+                    s[n] = k .. "=" .. tostring(v)
+                end
+            else
                 n = n + 1
                 s[n] = k .. "=" .. tostring(v)
             end
-        else
-            n = n + 1
-            s[n] = k .. "=" .. tostring(v)
         end
+        return concat(s, sep or " | ")
+    else
+        return ""
     end
-    return concat(s, sep or " | ")
 end
 
 function table.print(t,...)
diff --git a/tex/context/base/luat-env.lua b/tex/context/base/luat-env.lua
index 4d9a44d42..c8a391e76 100644
--- a/tex/context/base/luat-env.lua
+++ b/tex/context/base/luat-env.lua
@@ -261,7 +261,7 @@ function environment.texfile(filename)
     return resolvers.findfile(filename,'tex')
 end
 
-function environment.luafile(filename)
+function environment.luafile(filename) -- needs checking
     local resolved = resolvers.findfile(filename,'tex') or ""
     if resolved ~= "" then
         return resolved
diff --git a/tex/context/base/m-pstricks.lua b/tex/context/base/m-pstricks.lua
index 7f795feac..b151e313a 100644
--- a/tex/context/base/m-pstricks.lua
+++ b/tex/context/base/m-pstricks.lua
@@ -39,12 +39,12 @@ local template = [[
 \stoptext
 ]]
 
-local modules = { }
+local loaded   = { }
 local graphics = 0
 
 function moduledata.pstricks.usemodule(names)
     for name in gmatch(names,"([^%s,]+)") do
-        modules[#modules+1] = format([[\readfile{%s}{}{}]],name)
+        loaded[#loaded+1] = format([[\readfile{%s}{}{}]],name)
     end
 end
 
@@ -55,10 +55,10 @@ function moduledata.pstricks.process(n)
     local tmpfile = name .. ".tmp"
     local epsfile = name .. ".ps"
     local pdffile = name .. ".pdf"
-    local modules = concat(modules,"\n")
+    local loaded = concat(loaded,"\n")
     os.remove(epsfile)
     os.remove(pdffile)
-    io.savedata(tmpfile,format(template,modules,data))
+    io.savedata(tmpfile,format(template,loaded,data))
     os.execute(format("mtxrun --script texexec %s --once --dvips",tmpfile))
     if lfs.isfile(epsfile) then
         os.execute(format("ps2pdf %s %s",epsfile,pdffile))
diff --git a/tex/context/base/mult-low.lua b/tex/context/base/mult-low.lua
index 7d5078b60..bffdd288f 100644
--- a/tex/context/base/mult-low.lua
+++ b/tex/context/base/mult-low.lua
@@ -271,6 +271,8 @@ return {
         --
         "startnointerference", "stopnointerference",
         --
+        "twodigits","threedigits",
+        --
         "strut", "setstrut", "strutbox", "strutht", "strutdp", "strutwd", "struthtdp", "begstrut", "endstrut", "lineheight",
     }
 }
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index c31ba403b..e5263bffa 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 8ad2ea57b..6a8332f63 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/util-sql-loggers.lua b/tex/context/base/util-sql-loggers.lua
new file mode 100644
index 000000000..b2dccf4b5
--- /dev/null
+++ b/tex/context/base/util-sql-loggers.lua
@@ -0,0 +1,277 @@
+if not modules then modules = { } end modules ['util-sql-loggers'] = {
+    version   = 1.001,
+    comment   = "companion to lmx-*",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This is experimental code and currently part of the base installation simply
+-- because it's easier to dirtribute this way. Eventually it will be documented
+-- and the related scripts will show up as well.
+
+local tonumber = tonumber
+local format = string.format
+local concat = table.concat
+local ostime, uuid, osfulltime = os.time, os.uuid, os.fulltime
+local random = math.random
+
+local sql           = utilities.sql
+local loggers       = { }
+sql.loggers         = loggers
+
+local trace_sql     = false  trackers.register("sql.loggers.trace", function(v) trace_sql = v end)
+local report        = logs.reporter("sql","loggers")
+
+loggers.newtoken    = sql.tokens.new
+local makeconverter = sql.makeconverter
+
+local function checkeddb(presets,datatable)
+    return sql.usedatabase(presets,datatable or presets.datatable or "loggers")
+end
+
+loggers.usedb = checkeddb
+
+local totype = {
+    ["error"]   = 1, [1] = 1, ["1"] = 1,
+    ["warning"] = 2, [2] = 2, ["2"] = 2,
+    ["debug"]   = 3, [3] = 3, ["3"] = 3,
+    ["info"]    = 4, [4] = 4, ["4"] = 4,
+}
+
+local fromtype = {
+    ["error"]   = "error",   [1] = "error",   ["1"] = "error",
+    ["warning"] = "warning", [2] = "warning", ["2"] = "warning",
+    ["debug"]   = "debug",   [3] = "debug",   ["3"] = "debug",
+    ["info"]    = "info",    [4] = "info",    ["4"] = "info",
+}
+
+table.setmetatableindex(totype,  function() return 4      end)
+table.setmetatableindex(fromtype,function() return "info" end)
+
+loggers.totype   = totype
+loggers.fromtype = fromtype
+
+local template =[[
+    CREATE TABLE IF NOT EXISTS %basename% (
+        `id`     int(11) NOT NULL AUTO_INCREMENT,
+        `time`   int(11) NOT NULL,
+        `type`   int(11) NOT NULL,
+        `action` varchar(15) NOT NULL,
+        `data`   longtext,
+        PRIMARY KEY (`id`),
+        UNIQUE KEY `id_unique_key` (`id`)
+    )
+    DEFAULT CHARSET = utf8 ;
+]]
+
+function loggers.createdb(presets,datatable)
+
+    local db = checkeddb(presets,datatable)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    report("datatable %q created in %q",db.name,db.base)
+
+    return db
+
+end
+
+local template =[[
+    DROP TABLE IF EXISTS %basename% ;
+]]
+
+function loggers.deletedb(presets,datatable)
+
+    local db = checkeddb(presets,datatable)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    report("datatable %q removed in %q",db.name,db.base)
+
+end
+
+local template =[[
+    INSERT INTO %basename% (
+        `time`,
+        `type`,
+        `action`,
+        `data`
+    ) VALUES (
+        %time%,
+        %type%,
+        '%action%',
+        '%[data]%'
+    ) ;
+]]
+
+function loggers.save(db,data) -- beware, we pass type and action in the data (saves a table)
+
+    if data then
+
+        local time   = ostime()
+        local kind   = totype[data.type]
+        local action = data.action or "unknown"
+
+        data.type   = nil
+        data.action = nil
+
+        db.execute {
+            template  = template,
+            variables = {
+                basename = db.basename,
+                time     = ostime(),
+                type     = kind,
+                action   = action,
+                data     = data and db.serialize(data,"return") or "",
+            },
+        }
+
+    end
+
+end
+
+-- local template =[[
+--     REMOVE FROM
+--         %basename%
+--     WHERE
+--         `token` = '%token%' ;
+-- ]]
+--
+-- function loggers.remove(db,token)
+--
+--     db.execute {
+--         template  = template,
+--         variables = {
+--             basename = db.basename,
+--             token    = token,
+--         },
+--     }
+--
+--     if trace_sql then
+--         report("removed: %s",token)
+--     end
+--
+-- end
+
+local template_nop =[[
+    SELECT
+        `time`,
+        `type`,
+        `action`,
+        `data`
+    FROM
+        %basename%
+    ORDER BY
+        `time`, `type`, `action`
+    DESC LIMIT
+        %limit% ;
+]]
+
+local template_yes =[[
+    SELECT
+        `time`,
+        `type`,
+        `action`,
+        `data`
+    FROM
+        %basename%
+    %WHERE%
+    ORDER BY
+        `time`, `type`, `action`
+    DESC LIMIT
+        %limit% ;
+]]
+
+local converter = makeconverter {
+ -- { name = "time",   type = os.localtime  },
+    { name = "time",   type = "number"      },
+    { name = "type",   type = fromtype      },
+    { name = "action", type = "string"      },
+    { name = "data",   type = "deserialize" },
+}
+
+function loggers.collect(db,specification)
+
+    specification = specification or { }
+
+    local start  = specification.start
+    local stop   = specification.stop
+    local limit  = specification.limit or 100
+    local kind   = specification.type
+    local action = specification.action
+
+    local filtered = start or stop
+
+    local where  = { }
+
+    if filtered then
+        local today = os.date("*t")
+
+        if type(start) ~= "table" then
+            start = { }
+        end
+        start = os.time {
+            day    = start.day    or today.day,
+            month  = start.month  or today.month,
+            year   = start.year   or today.year,
+            hour   = start.hour   or 0,
+            minute = start.minute or 0,
+            second = start.second or 0,
+            isdst  = true,
+        }
+
+        if type(stop) ~= "table" then
+            stop = { }
+        end
+        stop = os.time {
+            day    = stop.day    or today.day,
+            month  = stop.month  or today.month,
+            year   = stop.year   or today.year,
+            hour   = stop.hour   or 24,
+            minute = stop.minute or 0,
+            second = stop.second or 0,
+            isdst  = true,
+        }
+
+     -- report("filter: %s => %s",start,stop)
+
+        where[#where+1] = format("`time` BETWEEN %s AND %s",start,stop)
+
+    end
+
+    if kind then
+        where[#where+1] = format("`type` = %s",totype[kind])
+    end
+
+    if action then
+        where[#where+1] = format("`action` = '%s'",action)
+    end
+
+    local records = db.execute {
+        template  = filtered and template_yes or template_nop,
+        converter = converter,
+        variables = {
+            basename = db.basename,
+            limit    = limit,
+            WHERE    = #where > 0 and format("WHERE\n%s",concat(where," AND ")) or "",
+        },
+    }
+
+    if trace_sql then
+        report("collected: %s loggers",#records)
+    end
+
+    return records, keys
+
+end
diff --git a/tex/context/base/util-sql-sessions.lua b/tex/context/base/util-sql-sessions.lua
new file mode 100644
index 000000000..40556dd5e
--- /dev/null
+++ b/tex/context/base/util-sql-sessions.lua
@@ -0,0 +1,349 @@
+if not modules then modules = { } end modules ['util-sql-sessions'] = {
+    version   = 1.001,
+    comment   = "companion to lmx-*",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This is experimental code and currently part of the base installation simply
+-- because it's easier to dirtribute this way. Eventually it will be documented
+-- and the related scripts will show up as well.
+
+-- maybe store threshold in session (in seconds)
+
+local tonumber = tonumber
+local format = string.format
+local ostime, uuid, osfulltime = os.time, os.uuid, os.fulltime
+local random = math.random
+
+-- In older frameworks we kept a session table in memory. This time we
+-- follow a route where we store session data in a sql table. Each session
+-- has a token (similar to what we do on q2p and pod services), a data
+-- blob which is just a serialized lua table (we could consider a dump instead)
+-- and two times: the creation and last accessed time. The first one is handy
+-- for statistics and the second one for cleanup. Both are just numbers so that
+-- we don't have to waste code on conversions. Anyhow, we provide variants so that
+-- we can always choose what is best.
+
+local sql         = utilities.sql
+local sessions    = { }
+sql.sessions      = sessions
+
+local trace_sql   = false  trackers.register("sql.sessions.trace", function(v) trace_sql = v end)
+local report      = logs.reporter("sql","sessions")
+
+sessions.newtoken = sql.tokens.new
+
+local function checkeddb(presets,datatable)
+    return sql.usedatabase(presets,datatable or presets.datatable or "sessions")
+end
+
+sessions.usedb = checkeddb
+
+local template =[[
+    CREATE TABLE IF NOT EXISTS %basename% (
+        `token`    varchar(50)       NOT NULL,
+        `data`     longtext          NOT NULL,
+        `created`  int(11)           NOT NULL,
+        `accessed` int(11)           NOT NULL,
+        UNIQUE KEY `token_unique_key` (`token`)
+    )
+    DEFAULT CHARSET = utf8 ;
+]]
+
+function sessions.createdb(presets,datatable)
+
+    local db = checkeddb(presets,datatable)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    report("datatable %q created in %q",db.name,db.base)
+
+    return db
+
+end
+
+local template =[[
+    DROP TABLE IF EXISTS %basename% ;
+]]
+
+function sessions.deletedb(presets,datatable)
+
+    local db = checkeddb(presets,datatable)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    report("datatable %q removed in %q",db.name,db.base)
+
+end
+
+local template =[[
+    INSERT INTO %basename% (
+        `token`,
+        `created`,
+        `accessed`,
+        `data`
+    ) VALUES (
+        '%token%',
+        %time%,
+        %time%,
+        '%[data]%'
+    ) ;
+]]
+
+function sessions.create(db,data)
+
+    local token  = sessions.newtoken()
+    local time   = ostime()
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            token    = token,
+            time     = time,
+            data     = db.serialize(data or { },"return")
+        },
+    }
+
+    if trace_sql then
+        report("created: %s at %s",token,osfulltime(time))
+    end
+
+    return {
+        token    = token,
+        created  = time,
+        accessed = time,
+        data     = data,
+    }
+end
+
+local template =[[
+    UPDATE
+        %basename%
+    SET
+        `data` = '%[data]%',
+        `accessed` = %time%
+    WHERE
+        `token` = '%token%' ;
+]]
+
+function sessions.save(db,session)
+
+    local time  = ostime()
+    local data  = db.serialize(session.data or { },"return")
+    local token = session.token
+
+    session.accessed = time
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            token    = token,
+            time     = ostime(),
+            data     = data,
+        },
+    }
+
+    if trace_sql then
+        report("saved: %s at %s",token,osfulltime(time))
+    end
+
+    return session
+end
+
+local template = [[
+    UPDATE
+        %basename%
+    SET
+        `accessed` = %time%
+    WHERE
+        `token` = '%token%' ;
+]]
+
+function sessions.touch(db,token)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            token    = token,
+            time     = ostime(),
+        },
+    }
+
+end
+
+local template = [[
+    UPDATE
+        %basename%
+    SET
+        `accessed` = %time%
+    WHERE
+        `token` = '%token%' ;
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `token` = '%token%' ;
+]]
+
+function sessions.restore(db,token)
+
+    local records, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            token    = token,
+            time     = ostime(),
+        },
+    }
+
+    local record = records and records[1]
+
+    if record then
+        if trace_sql then
+            report("restored: %s",token)
+        end
+        record.data = db.deserialize(record.data or "")
+        return record, keys
+    elseif trace_sql then
+        report("unknown: %s",token)
+    end
+
+end
+
+local template =[[
+    DELETE FROM
+        %basename%
+    WHERE
+        `token` = '%token%' ;
+]]
+
+function sessions.remove(db,token)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            token    = token,
+        },
+    }
+
+    if trace_sql then
+        report("removed: %s",token)
+    end
+
+end
+
+local template_collect_yes =[[
+    SELECT
+        *
+    FROM
+        %basename%
+    ORDER BY
+        `created` ;
+]]
+
+local template_collect_nop =[[
+    SELECT
+        `accessed`,
+        `created`,
+        `accessed`,
+        `token`
+    FROM
+        %basename%
+    ORDER BY
+        `created` ;
+]]
+
+function sessions.collect(db,nodata)
+
+    local records, keys = db.execute {
+        template  = nodata and template_collect_nop or template_collect_yes,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    if not nodata then
+        db.unpackdata(records)
+    end
+
+    if trace_sql then
+        report("collected: %s sessions",#records)
+    end
+
+    return records, keys
+
+end
+
+local template_cleanup_yes =[[
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `accessed` < %time%
+    ORDER BY
+        `created` ;
+    DELETE FROM
+        %basename%
+    WHERE
+        `accessed` < %time% ;
+]]
+
+local template_cleanup_nop =[[
+    SELECT
+        `accessed`,
+        `created`,
+        `accessed`,
+        `token`
+    FROM
+        %basename%
+    WHERE
+        `accessed` < %time%
+    ORDER BY
+        `created` ;
+    DELETE FROM
+        %basename%
+    WHERE
+        `accessed` < %time% ;
+]]
+
+function sessions.cleanupdb(db,delta,nodata)
+
+    local time = ostime()
+
+    local records, keys = db.execute {
+        template  = nodata and template_cleanup_nop or template_cleanup_yes,
+        variables = {
+            basename = db.basename,
+            time     = time - delta
+        },
+    }
+
+    if not nodata then
+        db.unpackdata(records)
+    end
+
+    if trace_sql then
+        report("cleaned: %s seconds before %s",delta,osfulltime(time))
+    end
+
+    return records, keys
+
+end
diff --git a/tex/context/base/util-sql-tickets.lua b/tex/context/base/util-sql-tickets.lua
new file mode 100644
index 000000000..ad885e34e
--- /dev/null
+++ b/tex/context/base/util-sql-tickets.lua
@@ -0,0 +1,698 @@
+if not modules then modules = { } end modules ['util-sql-tickets'] = {
+    version   = 1.001,
+    comment   = "companion to lmx-*",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- This is experimental code and currently part of the base installation simply
+-- because it's easier to dirtribute this way. Eventually it will be documented
+-- and the related scripts will show up as well.
+
+local tonumber = tonumber
+local format = string.format
+local ostime, uuid, osfulltime = os.time, os.uuid, os.fulltime
+local random = math.random
+local concat = table.concat
+
+local sql         = utilities.sql
+local tickets     = { }
+sql.tickets       = tickets
+
+local trace_sql   = false  trackers.register("sql.tickets.trace", function(v) trace_sql = v end)
+local report      = logs.reporter("sql","tickets")
+
+local serialize   = sql.serialize
+local deserialize = sql.deserialize
+local execute     = sql.execute
+
+tickets.newtoken  = sql.tokens.new
+
+local statustags  = { [0] = -- beware index can be string or number, maybe status should be a string in the database
+    "unknown",
+    "pending",
+    "busy",
+    "finished",
+    "error",
+    "deleted",
+}
+
+local status = table.swapped(statustags)
+
+tickets.status     = status
+tickets.statustags = statustags
+
+local function checkeddb(presets,datatable)
+    return sql.usedatabase(presets,datatable or presets.datatable or "tickets")
+end
+
+tickets.usedb = checkeddb
+
+local template =[[
+    CREATE TABLE IF NOT EXISTS %basename% (
+        `id`        int(11)     NOT NULL AUTO_INCREMENT,
+        `token`     varchar(50) NOT NULL,
+        `subtoken`  INT(11)     NOT NULL,
+        `created`   int(11)     NOT NULL,
+        `accessed`  int(11)     NOT NULL,
+        `category`  int(11)     NOT NULL,
+        `status`    int(11)     NOT NULL,
+        `usertoken` varchar(50) NOT NULL,
+        `data`      longtext    NOT NULL,
+        `comment`   longtext    NOT NULL,
+
+        PRIMARY KEY                     (`id`),
+        UNIQUE INDEX `id_unique_index`  (`id` ASC),
+        KEY          `token_unique_key` (`token`)
+    )
+    DEFAULT CHARSET = utf8 ;
+]]
+
+function tickets.createdb(presets,datatable)
+
+    local db = checkeddb(presets,datatable)
+
+    local data, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    report("datatable %q created in %q",db.name,db.base)
+
+    return db
+
+end
+
+local template =[[
+    DROP TABLE IF EXISTS %basename% ;
+]]
+
+function tickets.deletedb(presets,datatable)
+
+    local db = checkeddb(presets,datatable)
+
+    local data, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+        },
+    }
+
+    report("datatable %q removed in %q",db.name,db.base)
+
+end
+
+local template =[[
+    LOCK TABLES
+        %basename%
+    WRITE ;
+    INSERT INTO %basename% (
+        `token`,
+        `subtoken`,
+        `created`,
+        `accessed`,
+        `status`,
+        `category`,
+        `usertoken`,
+        `data`,
+        `comment`
+    ) VALUES (
+        '%token%',
+         %subtoken%,
+         %time%,
+         %time%,
+         %status%,
+         %category%,
+        '%usertoken%',
+        '%[data]%',
+        '%[comment]%'
+    ) ;
+    SELECT
+        LAST_INSERT_ID() AS `id` ;
+    UNLOCK TABLES ;
+]]
+
+function tickets.create(db,ticket)
+
+    local token     = ticket.token or tickets.newtoken()
+    local time      = ostime()
+    local status    = ticket.status or 0
+    local category  = ticket.category or 0
+    local subtoken  = ticket.subtoken or 0
+    local usertoken = ticket.usertoken or ""
+    local comment   = ticket.comment or ""
+
+    local result, message = db.execute {
+        template  = template,
+        variables = {
+            basename  = db.basename,
+            token     = token,
+            subtoken  = subtoken,
+            time      = time,
+            status    = status,
+            category  = category,
+            usertoken = usertoken,
+            data      = db.serialize(ticket.data or { },"return"),
+            comment   = comment,
+        },
+    }
+
+    if trace_sql then
+        report("created: %s at %s",token,osfulltime(time))
+    end
+
+    local r = result and result[1]
+
+    if r then
+
+        return {
+            id        = r.id,
+            token     = token,
+            subtoken  = subtoken,
+            created   = time,
+            accessed  = time,
+            status    = status,
+            category  = category,
+            usertoken = usertoken,
+            data      = data,
+            comment   = comment,
+        }
+
+    end
+end
+
+local template =[[
+    LOCK TABLES
+        %basename%
+    WRITE ;
+    UPDATE %basename% SET
+        `data` = '%[data]%',
+        `status` = %status%,
+        `accessed` = %time%
+    WHERE
+        `id` = %id% ;
+    UNLOCK TABLES ;
+]]
+
+function tickets.save(db,ticket)
+
+    local time     = ostime()
+    local data     = db.serialize(ticket.data or { },"return")
+    local status   = ticket.status or 0
+    local id       = ticket.id
+
+    if not status then
+        status = 0
+        ticket.status = 0
+    end
+
+    ticket.accessed = time
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            id       = id,
+            time     = ostime(),
+            status   = status,
+            data     = data,
+        },
+    }
+
+    if trace_sql then
+        report("saved: id %s, time %s",id,osfulltime(time))
+    end
+
+    return ticket
+end
+
+local template =[[
+    UPDATE
+        %basename%
+    SET
+        `accessed` = %time%
+    WHERE
+        `token` = '%token%' ;
+
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `id` = %id% ;
+]]
+
+function tickets.restore(db,id)
+
+    local record, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            id       = id,
+            time     = ostime(),
+        },
+    }
+
+    local record = record and record[1]
+
+    if record then
+        if trace_sql then
+            report("restored: id %s",id)
+        end
+        record.data = db.deserialize(record.data or "")
+        return record
+    elseif trace_sql then
+        report("unknown: id %s",id)
+    end
+
+end
+
+local template =[[
+    DELETE FROM
+        %basename%
+    WHERE
+        `id` = %id% ;
+]]
+
+function tickets.remove(db,id)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            id       = id,
+        },
+    }
+
+    if trace_sql then
+        report("removed: id %s",id)
+    end
+
+end
+
+local template_yes =[[
+    SELECT
+        *
+    FROM
+        %basename%
+    ORDER BY
+        `created` ;
+]]
+
+local template_nop =[[
+    SELECT
+        `created`,
+        `usertoken`,
+        `accessed`,
+        `status`
+    FROM
+        %basename%
+    ORDER BY
+        `created` ;
+]]
+
+function tickets.collect(db,nodata)
+
+    local records, keys = db.execute {
+        template  = nodata and template_nop or template_yes,
+        variables = {
+            basename = db.basename,
+            token    = token,
+        },
+    }
+
+    if not nodata then
+        db.unpackdata(records)
+    end
+
+    if trace_sql then
+        report("collected: %s tickets",#records)
+    end
+
+    return records, keys
+
+end
+
+local template =[[
+    DELETE FROM
+        %basename%
+    WHERE
+        `accessed` < %time% OR `status` = 5 ;
+]]
+
+local template_cleanup_yes =[[
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `accessed` < %time%
+    ORDER BY
+        `created` ;
+    DELETE FROM
+        %basename%
+    WHERE
+        `accessed` < %time% OR `status` = 5 ;
+]]
+
+local template_cleanup_nop =[[
+    SELECT
+        `accessed`,
+        `created`,
+        `accessed`,
+        `token`
+        `usertoken`
+    FROM
+        %basename%
+    WHERE
+        `accessed` < %time%
+    ORDER BY
+        `created` ;
+    DELETE FROM
+        %basename%
+    WHERE
+        `accessed` < %time% OR `status` = 5 ;
+]]
+
+function tickets.cleanupdb(db,delta,nodata) -- maybe delta in db
+
+    local time = delta and (ostime() - delta) or 0
+
+    local records, keys = db.execute {
+        template  = nodata and template_cleanup_nop or template_cleanup_yes,
+        variables = {
+            basename = db.basename,
+            time     = time,
+        },
+    }
+
+    if not nodata then
+        db.unpackdata(records)
+    end
+
+    if trace_sql then
+        report("cleaned: %s seconds before %s",delta,osfulltime(time))
+    end
+
+    return records, keys
+
+end
+
+-- status related functions
+
+local template =[[
+    SELECT
+        `status`
+    FROM
+        %basename%
+    WHERE
+        `token` = '%token%' ;
+]]
+
+function tickets.getstatus(db,token)
+
+    local record, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            token    = token,
+        },
+    }
+
+    local record = record and record[1]
+
+    return record and record.status or 0
+
+end
+
+local template =[[
+    SELECT
+        `status`
+    FROM
+        %basename%
+    WHERE
+        `status` = 5 OR `accessed` < %time% ;
+]]
+
+function tickets.getobsolete(db,delta)
+
+    local time = delta and (ostime() - delta) or 0
+
+    local records = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            time     = time,
+        },
+    }
+
+    db.unpackdata(records)
+
+    return records
+
+end
+
+local template =[[
+    SELECT
+        `id`
+    FROM
+        %basename%
+    WHERE
+        `status` = %status%
+    LIMIT
+        1 ;
+]]
+
+function tickets.hasstatus(db,status)
+
+    local record = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            status   = status or 0,
+        },
+    }
+
+    return record and #record > 0 or false
+
+end
+
+local template =[[
+    UPDATE
+        %basename%
+    SET
+        `status` = %status%,
+        `accessed` = %time%
+    WHERE
+        `id` = %id% ;
+]]
+
+function tickets.setstatus(db,id,status)
+
+    local record, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            id       = id,
+            time     = ostime(),
+            status   = status or 0,
+        },
+    }
+
+end
+
+local template =[[
+    DELETE FROM
+        %basename%
+    WHERE
+        `status` IN (%status%) ;
+]]
+
+function tickets.prunedb(db,status)
+
+    if type(status) == "table" then
+        status = concat(status,",")
+    end
+
+    local data, keys = db.execute {
+        template  = template,
+        variables = {
+            basename = db.basename,
+            status   = status or 0,
+        },
+    }
+
+    if trace_sql then
+        report("pruned: status %s removed",status)
+    end
+
+end
+
+local template_a = [[
+    LOCK TABLES
+        %basename%
+    WRITE ;
+    SET
+        @first_token = "?" ;
+    SELECT
+        `token`
+    INTO
+        @first_token
+    FROM
+        %basename%
+    WHERE
+        `status` = %status%
+    ORDER BY
+        `id`
+    LIMIT 1 ;
+    UPDATE
+        %basename%
+    SET
+        `status` = %newstatus%,
+        `accessed` = %time%
+    WHERE
+        `token` = @first_token ;
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `token` = @first_token
+    ORDER BY
+        `id` ;
+    UNLOCK TABLES ;
+]]
+
+local template_b = [[
+    SET
+        @first_token = "?" ;
+    SELECT
+        `token`
+    INTO
+        @first_token
+    FROM
+        %basename%
+    WHERE
+        `status` = %status%
+    ORDER BY
+        `id`
+    LIMIT 1 ;
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `token` = @first_token
+    ORDER BY
+        `id` ;
+]]
+
+function tickets.getfirstwithstatus(db,status,newstatus)
+
+    local records
+
+    if type(newstatus) == "number" then
+
+        records = db.execute {
+            template  = template_a,
+            variables = {
+                basename  = db.basename,
+                status    = status or 0,
+                newstatus = newstatus,
+                time      = ostime(),
+            },
+        }
+
+
+    else
+
+        records = db.execute {
+            template  = template_b,
+            variables = {
+                basename = db.basename,
+                status   = status or 0,
+            },
+        }
+
+    end
+
+    if type(records) == "table" and #records > 0 then
+
+        for i=1,#records do
+            local record = records[i]
+            record.data = db.deserialize(record.data or "")
+            record.status = newstatus
+        end
+
+        return records
+
+    end
+end
+
+local template =[[
+    SELECT
+        *
+    FROM
+        %basename%
+    WHERE
+        `usertoken` = '%usertoken%' AND `status` != 5
+    ORDER BY
+        `created` ;
+]]
+
+function tickets.getusertickets(db,usertoken)
+
+    -- todo: update accessed
+    -- todo: get less fields
+    -- maybe only data for status changed (hard to check)
+
+    local records, keys = db.execute {
+        template  = template,
+        variables = {
+            basename  = db.basename,
+            usertoken = usertoken,
+        },
+    }
+
+    db.unpackdata(records)
+
+    return records
+
+end
+
+local template =[[
+    LOCK TABLES
+        %basename%
+    WRITE ;
+    UPDATE %basename% SET
+        `status` = 5
+    WHERE
+        `usertoken` = '%usertoken%' ;
+    UNLOCK TABLES ;
+]]
+
+function tickets.removeusertickets(db,usertoken)
+
+    db.execute {
+        template  = template,
+        variables = {
+            basename  = db.basename,
+            usertoken = usertoken,
+        },
+    }
+
+    if trace_sql then
+        report("removed: usertoken %s",usertoken)
+    end
+
+end
+
+-- -- left-overs --
+
+-- LOCK TABLES `m4alltickets` WRITE ;
+-- CREATE TEMPORARY TABLE ticketset SELECT * FROM m4alltickets WHERE token = @first_token ;
+-- DROP TABLE ticketset ;
+-- UNLOCK TABLES ;
diff --git a/tex/context/base/util-sql.lua b/tex/context/base/util-sql.lua
index f86607377..fea60c96a 100644
--- a/tex/context/base/util-sql.lua
+++ b/tex/context/base/util-sql.lua
@@ -25,6 +25,9 @@ if not modules then modules = { } end modules ['util-sql'] = {
 -- efficiency issues (like creating a keys and types table for each row) but that could be
 -- optimized. Anyhow, fecthing results can be done as follows:
 
+-- We use the template mechanism from util-tpl which inturn is just using the dos cq
+-- windows convention of %whatever% variables that I've used for ages.
+
 -- local function collect_1(r)
 --     local t = { }
 --     for i=1,r:numrows() do
@@ -75,7 +78,7 @@ if not modules then modules = { } end modules ['util-sql'] = {
 
 local format, match = string.format, string.match
 local random = math.random
-local rawset, setmetatable, loadstring, type = rawset, setmetatable, loadstring, type
+local rawset, setmetatable, getmetatable, loadstring, type = rawset, setmetatable, getmetatable, loadstring, type
 local P, S, V, C, Cs, Ct, Cc, Cg, Cf, patterns, lpegmatch = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.patterns, lpeg.match
 local concat = table.concat
 
@@ -87,6 +90,9 @@ local trace_sql       = false  trackers.register("sql.trace",  function(v) trace
 local trace_queries   = false  trackers.register("sql.queries",function(v) trace_queries = v end)
 local report_state    = logs.reporter("sql")
 
+-- trace_sql     = true
+-- trace_queries = true
+
 utilities.sql         = utilities.sql or { }
 local sql             = utilities.sql
 
@@ -96,8 +102,11 @@ local loadtemplate    = utilities.templates.load
 local methods         = { }
 sql.methods           = methods
 
-sql.serialize         = table.fastserialize
-sql.deserialize       = table.deserialize
+local serialize       = table.fastserialize
+local deserialize     = table.deserialize
+
+sql.serialize         = serialize
+sql.deserialize       = deserialize
 
 local defaults     = { __index =
     {
@@ -207,14 +216,19 @@ local function validspecification(specification)
         presets = dofile(presets)
     end
     if type(presets) == "table" then
-        setmetatable(presets,defaults)
+        local m = getmetatable(presets)
+        if m then
+            setmetatable(m,defaults)
+        else
+            setmetatable(presets,defaults)
+        end
         setmetatable(specification,{ __index = presets })
     else
         setmetatable(specification,defaults)
     end
-    local templatefile = specification.templatefile
-    local queryfile    = specification.queryfile  or file.nameonly(templatefile) .. "-temp.sql"
-    local resultfile   = specification.resultfile or file.nameonly(templatefile) .. "-temp.dat"
+    local templatefile = specification.templatefile or "query"
+    local queryfile    = specification.queryfile  or presets.queryfile  or file.nameonly(templatefile) .. "-temp.sql"
+    local resultfile   = specification.resultfile or presets.resultfile or file.nameonly(templatefile) .. "-temp.dat"
     specification.queryfile  = queryfile
     specification.resultfile = resultfile
     if trace_sql then
@@ -310,7 +324,7 @@ sql.splitdata = splitdata
 
 local function execute(specification)
     if trace_sql then
-        report_state("executing")
+        report_state("executing client")
     end
     if not validspecification(specification) then
         report_state("error in specification")
@@ -342,6 +356,7 @@ methods.client = {
     execute     = execute,
     serialize   = serialize,
     deserialize = deserialize,
+    usesfiles   = true,
 }
 
 local function dataloaded(specification)
@@ -368,7 +383,7 @@ end
 
 local function execute(specification)
     if trace_sql then
-        report_state("executing")
+        report_state("executing lmxsql")
     end
     if not validspecification(specification) then
         report_state("error in specification")
@@ -400,6 +415,7 @@ methods.lmxsql = {
     execute     = execute,
     serialize   = serialize,
     deserialize = deserialize,
+    usesfiles   = true,
 }
 
 local mysql = nil
@@ -447,7 +463,7 @@ local query      = whitespace
                  * whitespace
 local splitter   = Ct(query * (separator * query)^0)
 
-local function datafetched(specification,query)
+local function datafetched(specification,query,converter)
     local id = specification.id
     local session, connection
     if id then
@@ -466,18 +482,32 @@ local function datafetched(specification,query)
         connection = connect(session,specification)
     end
     if not connection then
+        report_state("error in connection: %s@%s to %s:%s",
+                specification.database or "no database",
+                specification.username or "no username",
+                specification.host     or "no host",
+                specification.port     or "no port"
+            )
         return { }, { }
     end
     query = lpegmatch(splitter,query)
-    local result, message
+    local result, message, okay
     for i=1,#query do
         local q = query[i]
-        result, message = connection:execute(q)
-        if message then
-            report_state("error in query: %s",string.collapsespaces(q))
+        local r, m = connection:execute(q)
+        if m then
+            report_state("error in query, stage 1: %s",string.collapsespaces(q))
+            message = message and format("%s\n%s",message,m) or m
+        end
+        local t = type(r)
+        if t == "userdata" then
+            result = r
+            okay = true
+        elseif t == "number" then
+            okay = true
         end
     end
-    if not result and id then
+    if not okay and id then
         if session then
             session:close()
         end
@@ -489,34 +519,46 @@ local function datafetched(specification,query)
         cache[id] = { session = session, connection = connection }
         for i=1,#query do
             local q = query[i]
-            result, message = connection:execute(q)
-            if message then
-                report_state("error in query: %s",string.collapsespaces(q))
+            local r, m = connection:execute(q)
+            if m then
+                report_state("error in query, stage 2: %s",string.collapsespaces(q))
+                message = message and format("%s\n%s",message,m) or m
+            end
+            local t = type(r)
+            if t == "userdata" then
+                result = r
+                okay = true
+            elseif t == "number" then
+                okay = true
             end
         end
     end
     local data, keys
-    if result and type(result) ~= "number" then
-        keys = result:getcolnames()
-        if keys then
-            local n = result:numrows() or 0
-            if n == 0 then
-                data = { }
-         -- elseif n == 1 then
-         --  -- data = { result:fetch({},"a") }
-            else
-                data = { }
-             -- for i=1,n do
-             --     data[i] = result:fetch({},"a")
-             -- end
-                local k = #keys
-                for i=1,n do
-                    local v = { result:fetch() }
-                    local d = { }
-                    for i=1,k do
-                        d[keys[i]] = v[i]
+    if result then
+        if converter then
+            data = converter(result,deserialize)
+        else
+            keys = result:getcolnames()
+            if keys then
+                local n = result:numrows() or 0
+                if n == 0 then
+                    data = { }
+             -- elseif n == 1 then
+             --  -- data = { result:fetch({},"a") }
+                else
+                    data = { }
+                 -- for i=1,n do
+                 --     data[i] = result:fetch({},"a")
+                 -- end
+                    local k = #keys
+                    for i=1,n do
+                        local v = { result:fetch() }
+                        local d = { }
+                        for i=1,k do
+                            d[keys[i]] = v[i]
+                        end
+                        data[#data+1] = d
                     end
-                    data[#data+1] = d
                 end
             end
         end
@@ -547,7 +589,7 @@ local function execute(specification)
         end
     end
     if trace_sql then
-        report_state("executing")
+        report_state("executing library")
     end
     if not validspecification(specification) then
         report_state("error in specification")
@@ -558,7 +600,7 @@ local function execute(specification)
         report_state("error in preparation")
         return
     end
-    local data, keys = datafetched(specification,query)
+    local data, keys = datafetched(specification,query,specification.converter)
     if not data then
         report_state("error in fetching")
         return
@@ -571,6 +613,7 @@ methods.library = {
     execute     = execute,
     serialize   = serialize,
     deserialize = deserialize,
+    usesfiles   = false,
 }
 
 -- -- --
@@ -590,6 +633,66 @@ end
 
 sql.setmethod("client")
 
+-- helper:
+
+local execute = sql.execute
+
+function sql.usedatabase(presets,datatable)
+    local name = datatable or presets.datatable
+    if name then
+        local method   = presets.method and sql.methods[presets.method] or sql.methods.client
+        local base     = presets.database or "test"
+        local basename = format("`%s`.`%s`",base,name)
+        m_execute   = execute
+        deserialize = deserialize
+        serialize   = serialize
+        if method then
+            m_execute   = method.execute     or m_execute
+            deserialize = method.deserialize or deserialize
+            serialize   = method.serialize   or serialize
+        end
+        local execute
+        if method.usesfiles then
+            local queryfile   = presets.queryfile  or format("%s-temp.sql",name)
+            local resultfile  = presets.resultfile or format("%s-temp.dat",name)
+            execute = function(specification) -- variables template
+                if not specification.presets    then specification.presets    = presets   end
+                if not specification.queryfile  then specification.queryfile  = queryfile end
+                if not specification.resultfile then specification.resultfile = queryfile end
+                return m_execute(specification)
+            end
+        else
+            execute = function(specification) -- variables template
+                if not specification.presets then specification.presets = presets end
+                return m_execute(specification)
+            end
+        end
+        local function unpackdata(records,name)
+            if records then
+                name = name or "data"
+                for i=1,#records do
+                    local record = records[i]
+                    local data = record[name]
+                    if data then
+                        record[name] = deserialize(data)
+                    end
+                end
+            end
+        end
+        return {
+            presets     = preset,
+            base        = base,
+            name        = name,
+            basename    = basename,
+            execute     = execute,
+            serialize   = serialize,
+            deserialize = deserialize,
+            unpackdata  = unpackdata,
+        }
+    end
+end
+
+
 -- local data = utilities.sql.prepare {
 --     templatefile = "test.sql",
 --     variables    = { },
@@ -631,12 +734,92 @@ sql.setmethod("client")
 sql.tokens = {
     length = 42, -- but in practice we will reserve some 50 characters
     new    = function()
-        return format("%s-%s",osuuid(),match(format("%x05",random(ostime())),".-(.....)$")) -- 36 + 1 + 5 = 42
-    end,                                               -- or random(0xFFFFF*osclock())
+        return format("%s-%x05",osuuid(),random(0xFFFFF)) -- 36 + 1 + 5 = 42
+    end,
 }
 
 -- -- --
 
+local converters = { }
+
+sql.converters = converters
+
+local template = [[
+local converters = utilities.sql.converters
+
+local tostring   = tostring
+local tonumber   = tonumber
+local toboolean  = toboolean
+
+%s
+
+return function(result,deserialize)
+    if not result then
+        return { }
+    end
+    local nofrows = result:numrows() or 0
+    if nofrows == 0 then
+        return { }
+    end
+    local data = { }
+    for i=1,nofrows do
+        local v = { result:fetch() }
+        data[#data+1] = {
+            %s
+        }
+    end
+    return data
+end
+]]
+
+function sql.makeconverter(entries,deserialize)
+    local shortcuts   = { }
+    local assignments = { }
+    for i=1,#entries do
+        local entry = entries[i]
+        local nam   = entry.name
+        local typ   = entry.type
+        if typ == "boolean" then
+            assignments[i] = format("[%q] = toboolean(v[%s],true),",nam,i)
+        elseif typ == "number" then
+            assignments[i] = format("[%q] = tonumber(v[%s]),",nam,i)
+        elseif type(typ) == "function" then
+            local c = #converters + 1
+            converters[c] = typ
+            shortcuts[#shortcuts+1] = format("local fun_%s = converters[%s]",c,c)
+            assignments[i] = format("[%q] = fun_%s(v[%s]),",nam,c,i)
+        elseif type(typ) == "table" then
+            local c = #converters + 1
+            converters[c] = typ
+            shortcuts[#shortcuts+1] = format("local tab_%s = converters[%s]",c,c)
+            assignments[i] = format("[%q] = tab_%s[v[%s]],",nam,#converters,i)
+        elseif typ == "deserialize" then
+            assignments[i] = format("[%q] = deserialize(v[%s]),",nam,i)
+        else
+            assignments[i] = format("[%q] = v[%s],",nam,i)
+        end
+    end
+    local code = string.format(template,table.concat(shortcuts,"\n"),table.concat(assignments,"\n            "))
+    local func = loadstring(code)
+    if type(func) == "function" then
+        return func(), code
+    else
+        return false, code
+    end
+end
+
+-- local func, code = sql.makeconverter {
+--     { name = "a", type = "number" },
+--     { name = "b", type = "string" },
+--     { name = "c", type = "boolean" },
+--     { name = "d", type = { x = "1" } },
+--     { name = "e", type = os.fulltime },
+-- }
+--
+-- print(code)
+
+-- -- --
+
 if tex and tex.systemmodes then
 
     local droptable = table.drop
diff --git a/tex/context/base/util-tpl.lua b/tex/context/base/util-tpl.lua
index f6648b224..4cde1863b 100644
--- a/tex/context/base/util-tpl.lua
+++ b/tex/context/base/util-tpl.lua
@@ -6,10 +6,9 @@ if not modules then modules = { } end modules ['util-tpl'] = {
     license   = "see context related readme files"
 }
 
--- experimental code
-
--- maybe make %% scanning optional
--- maybe use $[ and ]$ or {{ }}
+-- This is experimental code. Coming from dos and windows, I've always used %whatever%
+-- as template variables so let's stick to it. After all, it's easy to parse and stands
+-- out well. A double %% is turned into a regular %.
 
 utilities.templates = utilities.templates or { }
 local templates     = utilities.templates
@@ -24,7 +23,7 @@ local P, C, Cs, Carg, lpegmatch = lpeg.P, lpeg.C, lpeg.Cs, lpeg.Carg, lpeg.match
 
 local replacer
 
-local function replacekey(k,t)
+local function replacekey(k,t,recursive)
     local v = t[k]
     if not v then
         if trace_template then
@@ -35,7 +34,11 @@ local function replacekey(k,t)
         if trace_template then
             report_template("setting key %q to value %q",k,v)
         end
-        return lpegmatch(replacer,v,1,t) -- recursive
+        if recursive then
+            return lpegmatch(replacer,v,1,t)
+        else
+            return v
+        end
     end
 end
 
@@ -56,9 +59,9 @@ local escapers = {
     end,
 }
 
-local function replacekeyunquoted(s,t,how) -- ".. \" "
+local function replacekeyunquoted(s,t,how,recurse) -- ".. \" "
     local escaper = how and escapers[how] or escapers.lua
-    return escaper(replacekey(s,t))
+    return escaper(replacekey(s,t,recurse))
 end
 
 local single      = P("%")  -- test %test% test   : resolves test
@@ -72,15 +75,15 @@ local nodouble    = double  / ''
 local nolquoted   = lquoted / ''
 local norquoted   = rquoted / ''
 
-local key         = nosingle * (C((1-nosingle)^1 * Carg(1))/replacekey) * nosingle
-local unquoted    = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2))/replacekeyunquoted) * norquoted
+local key         = nosingle * (C((1-nosingle)^1 * Carg(1) * Carg(2) * Carg(3))/replacekey) * nosingle
+local unquoted    = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2) * Carg(3))/replacekeyunquoted) * norquoted
 local any         = P(1)
 
       replacer    = Cs((unquoted + escape + key + any)^0)
 
-local function replace(str,mapping,how)
+local function replace(str,mapping,how,recurse)
     if mapping then
-        return lpegmatch(replacer,str,1,mapping,how or "lua") or str
+        return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str
     else
         return str
     end
@@ -91,25 +94,24 @@ end
 
 templates.replace = replace
 
-function templates.load(filename,mapping)
+function templates.load(filename,mapping,how,recurse)
     local data = io.loaddata(filename) or ""
     if mapping and next(mapping) then
-        return replace(data,mapping)
+        return replace(data,mapping,how,recurse)
     else
         return data
     end
 end
 
-function templates.resolve(t,mapping)
+function templates.resolve(t,mapping,how,recurse)
     if not mapping then
         mapping = t
     end
     for k, v in next, t do
-        t[k] = replace(v,mapping)
+        t[k] = replace(v,mapping,how,recurse)
     end
     return t
 end
 
 -- inspect(utilities.templates.replace("test %one% test", { one = "%two%", two = "two" }))
 -- inspect(utilities.templates.resolve({ one = "%two%", two = "two", three = "%three%" }))
-
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index b40cb3eae..323c70c3e 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 09/16/12 23:18:22
+-- merge date  : 09/21/12 20:58:19
 
 do -- begin closure to overcome local limits and interference
 
@@ -1059,23 +1059,27 @@ function table.reversed(t)
     end
 end
 
-function table.sequenced(t,sep,simple) -- hash only
-    local s, n = { }, 0
-    for k, v in sortedhash(t) do
-        if simple then
-            if v == true then
-                n = n + 1
-                s[n] = k
-            elseif v and v~= "" then
+function table.sequenced(t,sep) -- hash only
+    if t then
+        local s, n = { }, 0
+        for k, v in sortedhash(t) do
+            if simple then
+                if v == true then
+                    n = n + 1
+                    s[n] = k
+                elseif v and v~= "" then
+                    n = n + 1
+                    s[n] = k .. "=" .. tostring(v)
+                end
+            else
                 n = n + 1
                 s[n] = k .. "=" .. tostring(v)
             end
-        else
-            n = n + 1
-            s[n] = k .. "=" .. tostring(v)
         end
+        return concat(s, sep or " | ")
+    else
+        return ""
     end
-    return concat(s, sep or " | ")
 end
 
 function table.print(t,...)