Merge pull request #31 from phi-gamma/masterHEADmaster

v2.5

16 files changed, 446 insertions, 134 deletions

diff --git a/Makefile b/Makefile
index 5e0fd61..5e0fd2a 100644
--- a/Makefile
+++ b/Makefile
@@ -125,3 +125,14 @@ mrproper: clean
 	@$(RM) -r $(DISTDIR)
 
 merge: $(MERGED)
+
+ifndef DESTDIR
+install:
+	$(error "in order to install you need to provide $$DESTDIR")
+else
+install: $(TDS_ZIP)
+	$(info installing to destination “$(DESTDIR)”)
+	install -dm755 "$(DESTDIR)"
+	unzip "$(TDS_ZIP)" -d "$(DESTDIR)"
+endif
+
diff --git a/NEWS b/NEWS
index 0a5f935..a5164af 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,7 @@
                         History of the lualibs package
+2017/02/01 v2.5/
+    * sync with Context beta as of 2017-02-01
+
 2016/04/06 v2.4/
     * sync with Context beta as of 2016-04-06
     * basic maintenance
diff --git a/lualibs-dir.lua b/lualibs-dir.lua
index 81ac65e..bc691d5 100644
--- a/lualibs-dir.lua
+++ b/lualibs-dir.lua
@@ -335,6 +335,36 @@ end
 
 dir.globfiles = globfiles
 
+local function globdirs(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    local noffiles = #files
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if not func or func(name) then
+                    noffiles = noffiles + 1
+                    files[noffiles] = path .. "/" .. name
+                    if recurse then
+                        globdirs(path .. "/" .. name,recurse,func,files)
+                    end
+                end
+            end
+        end
+    end
+    return files
+end
+
+dir.globdirs = globdirs
+
+-- inspect(globdirs("e:/tmp"))
+
 -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
 -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
 -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
@@ -557,9 +587,13 @@ file.expandname = dir.expandname -- for convenience
 local stack = { }
 
 function dir.push(newdir)
-    insert(stack,currentdir())
+    local curdir = currentdir()
+    insert(stack,curdir)
     if newdir and newdir ~= "" then
         chdir(newdir)
+        return newdir
+    else
+        return curdir
     end
 end
 
diff --git a/lualibs-file.lua b/lualibs-file.lua
index b6822e9..f2a27ad 100644
--- a/lualibs-file.lua
+++ b/lualibs-file.lua
@@ -607,14 +607,17 @@ function file.robustname(str,strict)
     end
 end
 
-file.readdata = io.loaddata
-file.savedata = io.savedata
+local loaddata = io.loaddata
+local savedata = io.savedata
+
+file.readdata  = loaddata
+file.savedata  = savedata
 
 function file.copy(oldname,newname)
     if oldname and newname then
-        local data = io.loaddata(oldname)
+        local data = loaddata(oldname)
         if data and data ~= "" then
-            file.savedata(newname,data)
+            savedata(newname,data)
         end
     end
 end
diff --git a/lualibs-io.lua b/lualibs-io.lua
index a91d44d..2039017 100644
--- a/lualibs-io.lua
+++ b/lualibs-io.lua
@@ -7,6 +7,7 @@ if not modules then modules = { } end modules ['l-io'] = {
 }
 
 local io = io
+local open, flush, write, read = io.open, io.flush, io.write, io.read
 local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
 local concat = table.concat
 local floor = math.floor
@@ -30,15 +31,13 @@ local function readall(f)
     local size = f:seek("end")
     if size == 0 then
         return ""
-    elseif size < 1024*1024 then
-        f:seek("set",0)
+    end
+    f:seek("set",0)
+    if size < 1024*1024 then
         return f:read('*all')
     else
-        local done = f:seek("set",0)
         local step
-        if size < 1024*1024 then
-            step = 1024 * 1024
-        elseif size > 16*1024*1024 then
+        if size > 16*1024*1024 then
             step = 16*1024*1024
         else
             step = floor(size/(1024*1024)) * 1024 * 1024 / 8
@@ -58,9 +57,8 @@ end
 io.readall = readall
 
 function io.loaddata(filename,textmode) -- return nil if empty
-    local f = io.open(filename,(textmode and 'r') or 'rb')
+    local f = open(filename,(textmode and 'r') or 'rb')
     if f then
-     -- local data = f:read('*all')
         local data = readall(f)
         f:close()
         if #data > 0 then
@@ -69,8 +67,55 @@ function io.loaddata(filename,textmode) -- return nil if empty
     end
 end
 
+function io.copydata(source,target,action)
+    local f = open(source,"rb")
+    if f then
+        local g = open(target,"wb")
+        if g then
+            local size = f:seek("end")
+            if size == 0 then
+                -- empty
+            else
+                f:seek("set",0)
+                if size < 1024*1024 then
+                    local data = f:read('*all')
+                    if action then
+                        data = action(data)
+                    end
+                    if data then
+                        g:write(data)
+                    end
+                else
+                    local step
+                    if size > 16*1024*1024 then
+                        step = 16*1024*1024
+                    else
+                        step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+                    end
+                    while true do
+                        local data = f:read(step)
+                        if data then
+                            if action then
+                                data = action(data)
+                            end
+                            if data then
+                                g:write(data)
+                            end
+                        else
+                            break
+                        end
+                    end
+                end
+            end
+            g:close()
+        end
+        f:close()
+        flush()
+    end
+end
+
 function io.savedata(filename,data,joiner)
-    local f = io.open(filename,"wb")
+    local f = open(filename,"wb")
     if f then
         if type(data) == "table" then
             f:write(concat(data,joiner or ""))
@@ -80,7 +125,7 @@ function io.savedata(filename,data,joiner)
             f:write(data or "")
         end
         f:close()
-        io.flush()
+        flush()
         return true
     else
         return false
@@ -90,7 +135,7 @@ end
 -- we can also chunk this one if needed: io.lines(filename,chunksize,"*l")
 
 function io.loadlines(filename,n) -- return nil if empty
-    local f = io.open(filename,'r')
+    local f = open(filename,'r')
     if not f then
         -- no file
     elseif n then
@@ -118,7 +163,7 @@ function io.loadlines(filename,n) -- return nil if empty
 end
 
 function io.loadchunk(filename,n)
-    local f = io.open(filename,'rb')
+    local f = open(filename,'rb')
     if f then
         local data = f:read(n or 1024)
         f:close()
@@ -129,7 +174,7 @@ function io.loadchunk(filename,n)
 end
 
 function io.exists(filename)
-    local f = io.open(filename)
+    local f = open(filename)
     if f == nil then
         return false
     else
@@ -139,7 +184,7 @@ function io.exists(filename)
 end
 
 function io.size(filename)
-    local f = io.open(filename)
+    local f = open(filename)
     if f == nil then
         return 0
     else
@@ -149,11 +194,11 @@ function io.size(filename)
     end
 end
 
-function io.noflines(f)
+local function noflines(f)
     if type(f) == "string" then
-        local f = io.open(filename)
+        local f = open(filename)
         if f then
-            local n = f and io.noflines(f) or 0
+            local n = f and noflines(f) or 0
             f:close()
             return n
         else
@@ -169,6 +214,10 @@ function io.noflines(f)
     end
 end
 
+io.noflines = noflines
+
+-- inlined is faster
+
 local nextchar = {
     [ 4] = function(f)
         return f:read(1,1,1,1)
@@ -250,16 +299,16 @@ end
 
 function io.ask(question,default,options)
     while true do
-        io.write(question)
+        write(question)
         if options then
-            io.write(format(" [%s]",concat(options,"|")))
+            write(format(" [%s]",concat(options,"|")))
         end
         if default then
-            io.write(format(" [%s]",default))
+            write(format(" [%s]",default))
         end
-        io.write(format(" "))
-        io.flush()
-        local answer = io.read()
+        write(format(" "))
+        flush()
+        local answer = read()
         answer = gsub(answer,"^%s*(.*)%s*$","%1")
         if answer == "" and default then
             return default
diff --git a/lualibs-lua.lua b/lualibs-lua.lua
index b90f37e..3571538 100644
--- a/lualibs-lua.lua
+++ b/lualibs-lua.lua
@@ -198,3 +198,4 @@ if flush then
     local popen   = io.popen   if popen   then function io.popen  (...) flush() return popen  (...) end end
 
 end
+
diff --git a/lualibs-number.lua b/lualibs-number.lua
index 001ca31..c6f1e33 100644
--- a/lualibs-number.lua
+++ b/lualibs-number.lua
@@ -13,6 +13,7 @@ local tostring, tonumber = tostring, tonumber
 local format, floor, match, rep = string.format, math.floor, string.match, string.rep
 local concat, insert = table.concat, table.insert
 local lpegmatch = lpeg.match
+local floor = math.floor
 
 number       = number or { }
 local number = number
@@ -205,3 +206,25 @@ end
 function number.bits(n)
     return { bits(n,1) }
 end
+
+function number.bytetodecimal(b)
+    local d = floor(b * 100 / 255 + 0.5)
+    if d > 100 then
+        return 100
+    elseif d < -100 then
+        return -100
+    else
+        return d
+    end
+end
+
+function number.decimaltobyte(d)
+    local b = floor(d * 255 / 100 + 0.5)
+    if b > 255 then
+        return 255
+    elseif b < -255 then
+        return -255
+    else
+        return b
+    end
+end
diff --git a/lualibs-string.lua b/lualibs-string.lua
index e9dc2bb..be8f397 100644
--- a/lualibs-string.lua
+++ b/lualibs-string.lua
@@ -75,19 +75,19 @@ local collapser    = patterns.collapser
 local longtostring = patterns.longtostring
 
 function string.strip(str)
-    return lpegmatch(stripper,str) or ""
+    return str and lpegmatch(stripper,str) or ""
 end
 
 function string.fullstrip(str)
-    return lpegmatch(fullstripper,str) or ""
+    return str and lpegmatch(fullstripper,str) or ""
 end
 
 function string.collapsespaces(str)
-    return lpegmatch(collapser,str) or ""
+    return str and lpegmatch(collapser,str) or ""
 end
 
 function string.longtostring(str)
-    return lpegmatch(longtostring,str) or ""
+    return str and lpegmatch(longtostring,str) or ""
 end
 
 -- function string.is_empty(str)
@@ -99,7 +99,7 @@ local pattern = P(" ")^0 * P(-1) -- maybe also newlines
 -- patterns.onlyspaces = pattern
 
 function string.is_empty(str)
-    if str == "" then
+    if not str or str == "" then
         return true
     else
         return lpegmatch(pattern,str) and true or false
@@ -163,7 +163,7 @@ function string.escapedpattern(str,simple)
 end
 
 function string.topattern(str,lowercase,strict)
-    if str=="" or type(str) ~= "string" then
+    if str == "" or type(str) ~= "string" then
         return ".*"
     elseif strict then
         str = lpegmatch(pattern_c,str)
@@ -177,6 +177,7 @@ function string.topattern(str,lowercase,strict)
     end
 end
 
+-- print(string.escapedpattern("abc*234",true))
 -- print(string.escapedpattern("12+34*.tex",false))
 -- print(string.escapedpattern("12+34*.tex",true))
 -- print(string.topattern     ("12+34*.tex",false,false))
diff --git a/lualibs-table.lua b/lualibs-table.lua
index d1e0592..39357bd 100644
--- a/lualibs-table.lua
+++ b/lualibs-table.lua
@@ -478,7 +478,7 @@ function table.fromhash(t)
     return hsh
 end
 
-local noquotes, hexify, handle, compact, inline, functions
+local noquotes, hexify, handle, compact, inline, functions, metacheck
 
 local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key
     'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
@@ -608,7 +608,8 @@ local function do_serialize(root,name,depth,level,indexed)
         if compact then
             last = #root
             for k=1,last do
-                if root[k] == nil then
+             -- if root[k] == nil then
+                if rawget(root,k) == nil then
                     last = k - 1
                     break
                 end
@@ -817,6 +818,7 @@ local function serialize(_handle,root,name,specification) -- handle wins
         functions = specification.functions
         compact   = specification.compact
         inline    = specification.inline and compact
+        metacheck = specification.metacheck
         if functions == nil then
             functions = true
         end
@@ -826,6 +828,9 @@ local function serialize(_handle,root,name,specification) -- handle wins
         if inline == nil then
             inline = compact
         end
+        if metacheck == nil then
+            metacheck = true
+        end
     else
         noquotes  = false
         hexify    = false
@@ -833,6 +838,7 @@ local function serialize(_handle,root,name,specification) -- handle wins
         compact   = true
         inline    = true
         functions = true
+        metacheck = true
     end
     if tname == "string" then
         if name == "return" then
@@ -857,8 +863,9 @@ local function serialize(_handle,root,name,specification) -- handle wins
     end
     if root then
         -- The dummy access will initialize a table that has a delayed initialization
-        -- using a metatable. (maybe explicitly test for metatable)
-        if getmetatable(root) then -- todo: make this an option, maybe even per subtable
+        -- using a metatable. (maybe explicitly test for metatable). This can crash on
+        -- metatables that check the index against a number.
+        if metacheck and getmetatable(root) then
             local dummy = root._w_h_a_t_e_v_e_r_
             root._w_h_a_t_e_v_e_r_ = nil
         end
@@ -964,6 +971,41 @@ end
 
 table.flattened = flattened
 
+local function collapsed(t,f,h)
+    if f == nil then
+        f = { }
+        h = { }
+    end
+    for k=1,#t do
+        local v = t[k]
+        if type(v) == "table" then
+            collapsed(v,f,h)
+        elseif not h[v] then
+            f[#f+1] = v
+            h[v] = true
+        end
+    end
+    return f
+end
+
+local function collapsedhash(t,h)
+    if h == nil then
+        h = { }
+    end
+    for k=1,#t do
+        local v = t[k]
+        if type(v) == "table" then
+            collapsedhash(v,h)
+        else
+            h[v] = true
+        end
+    end
+    return h
+end
+
+table.collapsed     = collapsed     -- 20% faster than unique(collapsed(t))
+table.collapsedhash = collapsedhash
+
 local function unnest(t,f) -- only used in mk, for old times sake
     if not f then          -- and only relevant for token lists
         f = { }            -- this one can become obsolete
@@ -1070,7 +1112,7 @@ function table.count(t)
     return n
 end
 
-function table.swapped(t,s) -- hash
+function table.swapped(t,s) -- hash, we need to make sure we don't mess up next
     local n = { }
     if s then
         for k, v in next, s do
@@ -1083,7 +1125,14 @@ function table.swapped(t,s) -- hash
     return n
 end
 
-function table.mirrored(t) -- hash
+function table.hashed(t) -- list, add hash to index (save because we are not yet mixed
+    for i=1,#t do
+        t[t[i]] = i
+    end
+    return t
+end
+
+function table.mirrored(t) -- hash, we need to make sure we don't mess up next
     local n = { }
     for k, v in next, t do
         n[v] = k
diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua
index a1d7fb0..12a4f64 100644
--- a/lualibs-trac-inf.lua
+++ b/lualibs-trac-inf.lua
@@ -61,12 +61,13 @@ local function stoptiming(instance)
         timer.timing = it - 1
     else
         local starttime = timer.starttime
-        if starttime then
-            local stoptime = clock()
-            local loadtime = stoptime - starttime
-            timer.stoptime = stoptime
-            timer.loadtime = timer.loadtime + loadtime
-            timer.timing = 0
+        if starttime and starttime > 0 then
+            local stoptime  = clock()
+            local loadtime  = stoptime - starttime
+            timer.stoptime  = stoptime
+            timer.loadtime  = timer.loadtime + loadtime
+            timer.timing    = 0
+            timer.starttime = 0
             return loadtime
         end
     end
@@ -183,7 +184,7 @@ end
 
 function statistics.runtime()
     stoptiming(statistics)
-    stoptiming(statistics) -- somehow we can start the timer twice, but where
+ --  stoptiming(statistics) -- somehow we can start the timer twice, but where
     return statistics.formatruntime(elapsedtime(statistics))
 end
 
diff --git a/lualibs-util-fil.lua b/lualibs-util-fil.lua
index 28c92c7..0f9731a 100644
--- a/lualibs-util-fil.lua
+++ b/lualibs-util-fil.lua
@@ -6,8 +6,10 @@ if not modules then modules = { } end modules ['util-fil'] = {
     license   = "see context related readme files"
 }
 
-local byte = string.byte
-local extract = bit32.extract
+local byte    = string.byte
+local char    = string.char
+local extract = bit32 and bit32.extract
+local floor   = math.floor
 
 -- Here are a few helpers (the starting point were old ones I used for parsing
 -- flac files). In Lua 5.3 we can probably do this better. Some code will move
@@ -36,6 +38,8 @@ function files.size(f)
     return f:seek("end")
 end
 
+files.getsize = files.size
+
 function files.setposition(f,n)
     if zerobased[f] then
         f:seek("set",n)
@@ -90,7 +94,8 @@ end
 function files.readinteger1(f)  -- one byte
     local n = byte(f:read(1))
     if n  >= 0x80 then
-        return n - 0xFF - 1
+     -- return n - 0xFF - 1
+        return n - 0x100
     else
         return n
     end
@@ -104,12 +109,27 @@ function files.readcardinal2(f)
     local a, b = byte(f:read(2),1,2)
     return 0x100 * a + b
 end
+function files.readcardinal2le(f)
+    local b, a = byte(f:read(2),1,2)
+    return 0x100 * a + b
+end
 
 function files.readinteger2(f)
     local a, b = byte(f:read(2),1,2)
     local n = 0x100 * a + b
     if n >= 0x8000 then
-        return n - 0xFFFF - 1
+     -- return n - 0xFFFF - 1
+        return n - 0x10000
+    else
+        return n
+    end
+end
+function files.readinteger2le(f)
+    local b, a = byte(f:read(2),1,2)
+    local n = 0x100 * a + b
+    if n >= 0x8000 then
+     -- return n - 0xFFFF - 1
+        return n - 0x10000
     else
         return n
     end
@@ -119,17 +139,57 @@ function files.readcardinal3(f)
     local a, b, c = byte(f:read(3),1,3)
     return 0x10000 * a + 0x100 * b + c
 end
+function files.readcardinal3le(f)
+    local c, b, a = byte(f:read(3),1,3)
+    return 0x10000 * a + 0x100 * b + c
+end
+
+function files.readinteger3(f)
+    local a, b, c = byte(f:read(3),1,3)
+    local n = 0x10000 * a + 0x100 * b + c
+    if n >= 0x80000 then
+     -- return n - 0xFFFFFF - 1
+        return n - 0x1000000
+    else
+        return n
+    end
+end
+function files.readinteger3le(f)
+    local c, b, a = byte(f:read(3),1,3)
+    local n = 0x10000 * a + 0x100 * b + c
+    if n >= 0x80000 then
+     -- return n - 0xFFFFFF - 1
+        return n - 0x1000000
+    else
+        return n
+    end
+end
 
 function files.readcardinal4(f)
     local a, b, c, d = byte(f:read(4),1,4)
     return 0x1000000 * a + 0x10000 * b + 0x100 * c + d
 end
+function files.readcardinal4le(f)
+    local d, c, b, a = byte(f:read(4),1,4)
+    return 0x1000000 * a + 0x10000 * b + 0x100 * c + d
+end
 
 function files.readinteger4(f)
     local a, b, c, d = byte(f:read(4),1,4)
     local n = 0x1000000 * a + 0x10000 * b + 0x100 * c + d
     if n >= 0x8000000 then
-        return n - 0xFFFFFFFF - 1
+     -- return n - 0xFFFFFFFF - 1
+        return n - 0x100000000
+    else
+        return n
+    end
+end
+function files.readinteger4le(f)
+    local d, c, b, a = byte(f:read(4),1,4)
+    local n = 0x1000000 * a + 0x10000 * b + 0x100 * c + d
+    if n >= 0x8000000 then
+     -- return n - 0xFFFFFFFF - 1
+        return n - 0x100000000
     else
         return n
     end
@@ -139,23 +199,28 @@ function files.readfixed4(f)
     local a, b, c, d = byte(f:read(4),1,4)
     local n = 0x100 * a + b
     if n >= 0x8000 then
-        return n - 0xFFFF - 1 + (0x100 * c + d)/0xFFFF
+     -- return n - 0xFFFF - 1 + (0x100 * c + d)/0xFFFF
+        return n - 0x10000    + (0x100 * c + d)/0xFFFF
     else
         return n              + (0x100 * c + d)/0xFFFF
     end
 end
 
-function files.read2dot14(f)
-    local a, b = byte(f:read(2),1,2)
-    local n = 0x100 * a + b
-    local m = extract(n,0,30)
-    if n > 0x7FFF then
-        n = extract(n,30,2)
-        return m/0x4000 - 4
-    else
-        n = extract(n,30,2)
-        return n + m/0x4000
+if extract then
+
+    function files.read2dot14(f)
+        local a, b = byte(f:read(2),1,2)
+        local n = 0x100 * a + b
+        local m = extract(n,0,30)
+        if n > 0x7FFF then
+            n = extract(n,30,2)
+            return m/0x4000 - 4
+        else
+            n = extract(n,30,2)
+            return n + m/0x4000
+        end
     end
+
 end
 
 function files.skipshort(f,n)
@@ -165,3 +230,32 @@ end
 function files.skiplong(f,n)
     f:read(4*(n or 1))
 end
+
+-- writers (kind of slow)
+
+function files.writecardinal2(f,n)
+    local a = char(n % 256)
+    n = floor(n/256)
+    local b = char(n % 256)
+    f:write(b,a)
+end
+
+function files.writecardinal4(f,n)
+    local a = char(n % 256)
+    n = floor(n/256)
+    local b = char(n % 256)
+    n = floor(n/256)
+    local c = char(n % 256)
+    n = floor(n/256)
+    local d = char(n % 256)
+    f:write(d,c,b,a)
+end
+
+function files.writestring(f,s)
+    f:write(char(byte(s,1,#s)))
+end
+
+function files.writebyte(f,b)
+    f:write(char(b))
+end
+
diff --git a/lualibs-util-jsn.lua b/lualibs-util-jsn.lua
index bbe25d8..e835c07 100644
--- a/lualibs-util-jsn.lua
+++ b/lualibs-util-jsn.lua
@@ -64,18 +64,19 @@ local jnumber    = (1-whitespace-rparent-rbrace-comma)^1 / tonumber
 local key        = jstring
 
 local jsonconverter = { "value",
-    object   = lbrace * Cf(Ct("") * V("pair") * (comma * V("pair"))^0,rawset) * rbrace,
-    pair     = Cg(optionalws * key * optionalws * colon * V("value")),
-    array    = Ct(lparent * V("value") * (comma * V("value"))^0 * rparent),
-    value    = optionalws * (jstring + V("object") + V("array") + jtrue + jfalse + jnull + jnumber + #rparent) * optionalws,
+    hash  = lbrace * Cf(Ct("") * (V("pair") * (comma * V("pair"))^0 + optionalws),rawset) * rbrace,
+    pair  = Cg(optionalws * key * optionalws * colon * V("value")),
+    array = Ct(lparent * (V("value") * (comma * V("value"))^0 + optionalws) * rparent),
+--  value = optionalws * (jstring + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber + #rparent) * optionalws,
+    value = optionalws * (jstring + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws,
 }
 
 -- local jsonconverter = { "value",
---     object   = lbrace * Cf(Ct("") * V("pair") * (comma * V("pair"))^0,rawset) * rbrace,
---     pair     = Cg(optionalws * V("string") * optionalws * colon * V("value")),
---     array    = Ct(lparent * V("value") * (comma * V("value"))^0 * rparent),
---     string   = jstring,
---     value    = optionalws * (V("string") + V("object") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws,
+--     hash   = lbrace * Cf(Ct("") * (V("pair") * (comma * V("pair"))^0 + optionalws),rawset) * rbrace,
+--     pair   = Cg(optionalws * V("string") * optionalws * colon * V("value")),
+--     array  = Ct(lparent * (V("value") * (comma * V("value"))^0 + optionalws) * rparent),
+--     string = jstring,
+--     value  = optionalws * (V("string") + V("hash") + V("array") + jtrue + jfalse + jnull + jnumber) * optionalws,
 -- }
 
 -- lpeg.print(jsonconverter) -- size 181
@@ -156,3 +157,5 @@ end
 -- inspect(tmp)
 
 -- inspect(json.tostring(true))
+
+return json
diff --git a/lualibs-util-lua.lua b/lualibs-util-lua.lua
index e1dcdc9..b334600 100644
--- a/lualibs-util-lua.lua
+++ b/lualibs-util-lua.lua
@@ -158,3 +158,21 @@ end
 -- luautilities.registerdatatype(lpeg.P("!"),"lpeg")
 --
 -- print(luautilities.datatype(lpeg.P("oeps")))
+
+-- These finalizers will only be invoked when we have a proper lua_close
+-- call (which is not happening in luatex tex node yes) or finish with an
+-- os.exit(n,true).
+
+local finalizers = { }
+
+setmetatable(finalizers, {
+    __gc = function(t)
+        for i=1,#t do
+            pcall(t[i]) -- let's not crash
+        end
+    end
+} )
+
+function luautilities.registerfinalizer(f)
+    finalizers[#finalizers+1] = f
+end
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index a54a4aa..fb51025 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -10,7 +10,7 @@ utilities         = utilities or { }
 utilities.strings = utilities.strings or { }
 local strings     = utilities.strings
 
-local format, gsub, rep, sub = string.format, string.gsub, string.rep, string.sub
+local format, gsub, rep, sub, find = string.format, string.gsub, string.rep, string.sub, string.find
 local load, dump = load, string.dump
 local tonumber, type, tostring = tonumber, type, tostring
 local unpack, concat = table.unpack, table.concat
@@ -385,6 +385,43 @@ function number.signed(i)
     end
 end
 
+-- maybe to util-num
+
+local digit  = patterns.digit
+local period = patterns.period
+local three  = digit * digit * digit
+
+local splitter = Cs (
+    (((1 - (three^1 * period))^1 + C(three)) * (Carg(1) * three)^1 + C((1-period)^1))
+  * (P(1)/"" * Carg(2)) * C(2)
+)
+
+patterns.formattednumber = splitter
+
+function number.formatted(n,sep1,sep2)
+    local s = type(s) == "string" and n or format("%0.2f",n)
+    if sep1 == true then
+        return lpegmatch(splitter,s,1,".",",")
+    elseif sep1 == "." then
+        return lpegmatch(splitter,s,1,sep1,sep2 or ",")
+    elseif sep1 == "," then
+        return lpegmatch(splitter,s,1,sep1,sep2 or ".")
+    else
+        return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".")
+    end
+end
+
+-- print(number.formatted(1))
+-- print(number.formatted(12))
+-- print(number.formatted(123))
+-- print(number.formatted(1234))
+-- print(number.formatted(12345))
+-- print(number.formatted(123456))
+-- print(number.formatted(1234567))
+-- print(number.formatted(12345678))
+-- print(number.formatted(12345678,true))
+-- print(number.formatted(1234.56,"!","?"))
+
 local zero      = P("0")^1 / ""
 local plus      = P("+")   / ""
 local minus     = P("-")
@@ -732,43 +769,6 @@ local format_W = function(f) -- handy when doing depth related indent
     return format("nspaces[%s]",tonumber(f) or 0)
 end
 
--- maybe to util-num
-
-local digit  = patterns.digit
-local period = patterns.period
-local three  = digit * digit * digit
-
-local splitter = Cs (
-    (((1 - (three^1 * period))^1 + C(three)) * (Carg(1) * three)^1 + C((1-period)^1))
-  * (P(1)/"" * Carg(2)) * C(2)
-)
-
-patterns.formattednumber = splitter
-
-function number.formatted(n,sep1,sep2)
-    local s = type(s) == "string" and n or format("%0.2f",n)
-    if sep1 == true then
-        return lpegmatch(splitter,s,1,".",",")
-    elseif sep1 == "." then
-        return lpegmatch(splitter,s,1,sep1,sep2 or ",")
-    elseif sep1 == "," then
-        return lpegmatch(splitter,s,1,sep1,sep2 or ".")
-    else
-        return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".")
-    end
-end
-
--- print(number.formatted(1))
--- print(number.formatted(12))
--- print(number.formatted(123))
--- print(number.formatted(1234))
--- print(number.formatted(12345))
--- print(number.formatted(123456))
--- print(number.formatted(1234567))
--- print(number.formatted(12345678))
--- print(number.formatted(12345678,true))
--- print(number.formatted(1234.56,"!","?"))
-
 local format_m = function(f)
     n = n + 1
     if not f or f == "" then
@@ -801,9 +801,16 @@ end
 local format_extension = function(extensions,f,name)
     local extension = extensions[name] or "tostring(%s)"
     local f = tonumber(f) or 1
+    local w = find(extension,"%.%.%.")
     if f == 0 then
+        if w then
+            extension = gsub(extension,"%.%.%.","")
+        end
         return extension
     elseif f == 1 then
+        if w then
+            extension = gsub(extension,"%.%.%.","%%s")
+        end
         n = n + 1
         local a = "a" .. n
         return format(extension,a,a) -- maybe more times?
@@ -811,10 +818,16 @@ local format_extension = function(extensions,f,name)
         local a = "a" .. (n + f + 1)
         return format(extension,a,a)
     else
+        if w then
+            extension = gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s")
+        end
+        -- we could fill an array and then n = n + 1 unpack(t,n,n+f) but as we
+        -- cache we don't save much and there are hardly any extensions anyway
         local t = { }
         for i=1,f do
             n = n + 1
-            t[#t+1] = "a" .. n
+         -- t[#t+1] = "a" .. n
+            t[i] = "a" .. n
         end
         return format(extension,unpack(t))
     end
diff --git a/lualibs-util-tab.lua b/lualibs-util-tab.lua
index d502058..0521a2a 100644
--- a/lualibs-util-tab.lua
+++ b/lualibs-util-tab.lua
@@ -12,7 +12,7 @@ local tables     = utilities.tables
 
 local format, gmatch, gsub, sub = string.format, string.gmatch, string.gsub, string.sub
 local concat, insert, remove, sort = table.concat, table.insert, table.remove, table.sort
-local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring
+local setmetatable, getmetatable, tonumber, tostring, rawget = setmetatable, getmetatable, tonumber, tostring, rawget
 local type, next, rawset, tonumber, tostring, load, select = type, next, rawset, tonumber, tostring, load, select
 local lpegmatch, P, Cs, Cc = lpeg.match, lpeg.P, lpeg.Cs, lpeg.Cc
 local sortedkeys, sortedpairs = table.sortedkeys, table.sortedpairs
@@ -169,7 +169,8 @@ function table.tocsv(t,specification)
                     r[f] = tostring(field)
                 end
             end
-            result[#result+1] = concat(r,separator)
+         -- result[#result+1] = concat(r,separator)
+            result[i+1] = concat(r,separator)
         end
         return concat(result,"\n")
     else
@@ -485,11 +486,12 @@ end
 
 local selfmapper = { __index = function(t,k) t[k] = k return k end }
 
-function table.twowaymapper(t)
-    if not t then
-        t = { }
-    else
-        for i=0,#t do
+function table.twowaymapper(t)    -- takes a 0/1 .. n indexed table and returns
+    if not t then                 -- it with  string-numbers as indices + reverse
+        t = { }                   -- mapping (all strings) .. used in cvs etc but
+    else                          -- typically a helper that one forgets about
+        local zero = rawget(t,0)  -- so it might move someplace else
+        for i=zero and 0 or 1,#t do
             local ti = t[i]       -- t[1]     = "one"
             if ti then
                 local i = tostring(i)
@@ -497,7 +499,6 @@ function table.twowaymapper(t)
                 t[ti]   = i       -- t["one"] = "1"
             end
         end
-        t[""] = t[0] or ""
     end
  -- setmetatableindex(t,"key")
     setmetatable(t,selfmapper)
@@ -616,7 +617,8 @@ local function serialize(root,name,specification)
                     return nil
                 end
             end
-            local haszero = t[0]
+         -- local haszero = t[0]
+            local haszero = rawget(t,0) -- don't trigger meta
             if n == nt then
                 local tt = { }
                 for i=1,nt do
@@ -680,7 +682,8 @@ local function serialize(root,name,specification)
             local last  = 0
             last = #root
             for k=1,last do
-                if root[k] == nil then
+                if rawget(root,k) == nil then
+             -- if root[k] == nil then
                     last = k - 1
                     break
                 end
@@ -810,7 +813,8 @@ local function serialize(root,name,specification)
 
     if root then
         -- The dummy access will initialize a table that has a delayed initialization
-        -- using a metatable. (maybe explicitly test for metatable)
+        -- using a metatable. (maybe explicitly test for metatable). This can crash on
+        -- metatables that check the index against a number.
         if getmetatable(root) then -- todo: make this an option, maybe even per subtable
             local dummy = root._w_h_a_t_e_v_e_r_ -- needed
             root._w_h_a_t_e_v_e_r_ = nil
@@ -833,5 +837,10 @@ end
 table.serialize = serialize
 
 if setinspector then
-    setinspector("table",function(v) if type(v) == "table" then print(serialize(v,"table",{})) return true end end)
+    setinspector("table",function(v)
+        if type(v) == "table" then
+            print(serialize(v,"table",{ metacheck = false }))
+            return true
+        end
+    end)
 end
diff --git a/lualibs.dtx b/lualibs.dtx
index 291049d..fe9525e 100644
--- a/lualibs.dtx
+++ b/lualibs.dtx
@@ -1,6 +1,6 @@
 % \iffalse meta-comment
 %
-% Copyright (C) 2009--2016 by
+% Copyright (C) 2009--2017 by
 %
 %       PRAGMA ADE / ConTeXt Development Team
 %       The LuaLaTeX Dev Team
@@ -37,7 +37,7 @@
 \input docstrip.tex
 \Msg{************************************************************************}
 \Msg{* Installation}
-\Msg{* Package: lualibs 2016-04-06 v2.4 Lua additional functions.}
+\Msg{* Package: lualibs 2017-02-01 v2.5 Lua additional functions.}
 \Msg{************************************************************************}
 
 \keepsilent
@@ -48,7 +48,7 @@
 \preamble
 This is a generated file.
 
-Copyright (C) 2009--2016 by
+Copyright (C) 2009--2017 by
         PRAGMA ADE / ConTeXt Development Team
         The LuaLaTeX Dev Team
 
@@ -107,7 +107,7 @@ and lualibs-extended.lua.
 %<*driver>
 \NeedsTeXFormat{LaTeX2e}
 \ProvidesFile{lualibs.drv}
-  [2016/04/06 v2.4 Lua Libraries.]
+  [2017/02/01 v2.5 Lua Libraries.]
 \documentclass{ltxdoc}
 \usepackage{fancyvrb,xspace}
 \usepackage[x11names]{xcolor}
@@ -208,7 +208,7 @@ and lualibs-extended.lua.
 % \GetFileInfo{lualibs.drv}
 %
 % \title{The \identifier{lualibs} package}
-% \date{2016/04/06 v2.4}
+% \date{2017/02/01 v2.5}
 % \author{Élie Roux      · \email{elie.roux@telecom-bretagne.eu}\\
 %         Philipp Gesang · \email{phg@phi-gamma.net}}
 %
@@ -427,8 +427,8 @@ lualibs = lualibs or { }
 
 lualibs.module_info = {
   name          = "lualibs",
-  version       = 2.4,
-  date          = "2016-04-06",
+  version       = 2.5,
+  date          = "2017-02-01",
   description   = "ConTeXt Lua standard libraries.",
   author        = "Hans Hagen, PRAGMA-ADE, Hasselt NL & Elie Roux & Philipp Gesang",
   copyright     = "PRAGMA ADE / ConTeXt Development Team",
@@ -582,8 +582,8 @@ local loadmodule        = lualibs.loadmodule
 
 local lualibs_basic_module = {
   name          = "lualibs-basic",
-  version       = 2.4,
-  date          = "2016-04-06",
+  version       = 2.5,
+  date          = "2017-02-01",
   description   = "ConTeXt Lua libraries -- basic collection.",
   author        = "Hans Hagen, PRAGMA-ADE, Hasselt NL & Elie Roux & Philipp Gesang",
   copyright     = "PRAGMA ADE / ConTeXt Development Team",
@@ -664,8 +664,8 @@ lualibs = lualibs or { }
 
 local lualibs_extended_module = {
   name          = "lualibs-extended",
-  version       = 2.4,
-  date          = "2016-04-06",
+  version       = 2.5,
+  date          = "2017-02-01",
   description   = "ConTeXt Lua libraries -- extended collection.",
   author        = "Hans Hagen, PRAGMA-ADE, Hasselt NL & Elie Roux & Philipp Gesang",
   copyright     = "PRAGMA ADE / ConTeXt Development Team",