sync with Context as of 2016-09-15

8 files changed, 139 insertions, 58 deletions

diff --git a/lualibs-dir.lua b/lualibs-dir.lua
index 81ac65e..db4125c 100644
--- a/lualibs-dir.lua
+++ b/lualibs-dir.lua
@@ -335,6 +335,36 @@ end
 
 dir.globfiles = globfiles
 
+local function globdirs(path,recurse,func,files) -- func == pattern or function
+    if type(func) == "string" then
+        local s = func
+        func = function(name) return find(name,s) end
+    end
+    files = files or { }
+    local noffiles = #files
+    for name in walkdir(path) do
+        if find(name,"^%.") then
+            --- skip
+        else
+            local mode = attributes(name,'mode')
+            if mode == "directory" then
+                if not func or func(name) then
+                    noffiles = noffiles + 1
+                    files[noffiles] = path .. "/" .. name
+                    if recurse then
+                        globdirs(path .. "/" .. name,recurse,func,files)
+                    end
+                end
+            end
+        end
+    end
+    return files
+end
+
+dir.globdirs = globdirs
+
+-- inspect(globdirs("e:/tmp"))
+
 -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex")
 -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex")
 -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex")
diff --git a/lualibs-lua.lua b/lualibs-lua.lua
index b90f37e..3571538 100644
--- a/lualibs-lua.lua
+++ b/lualibs-lua.lua
@@ -198,3 +198,4 @@ if flush then
     local popen   = io.popen   if popen   then function io.popen  (...) flush() return popen  (...) end end
 
 end
+
diff --git a/lualibs-string.lua b/lualibs-string.lua
index e9dc2bb..88297f2 100644
--- a/lualibs-string.lua
+++ b/lualibs-string.lua
@@ -177,6 +177,7 @@ function string.topattern(str,lowercase,strict)
     end
 end
 
+-- print(string.escapedpattern("abc*234",true))
 -- print(string.escapedpattern("12+34*.tex",false))
 -- print(string.escapedpattern("12+34*.tex",true))
 -- print(string.topattern     ("12+34*.tex",false,false))
diff --git a/lualibs-table.lua b/lualibs-table.lua
index d1e0592..498f518 100644
--- a/lualibs-table.lua
+++ b/lualibs-table.lua
@@ -478,7 +478,7 @@ function table.fromhash(t)
     return hsh
 end
 
-local noquotes, hexify, handle, compact, inline, functions
+local noquotes, hexify, handle, compact, inline, functions, metacheck
 
 local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key
     'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
@@ -608,7 +608,8 @@ local function do_serialize(root,name,depth,level,indexed)
         if compact then
             last = #root
             for k=1,last do
-                if root[k] == nil then
+             -- if root[k] == nil then
+                if rawget(root,k) == nil then
                     last = k - 1
                     break
                 end
@@ -817,6 +818,7 @@ local function serialize(_handle,root,name,specification) -- handle wins
         functions = specification.functions
         compact   = specification.compact
         inline    = specification.inline and compact
+        metacheck = specification.metacheck
         if functions == nil then
             functions = true
         end
@@ -826,6 +828,9 @@ local function serialize(_handle,root,name,specification) -- handle wins
         if inline == nil then
             inline = compact
         end
+        if metacheck == nil then
+            metacheck = true
+        end
     else
         noquotes  = false
         hexify    = false
@@ -833,6 +838,7 @@ local function serialize(_handle,root,name,specification) -- handle wins
         compact   = true
         inline    = true
         functions = true
+        metacheck = true
     end
     if tname == "string" then
         if name == "return" then
@@ -857,8 +863,9 @@ local function serialize(_handle,root,name,specification) -- handle wins
     end
     if root then
         -- The dummy access will initialize a table that has a delayed initialization
-        -- using a metatable. (maybe explicitly test for metatable)
-        if getmetatable(root) then -- todo: make this an option, maybe even per subtable
+        -- using a metatable. (maybe explicitly test for metatable). This can crash on
+        -- metatables that check the index against a number.
+        if metacheck and getmetatable(root) then
             local dummy = root._w_h_a_t_e_v_e_r_
             root._w_h_a_t_e_v_e_r_ = nil
         end
diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua
index a1d7fb0..12a4f64 100644
--- a/lualibs-trac-inf.lua
+++ b/lualibs-trac-inf.lua
@@ -61,12 +61,13 @@ local function stoptiming(instance)
         timer.timing = it - 1
     else
         local starttime = timer.starttime
-        if starttime then
-            local stoptime = clock()
-            local loadtime = stoptime - starttime
-            timer.stoptime = stoptime
-            timer.loadtime = timer.loadtime + loadtime
-            timer.timing = 0
+        if starttime and starttime > 0 then
+            local stoptime  = clock()
+            local loadtime  = stoptime - starttime
+            timer.stoptime  = stoptime
+            timer.loadtime  = timer.loadtime + loadtime
+            timer.timing    = 0
+            timer.starttime = 0
             return loadtime
         end
     end
@@ -183,7 +184,7 @@ end
 
 function statistics.runtime()
     stoptiming(statistics)
-    stoptiming(statistics) -- somehow we can start the timer twice, but where
+ --  stoptiming(statistics) -- somehow we can start the timer twice, but where
     return statistics.formatruntime(elapsedtime(statistics))
 end
 
diff --git a/lualibs-util-lua.lua b/lualibs-util-lua.lua
index e1dcdc9..b334600 100644
--- a/lualibs-util-lua.lua
+++ b/lualibs-util-lua.lua
@@ -158,3 +158,21 @@ end
 -- luautilities.registerdatatype(lpeg.P("!"),"lpeg")
 --
 -- print(luautilities.datatype(lpeg.P("oeps")))
+
+-- These finalizers will only be invoked when we have a proper lua_close
+-- call (which is not happening in luatex tex node yes) or finish with an
+-- os.exit(n,true).
+
+local finalizers = { }
+
+setmetatable(finalizers, {
+    __gc = function(t)
+        for i=1,#t do
+            pcall(t[i]) -- let's not crash
+        end
+    end
+} )
+
+function luautilities.registerfinalizer(f)
+    finalizers[#finalizers+1] = f
+end
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index a54a4aa..fb51025 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -10,7 +10,7 @@ utilities         = utilities or { }
 utilities.strings = utilities.strings or { }
 local strings     = utilities.strings
 
-local format, gsub, rep, sub = string.format, string.gsub, string.rep, string.sub
+local format, gsub, rep, sub, find = string.format, string.gsub, string.rep, string.sub, string.find
 local load, dump = load, string.dump
 local tonumber, type, tostring = tonumber, type, tostring
 local unpack, concat = table.unpack, table.concat
@@ -385,6 +385,43 @@ function number.signed(i)
     end
 end
 
+-- maybe to util-num
+
+local digit  = patterns.digit
+local period = patterns.period
+local three  = digit * digit * digit
+
+local splitter = Cs (
+    (((1 - (three^1 * period))^1 + C(three)) * (Carg(1) * three)^1 + C((1-period)^1))
+  * (P(1)/"" * Carg(2)) * C(2)
+)
+
+patterns.formattednumber = splitter
+
+function number.formatted(n,sep1,sep2)
+    local s = type(s) == "string" and n or format("%0.2f",n)
+    if sep1 == true then
+        return lpegmatch(splitter,s,1,".",",")
+    elseif sep1 == "." then
+        return lpegmatch(splitter,s,1,sep1,sep2 or ",")
+    elseif sep1 == "," then
+        return lpegmatch(splitter,s,1,sep1,sep2 or ".")
+    else
+        return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".")
+    end
+end
+
+-- print(number.formatted(1))
+-- print(number.formatted(12))
+-- print(number.formatted(123))
+-- print(number.formatted(1234))
+-- print(number.formatted(12345))
+-- print(number.formatted(123456))
+-- print(number.formatted(1234567))
+-- print(number.formatted(12345678))
+-- print(number.formatted(12345678,true))
+-- print(number.formatted(1234.56,"!","?"))
+
 local zero      = P("0")^1 / ""
 local plus      = P("+")   / ""
 local minus     = P("-")
@@ -732,43 +769,6 @@ local format_W = function(f) -- handy when doing depth related indent
     return format("nspaces[%s]",tonumber(f) or 0)
 end
 
--- maybe to util-num
-
-local digit  = patterns.digit
-local period = patterns.period
-local three  = digit * digit * digit
-
-local splitter = Cs (
-    (((1 - (three^1 * period))^1 + C(three)) * (Carg(1) * three)^1 + C((1-period)^1))
-  * (P(1)/"" * Carg(2)) * C(2)
-)
-
-patterns.formattednumber = splitter
-
-function number.formatted(n,sep1,sep2)
-    local s = type(s) == "string" and n or format("%0.2f",n)
-    if sep1 == true then
-        return lpegmatch(splitter,s,1,".",",")
-    elseif sep1 == "." then
-        return lpegmatch(splitter,s,1,sep1,sep2 or ",")
-    elseif sep1 == "," then
-        return lpegmatch(splitter,s,1,sep1,sep2 or ".")
-    else
-        return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".")
-    end
-end
-
--- print(number.formatted(1))
--- print(number.formatted(12))
--- print(number.formatted(123))
--- print(number.formatted(1234))
--- print(number.formatted(12345))
--- print(number.formatted(123456))
--- print(number.formatted(1234567))
--- print(number.formatted(12345678))
--- print(number.formatted(12345678,true))
--- print(number.formatted(1234.56,"!","?"))
-
 local format_m = function(f)
     n = n + 1
     if not f or f == "" then
@@ -801,9 +801,16 @@ end
 local format_extension = function(extensions,f,name)
     local extension = extensions[name] or "tostring(%s)"
     local f = tonumber(f) or 1
+    local w = find(extension,"%.%.%.")
     if f == 0 then
+        if w then
+            extension = gsub(extension,"%.%.%.","")
+        end
         return extension
     elseif f == 1 then
+        if w then
+            extension = gsub(extension,"%.%.%.","%%s")
+        end
         n = n + 1
         local a = "a" .. n
         return format(extension,a,a) -- maybe more times?
@@ -811,10 +818,16 @@ local format_extension = function(extensions,f,name)
         local a = "a" .. (n + f + 1)
         return format(extension,a,a)
     else
+        if w then
+            extension = gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s")
+        end
+        -- we could fill an array and then n = n + 1 unpack(t,n,n+f) but as we
+        -- cache we don't save much and there are hardly any extensions anyway
         local t = { }
         for i=1,f do
             n = n + 1
-            t[#t+1] = "a" .. n
+         -- t[#t+1] = "a" .. n
+            t[i] = "a" .. n
         end
         return format(extension,unpack(t))
     end
diff --git a/lualibs-util-tab.lua b/lualibs-util-tab.lua
index d502058..9266598 100644
--- a/lualibs-util-tab.lua
+++ b/lualibs-util-tab.lua
@@ -12,7 +12,7 @@ local tables     = utilities.tables
 
 local format, gmatch, gsub, sub = string.format, string.gmatch, string.gsub, string.sub
 local concat, insert, remove, sort = table.concat, table.insert, table.remove, table.sort
-local setmetatable, getmetatable, tonumber, tostring = setmetatable, getmetatable, tonumber, tostring
+local setmetatable, getmetatable, tonumber, tostring, rawget = setmetatable, getmetatable, tonumber, tostring, rawget
 local type, next, rawset, tonumber, tostring, load, select = type, next, rawset, tonumber, tostring, load, select
 local lpegmatch, P, Cs, Cc = lpeg.match, lpeg.P, lpeg.Cs, lpeg.Cc
 local sortedkeys, sortedpairs = table.sortedkeys, table.sortedpairs
@@ -169,7 +169,8 @@ function table.tocsv(t,specification)
                     r[f] = tostring(field)
                 end
             end
-            result[#result+1] = concat(r,separator)
+         -- result[#result+1] = concat(r,separator)
+            result[i+1] = concat(r,separator)
         end
         return concat(result,"\n")
     else
@@ -489,7 +490,8 @@ function table.twowaymapper(t)
     if not t then
         t = { }
     else
-        for i=0,#t do
+        local zero = rawget(t,0)
+        for i=zero or 1,#t do
             local ti = t[i]       -- t[1]     = "one"
             if ti then
                 local i = tostring(i)
@@ -497,7 +499,7 @@ function table.twowaymapper(t)
                 t[ti]   = i       -- t["one"] = "1"
             end
         end
-        t[""] = t[0] or ""
+        t[""] = zero or ""
     end
  -- setmetatableindex(t,"key")
     setmetatable(t,selfmapper)
@@ -616,7 +618,8 @@ local function serialize(root,name,specification)
                     return nil
                 end
             end
-            local haszero = t[0]
+         -- local haszero = t[0]
+            local haszero = rawget(t,0) -- don't trigger meta
             if n == nt then
                 local tt = { }
                 for i=1,nt do
@@ -680,7 +683,8 @@ local function serialize(root,name,specification)
             local last  = 0
             last = #root
             for k=1,last do
-                if root[k] == nil then
+                if rawget(root,k) == nil then
+             -- if root[k] == nil then
                     last = k - 1
                     break
                 end
@@ -810,7 +814,8 @@ local function serialize(root,name,specification)
 
     if root then
         -- The dummy access will initialize a table that has a delayed initialization
-        -- using a metatable. (maybe explicitly test for metatable)
+        -- using a metatable. (maybe explicitly test for metatable). This can crash on
+        -- metatables that check the index against a number.
         if getmetatable(root) then -- todo: make this an option, maybe even per subtable
             local dummy = root._w_h_a_t_e_v_e_r_ -- needed
             root._w_h_a_t_e_v_e_r_ = nil
@@ -833,5 +838,10 @@ end
 table.serialize = serialize
 
 if setinspector then
-    setinspector("table",function(v) if type(v) == "table" then print(serialize(v,"table",{})) return true end end)
+    setinspector("table",function(v)
+        if type(v) == "table" then
+            print(serialize(v,"table",{ metacheck = false }))
+            return true
+        end
+    end)
 end