Merge pull request #25 from phi-gamma/master

prepare release

10 files changed, 529 insertions, 221 deletions

diff --git a/Makefile b/Makefile
index a50c63d..5e0fd61 100644
--- a/Makefile
+++ b/Makefile
@@ -34,6 +34,7 @@ DISTDIR = ./lualibs
 TEXMFROOT = ./texmf
 
 CTAN_ZIP = $(NAME).zip
+CTAN_ZIPSIG	= $(CTAN_ZIP).asc
 TDS_ZIP = $(NAME).tds.zip
 ZIPS = $(CTAN_ZIP) $(TDS_ZIP)
 
@@ -46,6 +47,7 @@ all: $(GENERATED) $(DOC_TEX)
 doc: $(COMPILED)
 unpack: $(UNPACKED)
 ctan: check $(CTAN_ZIP)
+sign: $(CTAN_ZIPSIG)
 tds: $(TDS_ZIP)
 world: all ctan
 
@@ -83,6 +85,11 @@ $(CTAN_ZIP): $(ALL_FILES) $(TDS_ZIP)
 	$(make-ctandir)
 	@zip -r -9 $@ $(DISTDIR) $(TDS_ZIP) >/dev/null
 
+$(CTAN_ZIPSIG): $(CTAN_ZIP)
+	@echo "Signing package $(CTAN_ZIP)"
+	@$(RM) -- $@
+	@gpg --batch --armor --detach-sign "$(CTAN_ZIP)"
+
 define run-install
 @mkdir -p $(RUNDIR) && cp $(RUNFILES) $(RUNDIR)
 @mkdir -p $(DOCDIR) && cp $(DOCFILES) $(DOCDIR)
diff --git a/lualibs-dir.lua b/lualibs-dir.lua
index 2572120..bcf28d0 100644
--- a/lualibs-dir.lua
+++ b/lualibs-dir.lua
@@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['l-dir'] = {
 -- dir.expandname will be merged with cleanpath and collapsepath
 
 local type, select = type, select
-local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub
+local find, gmatch, match, gsub, sub = string.find, string.gmatch, string.match, string.gsub, string.sub
 local concat, insert, remove, unpack = table.concat, table.insert, table.remove, table.unpack
 local lpegmatch = lpeg.match
 
@@ -21,8 +21,8 @@ local lfs = lfs
 
 local attributes = lfs.attributes
 local walkdir    = lfs.dir
-local isdir      = lfs.isdir
-local isfile     = lfs.isfile
+local isdir      = lfs.isdir  -- not robust, will be overloaded anyway
+local isfile     = lfs.isfile -- not robust, will be overloaded anyway
 local currentdir = lfs.currentdir
 local chdir      = lfs.chdir
 local mkdir      = lfs.mkdir
@@ -31,20 +31,41 @@ local onwindows  = os.type == "windows" or find(os.getenv("PATH"),";",1,true)
 
 -- in case we load outside luatex
 
-if not isdir then
-    function isdir(name)
-        local a = attributes(name)
-        return a and a.mode == "directory"
+if onwindows then
+
+    -- lfs.isdir does not like trailing /
+    -- lfs.dir accepts trailing /
+
+    local tricky = S("/\\") * P(-1)
+
+    isdir = function(name)
+        if lpegmatch(tricky,name) then
+            return attributes(name,"mode") == "directory"
+        else
+            return attributes(name.."/.","mode") == "directory"
+        end
     end
-    lfs.isdir = isdir
-end
 
-if not isfile then
-    function isfile(name)
-        local a = attributes(name)
-        return a and a.mode == "file"
+    isfile = function(name)
+        return attributes(name,"mode") == "file"
     end
+
+    lfs.isdir  = isdir
     lfs.isfile = isfile
+
+else
+
+    isdir = function(name)
+        return attributes(name,"mode") == "directory"
+    end
+
+    isfile = function(name)
+        return attributes(name,"mode") == "file"
+    end
+
+    lfs.isdir  = isdir
+    lfs.isfile = isfile
+
 end
 
 -- handy
@@ -53,63 +74,104 @@ function dir.current()
     return (gsub(currentdir(),"\\","/"))
 end
 
--- optimizing for no find (*) does not save time
-
---~ local function globpattern(path,patt,recurse,action) -- fails in recent luatex due to some change in lfs
---~     local ok, scanner
---~     if path == "/" then
---~         ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe
---~     else
---~         ok, scanner = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
---~     end
---~     if ok and type(scanner) == "function" then
---~         if not find(path,"/$") then path = path .. '/' end
---~         for name in scanner do
---~             local full = path .. name
---~             local mode = attributes(full,'mode')
---~             if mode == 'file' then
---~                 if find(full,patt) then
---~                     action(full)
---~                 end
---~             elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
---~                 globpattern(full,patt,recurse,action)
---~             end
---~         end
---~     end
---~ end
-
-local lfsisdir = isdir
-
-local function isdir(path)
-    path = gsub(path,"[/\\]+$","")
-    return lfsisdir(path)
-end
+-- somewhat optimized
 
-lfs.isdir = isdir
+local function glob_pattern_function(path,patt,recurse,action)
+    if isdir(path) then
+        local usedpath
+        if path == "/" then
+            usedpath = "/."
+        elseif not find(path,"/$") then
+            usedpath = path .. "/."
+            path = path .. "/"
+        else
+            usedpath = path
+        end
+        local dirs
+        for name in walkdir(usedpath) do
+            if name ~= "." and name ~= ".." then
+                local full = path .. name
+                local mode = attributes(full,'mode')
+                if mode == 'file' then
+                    if not patt or find(full,patt) then
+                        action(full)
+                    end
+                elseif recurse and mode == "directory" then
+                    if not dirs then
+                        dirs = { full }
+                    else
+                        dirs[#dirs+1] = full
+                    end
+                end
+            end
+        end
+        if dirs then
+            for i=1,#dirs do
+                glob_pattern_function(dirs[i],patt,recurse,action)
+            end
+        end
+    end
+end
 
-local function globpattern(path,patt,recurse,action)
-    if path == "/" then
-        path = path .. "."
-    elseif not find(path,"/$") then
-        path = path .. '/'
+local function glob_pattern_table(path,patt,recurse,result)
+    if not result then
+        result = { }
     end
-    if isdir(path) then -- lfs.isdir does not like trailing /
-        for name in walkdir(path) do -- lfs.dir accepts trailing /
-            local full = path .. name
-            local mode = attributes(full,'mode')
-            if mode == 'file' then
-                if find(full,patt) then
-                    action(full)
+    if isdir(path) then
+        local usedpath
+        if path == "/" then
+            usedpath = "/."
+        elseif not find(path,"/$") then
+            usedpath = path .. "/."
+            path = path .. "/"
+        else
+            usedpath = path
+        end
+        local dirs
+        for name in walkdir(usedpath) do
+            if name ~= "." and name ~= ".." then
+                local full = path .. name
+                local mode = attributes(full,'mode')
+                if mode == 'file' then
+                    if not patt or find(full,patt) then
+                        result[#result+1] = full
+                    end
+                elseif recurse and mode == "directory" then
+                    if not dirs then
+                        dirs = { full }
+                    else
+                        dirs[#dirs+1] = full
+                    end
                 end
-            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
-                globpattern(full,patt,recurse,action)
             end
         end
+        if dirs then
+            for i=1,#dirs do
+                glob_pattern_table(dirs[i],patt,recurse,result)
+            end
+        end
+    end
+    return result
+end
+
+local function globpattern(path,patt,recurse,method)
+    local kind = type(method)
+    if pattern and sub(patt,1,-3) == path then
+        patt = false
+    end
+    if kind == "function" then
+        return glob_pattern_function(path,patt,recurse,method)
+    elseif kind == "table" then
+        return glob_pattern_table(path,patt,recurse,method)
+    else
+        return glob_pattern_table(path,patt,recurse,{ })
     end
 end
 
 dir.globpattern = globpattern
 
+-- never or seldom used so far:
+
 local function collectpattern(path,patt,recurse,result)
     local ok, scanner
     result = result or { }
@@ -119,18 +181,26 @@ local function collectpattern(path,patt,recurse,result)
         ok, scanner, first = xpcall(function() return walkdir(path)      end, function() end) -- kepler safe
     end
     if ok and type(scanner) == "function" then
-        if not find(path,"/$") then path = path .. '/' end
+        if not find(path,"/$") then
+            path = path .. '/'
+        end
         for name in scanner, first do
-            local full = path .. name
-            local attr = attributes(full)
-            local mode = attr.mode
-            if mode == 'file' then
-                if find(full,patt) then
+            if name == "." then
+                -- skip
+            elseif name == ".." then
+                -- skip
+            else
+                local full = path .. name
+                local attr = attributes(full)
+                local mode = attr.mode
+                if mode == 'file' then
+                    if find(full,patt) then
+                        result[name] = attr
+                    end
+                elseif recurse and mode == "directory" then
+                    attr.list = collectpattern(full,patt,recurse)
                     result[name] = attr
                 end
-            elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then
-                attr.list = collectpattern(full,patt,recurse)
-                result[name] = attr
             end
         end
     end
@@ -143,15 +213,10 @@ local separator
 
 if onwindows then -- we could sanitize here
 
---     pattern = Ct {
---         [1] = (C(P(".") + S("/\\")^1) + C(R("az","AZ") * P(":") * S("/\\")^0) + Cc("./")) * V(2) * V(3),
---         [2] = C(((1-S("*?/\\"))^0 * S("/\\"))^0),
---         [3] = C(P(1)^0)
---     }
-
     local slash = S("/\\") / "/"
 
-    pattern = Ct {
+--     pattern = Ct {
+    pattern = {
         [1] = (Cs(P(".") + slash^1) + Cs(R("az","AZ") * P(":") * slash^0) + Cc("./")) * V(2) * V(3),
         [2] = Cs(((1-S("*?/\\"))^0 * slash)^0),
         [3] = Cs(P(1)^0)
@@ -159,7 +224,8 @@ if onwindows then -- we could sanitize here
 
 else -- assume unix
 
-    pattern = Ct {
+--     pattern = Ct {
+    pattern = {
         [1] = (C(P(".") + P("/")^1) + Cc("./")) * V(2) * V(3),
         [2] = C(((1-S("*?/"))^0 * P("/"))^0),
         [3] = C(P(1)^0)
@@ -186,12 +252,11 @@ local function glob(str,t)
         elseif isfile(str) then
             t(str)
         else
-            local split = lpegmatch(pattern,str) -- we could use the file splitter
-            if split then
-                local root, path, base = split[1], split[2], split[3]
+            local root, path, base = lpegmatch(pattern,str) -- we could use the file splitter
+            if root and path and base then
                 local recurse = find(base,"**",1,true) -- find(base,"%*%*")
-                local start = root .. path
-                local result = lpegmatch(filter,start .. base)
+                local start   = root .. path
+                local result  = lpegmatch(filter,start .. base)
                 globpattern(start,result,recurse,t)
             end
         end
@@ -210,16 +275,12 @@ local function glob(str,t)
                 return { str }
             end
         else
-            local split = lpegmatch(pattern,str) -- we could use the file splitter
-            if split then
-                local t = t or { }
-                local action = action or function(name) t[#t+1] = name end
-                local root, path, base = split[1], split[2], split[3]
-                local recurse =  find(base,"**",1,true) -- find(base,"%*%*")
-                local start = root .. path
-                local result = lpegmatch(filter,start .. base)
-                globpattern(start,result,recurse,action)
-                return t
+            local root, path, base = lpegmatch(pattern,str) -- we could use the file splitter
+            if root and path and base then
+                local recurse = find(base,"**",1,true) -- find(base,"%*%*")
+                local start   = root .. path
+                local result  = lpegmatch(filter,start .. base)
+                return globpattern(start,result,recurse,t)
             else
                 return { }
             end
@@ -229,11 +290,20 @@ end
 
 dir.glob = glob
 
---~ list = dir.glob("**/*.tif")
---~ list = dir.glob("/**/*.tif")
---~ list = dir.glob("./**/*.tif")
---~ list = dir.glob("oeps/**/*.tif")
---~ list = dir.glob("/oeps/**/*.tif")
+-- local c = os.clock()
+-- local t = dir.glob("e:/**")
+-- local t = dir.glob("t:/sources/**")
+-- local t = dir.glob("t:/**")
+-- print(os.clock()-c,#t)
+
+-- for i=1,3000 do print(t[i]) end
+-- for i=1,10 do print(t[i]) end
+
+-- list = dir.glob("**/*.tif")
+-- list = dir.glob("/**/*.tif")
+-- list = dir.glob("./**/*.tif")
+-- list = dir.glob("oeps/**/*.tif")
+-- list = dir.glob("/oeps/**/*.tif")
 
 local function globfiles(path,recurse,func,files) -- func == pattern or function
     if type(func) == "string" then
@@ -275,10 +345,10 @@ function dir.ls(pattern)
     return concat(glob(pattern),"\n")
 end
 
---~ mkdirs("temp")
---~ mkdirs("a/b/c")
---~ mkdirs(".","/a/b/c")
---~ mkdirs("a","b","c")
+-- mkdirs("temp")
+-- mkdirs("a/b/c")
+-- mkdirs(".","/a/b/c")
+-- mkdirs("a","b","c")
 
 local make_indeed = true -- false
 
@@ -347,17 +417,17 @@ if onwindows then
         return pth, (isdir(pth) == true)
     end
 
-    --~ print(dir.mkdirs("","","a","c"))
-    --~ print(dir.mkdirs("a"))
-    --~ print(dir.mkdirs("a:"))
-    --~ print(dir.mkdirs("a:/b/c"))
-    --~ print(dir.mkdirs("a:b/c"))
-    --~ print(dir.mkdirs("a:/bbb/c"))
-    --~ print(dir.mkdirs("/a/b/c"))
-    --~ print(dir.mkdirs("/aaa/b/c"))
-    --~ print(dir.mkdirs("//a/b/c"))
-    --~ print(dir.mkdirs("///a/b/c"))
-    --~ print(dir.mkdirs("a/bbb//ccc/"))
+    -- print(dir.mkdirs("","","a","c"))
+    -- print(dir.mkdirs("a"))
+    -- print(dir.mkdirs("a:"))
+    -- print(dir.mkdirs("a:/b/c"))
+    -- print(dir.mkdirs("a:b/c"))
+    -- print(dir.mkdirs("a:/bbb/c"))
+    -- print(dir.mkdirs("/a/b/c"))
+    -- print(dir.mkdirs("/aaa/b/c"))
+    -- print(dir.mkdirs("//a/b/c"))
+    -- print(dir.mkdirs("///a/b/c"))
+    -- print(dir.mkdirs("a/bbb//ccc/"))
 
 else
 
@@ -408,13 +478,13 @@ else
         return pth, (isdir(pth) == true)
     end
 
-    --~ print(dir.mkdirs("","","a","c"))
-    --~ print(dir.mkdirs("a"))
-    --~ print(dir.mkdirs("/a/b/c"))
-    --~ print(dir.mkdirs("/aaa/b/c"))
-    --~ print(dir.mkdirs("//a/b/c"))
-    --~ print(dir.mkdirs("///a/b/c"))
-    --~ print(dir.mkdirs("a/bbb//ccc/"))
+    -- print(dir.mkdirs("","","a","c"))
+    -- print(dir.mkdirs("a"))
+    -- print(dir.mkdirs("/a/b/c"))
+    -- print(dir.mkdirs("/aaa/b/c"))
+    -- print(dir.mkdirs("//a/b/c"))
+    -- print(dir.mkdirs("///a/b/c"))
+    -- print(dir.mkdirs("a/bbb//ccc/"))
 
 end
 
@@ -424,7 +494,7 @@ dir.makedirs = dir.mkdirs
 
 if onwindows then
 
-    function dir.expandname(str) -- will be merged with cleanpath and collapsepath
+    function dir.expandname(str) -- will be merged with cleanpath and collapsepath\
         local first, nothing, last = match(str,"^(//)(//*)(.*)$")
         if first then
             first = dir.current() .. "/" -- dir.current sanitizes
diff --git a/lualibs-file.lua b/lualibs-file.lua
index ebb2b39..c05372a 100644
--- a/lualibs-file.lua
+++ b/lualibs-file.lua
@@ -495,6 +495,23 @@ function file.collapsepath(str,anchor) -- anchor: false|nil, true, "."
     end
 end
 
+-- better this way:
+
+local tricky     = S("/\\") * P(-1)
+local attributes = lfs.attributes
+
+function lfs.isdir(name)
+    if lpegmatch(tricky,name) then
+        return attributes(name,"mode") == "directory"
+    else
+        return attributes(name.."/.","mode") == "directory"
+    end
+end
+
+function lfs.isfile(name)
+    return attributes(name,"mode") == "file"
+end
+
 -- local function test(str,...)
 --    print(string.format("%-20s %-15s %-30s %-20s",str,file.collapsepath(str),file.collapsepath(str,true),file.collapsepath(str,".")))
 -- end
diff --git a/lualibs-lpeg.lua b/lualibs-lpeg.lua
index 6feb708..f3fd28b 100644
--- a/lualibs-lpeg.lua
+++ b/lualibs-lpeg.lua
@@ -107,7 +107,8 @@ local uppercase        = R("AZ")
 local underscore       = P("_")
 local hexdigit         = digit + lowercase + uppercase
 local cr, lf, crlf     = P("\r"), P("\n"), P("\r\n")
-local newline          = crlf + S("\r\n") -- cr + lf
+----- newline          = crlf + S("\r\n") -- cr + lf
+local newline          = P("\r") * (P("\n") + P(true)) + P("\n")
 local escaped          = P("\\") * anything
 local squote           = P("'")
 local dquote           = P('"')
@@ -141,8 +142,8 @@ patterns.utfbom_16_be  = utfbom_16_be
 patterns.utfbom_16_le  = utfbom_16_le
 patterns.utfbom_8      = utfbom_8
 
-patterns.utf_16_be_nl  = P("\000\r\000\n") + P("\000\r") + P("\000\n")
-patterns.utf_16_le_nl  = P("\r\000\n\000") + P("\r\000") + P("\n\000")
+patterns.utf_16_be_nl  = P("\000\r\000\n") + P("\000\r") + P("\000\n") -- P("\000\r") * (P("\000\n") + P(true)) + P("\000\n")
+patterns.utf_16_le_nl  = P("\r\000\n\000") + P("\r\000") + P("\n\000") -- P("\r\000") * (P("\n\000") + P(true)) + P("\n\000")
 
 patterns.utf8one       = R("\000\127")
 patterns.utf8two       = R("\194\223") * utf8next
@@ -224,9 +225,12 @@ patterns.integer       = sign^-1 * digit^1
 patterns.unsigned      = digit^0 * period * digit^1
 patterns.float         = sign^-1 * patterns.unsigned
 patterns.cunsigned     = digit^0 * comma * digit^1
+patterns.cpunsigned    = digit^0 * (period + comma) * digit^1
 patterns.cfloat        = sign^-1 * patterns.cunsigned
+patterns.cpfloat       = sign^-1 * patterns.cpunsigned
 patterns.number        = patterns.float + patterns.integer
 patterns.cnumber       = patterns.cfloat + patterns.integer
+patterns.cpnumber      = patterns.cpfloat + patterns.integer
 patterns.oct           = zero * octdigit^1
 patterns.octal         = patterns.oct
 patterns.HEX           = zero * P("X") * (digit+uppercase)^1
@@ -495,7 +499,6 @@ function lpeg.finder(lst,makefunction,isutf) -- beware: slower than find with 'p
         pattern = P(lst)
     end
     if isutf then
---         pattern = ((utf8char or 1)-pattern)^0 * pattern
         pattern = ((utf8char or 1)-pattern)^0 * pattern
     else
         pattern = (1-pattern)^0 * pattern
@@ -812,21 +815,76 @@ end
 
 -- experiment:
 
-local function make(t)
-    local p
+-- local function make(t)
+--     local p
+--     local keys = sortedkeys(t)
+--     for i=1,#keys do
+--         local k = keys[i]
+--         local v = t[k]
+--         if not p then
+--             if next(v) then
+--                 p = P(k) * make(v)
+--             else
+--                 p = P(k)
+--             end
+--         else
+--             if next(v) then
+--                 p = p + P(k) * make(v)
+--             else
+--                 p = p + P(k)
+--             end
+--         end
+--     end
+--     return p
+-- end
+
+-- local function make(t)
+--     local p = P(false)
+--     local keys = sortedkeys(t)
+--     for i=1,#keys do
+--         local k = keys[i]
+--         local v = t[k]
+--         if next(v) then
+--             p = p + P(k) * make(v)
+--         else
+--             p = p + P(k)
+--         end
+--     end
+--     return p
+-- end
+
+-- function lpeg.utfchartabletopattern(list) -- goes to util-lpg
+--     local tree = { }
+--     for i=1,#list do
+--         local t = tree
+--         for c in gmatch(list[i],".") do
+--             local tc = t[c]
+--             if not tc then
+--                 tc = { }
+--                 t[c] = tc
+--             end
+--             t = tc
+--         end
+--     end
+--     return make(tree)
+-- end
+
+local function make(t,hash)
+    local p = P(false)
     local keys = sortedkeys(t)
     for i=1,#keys do
         local k = keys[i]
         local v = t[k]
-        if not p then
+        local h = hash[v]
+        if h then
             if next(v) then
-                p = P(k) * make(v)
+                p = p + P(k) * (make(v,hash) + P(true))
             else
-                p = P(k)
+                p = p + P(k) * P(true)
             end
         else
             if next(v) then
-                p = p + P(k) * make(v)
+                p = p + P(k) * make(v,hash)
             else
                 p = p + P(k)
             end
@@ -837,16 +895,38 @@ end
 
 function lpeg.utfchartabletopattern(list) -- goes to util-lpg
     local tree = { }
-    for i=1,#list do
-        local t = tree
-        for c in gmatch(list[i],".") do
-            if not t[c] then
-                t[c] = { }
+    local hash = { }
+    local n = #list
+    if n == 0 then
+        -- we could always use this branch
+        for s in next, list do
+            local t = tree
+            for c in gmatch(s,".") do
+                local tc = t[c]
+                if not tc then
+                    tc = { }
+                    t[c] = tc
+                end
+                t = tc
+            end
+            hash[t] = s
+        end
+    else
+        for i=1,n do
+            local t = tree
+            local s = list[i]
+            for c in gmatch(s,".") do
+                local tc = t[c]
+                if not tc then
+                    tc = { }
+                    t[c] = tc
+                end
+                t = tc
             end
-            t = t[c]
+            hash[t] = s
         end
     end
-    return make(tree)
+    return make(tree,hash)
 end
 
 -- inspect ( lpeg.utfchartabletopattern {
diff --git a/lualibs-table.lua b/lualibs-table.lua
index d231830..e642106 100644
--- a/lualibs-table.lua
+++ b/lualibs-table.lua
@@ -164,14 +164,14 @@ local function sortedhash(t,cmp)
         end
         local n = 0
         local m = #s
-        local function kv(s)
+        local function kv() -- (s)
             if n < m then
                 n = n + 1
                 local k = s[n]
                 return k, t[k]
             end
         end
-        return kv, s
+        return kv -- , s
     else
         return nothing
     end
@@ -400,7 +400,7 @@ local function simple_table(t)
                 if tv == "number" then
                     nt = nt + 1
                     if hexify then
-                        tt[nt] = format("0x%04X",v)
+                        tt[nt] = format("0x%X",v)
                     else
                         tt[nt] = tostring(v) -- tostring not needed
                     end
@@ -451,7 +451,7 @@ local function do_serialize(root,name,depth,level,indexed)
             local tn = type(name)
             if tn == "number" then
                 if hexify then
-                    handle(format("%s[0x%04X]={",depth,name))
+                    handle(format("%s[0x%X]={",depth,name))
                 else
                     handle(format("%s[%s]={",depth,name))
                 end
@@ -502,7 +502,7 @@ local function do_serialize(root,name,depth,level,indexed)
             if compact and first and tk == "number" and k >= first and k <= last then
                 if tv == "number" then
                     if hexify then
-                        handle(format("%s 0x%04X,",depth,v))
+                        handle(format("%s 0x%X,",depth,v))
                     else
                         handle(format("%s %s,",depth,v)) -- %.99g
                     end
@@ -543,25 +543,25 @@ local function do_serialize(root,name,depth,level,indexed)
             elseif tv == "number" then
                 if tk == "number" then
                     if hexify then
-                        handle(format("%s [0x%04X]=0x%04X,",depth,k,v))
+                        handle(format("%s [0x%X]=0x%X,",depth,k,v))
                     else
                         handle(format("%s [%s]=%s,",depth,k,v)) -- %.99g
                     end
                 elseif tk == "boolean" then
                     if hexify then
-                        handle(format("%s [%s]=0x%04X,",depth,k and "true" or "false",v))
+                        handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v))
                     else
                         handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) -- %.99g
                     end
                 elseif noquotes and not reserved[k] and lpegmatch(propername,k) then
                     if hexify then
-                        handle(format("%s %s=0x%04X,",depth,k,v))
+                        handle(format("%s %s=0x%X,",depth,k,v))
                     else
                         handle(format("%s %s=%s,",depth,k,v)) -- %.99g
                     end
                 else
                     if hexify then
-                        handle(format("%s [%q]=0x%04X,",depth,k,v))
+                        handle(format("%s [%q]=0x%X,",depth,k,v))
                     else
                         handle(format("%s [%q]=%s,",depth,k,v)) -- %.99g
                     end
@@ -570,7 +570,7 @@ local function do_serialize(root,name,depth,level,indexed)
                 if reduce and tonumber(v) then
                     if tk == "number" then
                         if hexify then
-                            handle(format("%s [0x%04X]=%s,",depth,k,v))
+                            handle(format("%s [0x%X]=%s,",depth,k,v))
                         else
                             handle(format("%s [%s]=%s,",depth,k,v))
                         end
@@ -584,7 +584,7 @@ local function do_serialize(root,name,depth,level,indexed)
                 else
                     if tk == "number" then
                         if hexify then
-                            handle(format("%s [0x%04X]=%q,",depth,k,v))
+                            handle(format("%s [0x%X]=%q,",depth,k,v))
                         else
                             handle(format("%s [%s]=%q,",depth,k,v))
                         end
@@ -600,7 +600,7 @@ local function do_serialize(root,name,depth,level,indexed)
                 if not next(v) then
                     if tk == "number" then
                         if hexify then
-                            handle(format("%s [0x%04X]={},",depth,k))
+                            handle(format("%s [0x%X]={},",depth,k))
                         else
                             handle(format("%s [%s]={},",depth,k))
                         end
@@ -616,7 +616,7 @@ local function do_serialize(root,name,depth,level,indexed)
                     if st then
                         if tk == "number" then
                             if hexify then
-                                handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", ")))
+                                handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", ")))
                             else
                                 handle(format("%s [%s]={ %s },",depth,k,concat(st,", ")))
                             end
@@ -636,7 +636,7 @@ local function do_serialize(root,name,depth,level,indexed)
             elseif tv == "boolean" then
                 if tk == "number" then
                     if hexify then
-                        handle(format("%s [0x%04X]=%s,",depth,k,v and "true" or "false"))
+                        handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false"))
                     else
                         handle(format("%s [%s]=%s,",depth,k,v and "true" or "false"))
                     end
@@ -653,7 +653,7 @@ local function do_serialize(root,name,depth,level,indexed)
                  -- local f = getinfo(v).what == "C" and dump(function(...) return v(...) end) or dump(v) -- maybe strip
                     if tk == "number" then
                         if hexify then
-                            handle(format("%s [0x%04X]=load(%q),",depth,k,f))
+                            handle(format("%s [0x%X]=load(%q),",depth,k,f))
                         else
                             handle(format("%s [%s]=load(%q),",depth,k,f))
                         end
@@ -668,7 +668,7 @@ local function do_serialize(root,name,depth,level,indexed)
             else
                 if tk == "number" then
                     if hexify then
-                        handle(format("%s [0x%04X]=%q,",depth,k,tostring(v)))
+                        handle(format("%s [0x%X]=%q,",depth,k,tostring(v)))
                     else
                         handle(format("%s [%s]=%q,",depth,k,tostring(v)))
                     end
@@ -727,7 +727,7 @@ local function serialize(_handle,root,name,specification) -- handle wins
         end
     elseif tname == "number" then
         if hexify then
-            handle(format("[0x%04X]={",name))
+            handle(format("[0x%X]={",name))
         else
             handle("[" .. name .. "]={")
         end
@@ -922,16 +922,27 @@ end
 table.identical = identical
 table.are_equal = are_equal
 
--- maybe also make a combined one
-
-function table.compact(t) -- remove empty tables, assumes subtables
-    if t then
-        for k, v in next, t do
-            if not next(v) then -- no type checking
-                t[k] = nil
+local function sparse(old,nest,keeptables)
+    local new  = { }
+    for k, v in next, old do
+        if not (v == "" or v == false) then
+            if nest and type(v) == "table" then
+                v = sparse(v,nest)
+                if keeptables or next(v) then
+                    new[k] = v
+                end
+            else
+                new[k] = v
             end
         end
     end
+    return new
+end
+
+table.sparse = sparse
+
+function table.compact(t)
+    return sparse(t,true,true)
 end
 
 function table.contains(t, v)
@@ -1114,3 +1125,49 @@ function table.values(t,s) -- optional sort flag
         return { }
     end
 end
+
+-- maybe this will move to util-tab.lua
+
+-- for k, v in table.filtered(t,pattern)          do ... end
+-- for k, v in table.filtered(t,pattern,true)     do ... end
+-- for k, v in table.filtered(t,pattern,true,cmp) do ... end
+
+function table.filtered(t,pattern,sort,cmp)
+    if t and type(pattern) == "string" then
+        if sort then
+            local s
+            if cmp then
+                -- it would be nice if the sort function would accept a third argument (or nicer, an optional first)
+                s = sortedhashkeys(t,function(a,b) return cmp(t,a,b) end)
+            else
+                s = sortedkeys(t) -- the robust one
+            end
+            local n = 0
+            local m = #s
+            local function kv(s)
+                while n < m do
+                    n = n + 1
+                    local k = s[n]
+                    if find(k,pattern) then
+                        return k, t[k]
+                    end
+                end
+            end
+            return kv, s
+        else
+            local n = next(t)
+            local function iterator()
+                while n do
+                    local k = n
+                    n = next(t,k)
+                    if find(k,pattern) then
+                        return k, t[k]
+                    end
+                end
+            end
+            return iterator, t
+        end
+    else
+        return nothing
+    end
+end
diff --git a/lualibs-unicode.lua b/lualibs-unicode.lua
index be61f3d..fb4ea37 100644
--- a/lualibs-unicode.lua
+++ b/lualibs-unicode.lua
@@ -34,25 +34,29 @@ local type = type
 local char, byte, format, sub, gmatch = string.char, string.byte, string.format, string.sub, string.gmatch
 local concat = table.concat
 local P, C, R, Cs, Ct, Cmt, Cc, Carg, Cp = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Carg, lpeg.Cp
-local lpegmatch, patterns = lpeg.match, lpeg.patterns
 
-local bytepairs     = string.bytepairs
+local lpegmatch       = lpeg.match
+local patterns        = lpeg.patterns
+local tabletopattern  = lpeg.utfchartabletopattern
 
-local finder        = lpeg.finder
-local replacer      = lpeg.replacer
+local bytepairs       = string.bytepairs
 
-local utfvalues     = utf.values
-local utfgmatch     = utf.gmatch -- not always present
+local finder          = lpeg.finder
+local replacer        = lpeg.replacer
+
+local utfvalues       = utf.values
+local utfgmatch       = utf.gmatch -- not always present
 
 local p_utftype       = patterns.utftype
 local p_utfstricttype = patterns.utfstricttype
 local p_utfoffset     = patterns.utfoffset
-local p_utf8char      = patterns.utf8char
+local p_utf8char      = patterns.utf8character
 local p_utf8byte      = patterns.utf8byte
 local p_utfbom        = patterns.utfbom
 local p_newline       = patterns.newline
 local p_whitespace    = patterns.whitespace
 
+
 if not unicode then
 
     unicode = { utf = utf } -- for a while
@@ -510,8 +514,20 @@ end
 
 -- a replacement for simple gsubs:
 
+-- function utf.remapper(mapping)
+--     local pattern = Cs((p_utf8char/mapping)^0)
+--     return function(str)
+--         if not str or str == "" then
+--             return ""
+--         else
+--             return lpegmatch(pattern,str)
+--         end
+--     end, pattern
+-- end
+
 function utf.remapper(mapping)
-    local pattern = Cs((p_utf8char/mapping)^0)
+    local pattern = type(mapping) == "table" and tabletopattern(mapping) or p_utf8char
+    local pattern = Cs((pattern/mapping + p_utf8char)^0)
     return function(str)
         if not str or str == "" then
             return ""
diff --git a/lualibs-util-dim.lua b/lualibs-util-dim.lua
index 6906149..bfffb10 100644
--- a/lualibs-util-dim.lua
+++ b/lualibs-util-dim.lua
@@ -24,13 +24,15 @@ local formatters        = string.formatters
 
 local texget            = tex and tex.get or function() return 65536*10*100 end
 
+local p_stripzeros      = lpeg.patterns.stripzeros
+
 --this might become another namespace
 
 number = number or { }
 local number = number
 
-number.tonumberf = function(n) return match(format("%.20f",n),"(.-0?)0*$") end -- one zero too much but alas
-number.tonumberg = function(n) return       format("%.20g",n)              end
+number.tonumberf = function(n) return lpegmatch(p_stripzeros,format("%.20f",n)) end
+number.tonumberg = function(n) return format("%.20g",n) end
 
 local dimenfactors = allocate {
     ["pt"] =             1/65536,
@@ -46,66 +48,65 @@ local dimenfactors = allocate {
     ["nc"] = ( 5080/65043)/65536
 }
 
---~ print(table.serialize(dimenfactors))
---~
---~  %.99g:
---~
---~  t={
---~   ["bp"]=1.5201782378580324e-005,
---~   ["cc"]=1.1883696112892098e-006,
---~   ["cm"]=5.3628510057769479e-007,
---~   ["dd"]=1.4260435335470516e-005,
---~   ["em"]=0.000152587890625,
---~   ["ex"]=6.103515625e-005,
---~   ["in"]=2.1113586636917117e-007,
---~   ["mm"]=5.3628510057769473e-008,
---~   ["nc"]=1.1917446679504327e-006,
---~   ["nd"]=1.4300936015405194e-005,
---~   ["pc"]=1.2715657552083333e-006,
---~   ["pt"]=1.52587890625e-005,
---~   ["sp"]=1,
---~  }
---~
---~  patched %s and tonumber
---~
---~  t={
---~   ["bp"]=0.00001520178238,
---~   ["cc"]=0.00000118836961,
---~   ["cm"]=0.0000005362851,
---~   ["dd"]=0.00001426043534,
---~   ["em"]=0.00015258789063,
---~   ["ex"]=0.00006103515625,
---~   ["in"]=0.00000021113587,
---~   ["mm"]=0.00000005362851,
---~   ["nc"]=0.00000119174467,
---~   ["nd"]=0.00001430093602,
---~   ["pc"]=0.00000127156576,
---~   ["pt"]=0.00001525878906,
---~   ["sp"]=1,
---~  }
+-- print(table.serialize(dimenfactors))
+--
+--  %.99g:
+--
+--  t={
+--   ["bp"]=1.5201782378580324e-005,
+--   ["cc"]=1.1883696112892098e-006,
+--   ["cm"]=5.3628510057769479e-007,
+--   ["dd"]=1.4260435335470516e-005,
+--   ["em"]=0.000152587890625,
+--   ["ex"]=6.103515625e-005,
+--   ["in"]=2.1113586636917117e-007,
+--   ["mm"]=5.3628510057769473e-008,
+--   ["nc"]=1.1917446679504327e-006,
+--   ["nd"]=1.4300936015405194e-005,
+--   ["pc"]=1.2715657552083333e-006,
+--   ["pt"]=1.52587890625e-005,
+--   ["sp"]=1,
+--  }
+--
+--  patched %s and tonumber
+--
+--  t={
+--   ["bp"]=0.00001520178238,
+--   ["cc"]=0.00000118836961,
+--   ["cm"]=0.0000005362851,
+--   ["dd"]=0.00001426043534,
+--   ["em"]=0.00015258789063,
+--   ["ex"]=0.00006103515625,
+--   ["in"]=0.00000021113587,
+--   ["mm"]=0.00000005362851,
+--   ["nc"]=0.00000119174467,
+--   ["nd"]=0.00001430093602,
+--   ["pc"]=0.00000127156576,
+--   ["pt"]=0.00001525878906,
+--   ["sp"]=1,
+--  }
 
 --[[ldx--
 <p>A conversion function that takes a number, unit (string) and optional
 format (string) is implemented using this table.</p>
 --ldx]]--
 
+local f_none = formatters["%s%s"]
+local f_true = formatters["%0.5f%s"]
 
 local function numbertodimen(n,unit,fmt)
     if type(n) == 'string' then
         return n
     else
         unit = unit or 'pt'
+        n = n * dimenfactors[unit]
         if not fmt then
-            fmt = "%s%s"
+            fmt = f_none(n,unit)
         elseif fmt == true then
-            fmt = "%0.5f%s"
+            fmt = f_true(n,unit)
+        else
+            return formatters[fmt](n,unit)
         end
-        return format(fmt,n*dimenfactors[unit],unit)
-     -- if fmt then
-     --     return format(fmt,n*dimenfactors[unit],unit)
-     -- else
-     --     return match(format("%.20f",n*dimenfactors[unit]),"(.-0?)0*$") .. unit
-     -- end
     end
 end
 
diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua
index 2cede91..f51f6fc 100644
--- a/lualibs-util-prs.lua
+++ b/lualibs-util-prs.lua
@@ -542,8 +542,8 @@ end
 
 --
 
-local pattern_math = Cs((P("%")/"\\percent " +  P("^")           * Cc("{") * lpegpatterns.integer * Cc("}") + P(1))^0)
-local pattern_text = Cs((P("%")/"\\percent " + (P("^")/"\\high") * Cc("{") * lpegpatterns.integer * Cc("}") + P(1))^0)
+local pattern_math = Cs((P("%")/"\\percent " +  P("^")           * Cc("{") * lpegpatterns.integer * Cc("}") + anything)^0)
+local pattern_text = Cs((P("%")/"\\percent " + (P("^")/"\\high") * Cc("{") * lpegpatterns.integer * Cc("}") + anything)^0)
 
 patterns.unittotex = pattern
 
@@ -551,7 +551,7 @@ function parsers.unittotex(str,textmode)
     return lpegmatch(textmode and pattern_text or pattern_math,str)
 end
 
-local pattern = Cs((P("^") / "<sup>" * lpegpatterns.integer * Cc("</sup>") + P(1))^0)
+local pattern = Cs((P("^") / "<sup>" * lpegpatterns.integer * Cc("</sup>") + anything)^0)
 
 function parsers.unittoxml(str)
     return lpegmatch(pattern,str)
@@ -648,3 +648,27 @@ function utilities.parsers.runtime(time)
     local seconds = mod(time,60)
     return days, hours, minutes, seconds
 end
+
+--
+
+local spacing = whitespace^0
+local apply   = P("->")
+local method  = C((1-apply)^1)
+local token   = lbrace * C((1-rbrace)^1) * rbrace + C(anything^1)
+
+local pattern = spacing * (method * spacing * apply + Carg(1)) * spacing * token
+
+function utilities.parsers.splitmethod(str,default)
+    if str then
+        return lpegmatch(pattern,str,1,default or false)
+    else
+        return default or false, ""
+    end
+end
+
+-- print(utilities.parsers.splitmethod(" foo -> {bar} "))
+-- print(utilities.parsers.splitmethod("foo->{bar}"))
+-- print(utilities.parsers.splitmethod("foo->bar"))
+-- print(utilities.parsers.splitmethod("foo"))
+-- print(utilities.parsers.splitmethod("{foo}"))
+-- print(utilities.parsers.splitmethod())
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index 52c48ba..2739a20 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -216,6 +216,7 @@ local striplinepatterns = {
     ["retain"]              = p_retain_normal,
     ["retain and collapse"] = p_retain_collapse,
     ["retain and no empty"] = p_retain_noempty,
+    ["collapse"]            = patterns.collapser, -- how about: stripper fullstripper
 }
 
 strings.striplinepatterns = striplinepatterns
@@ -224,6 +225,8 @@ function strings.striplines(str,how)
     return str and lpegmatch(how and striplinepatterns[how] or p_prune_collapse,str) or str
 end
 
+-- also see: string.collapsespaces
+
 strings.striplong = strings.striplines -- for old times sake
 
 -- local str = table.concat( {
@@ -284,6 +287,7 @@ end
 -- octal            %...o   number
 -- string           %...s   string number
 -- float            %...f   number
+-- checked float    %...F   number
 -- exponential      %...e   number
 -- exponential      %...E   number
 -- autofloat        %...g   number
@@ -524,9 +528,22 @@ local format_f = function(f)
     return format("format('%%%sf',a%s)",f,n)
 end
 
-local format_F = function(f)
+-- The next one formats an integer as integer and very small values as zero. This is needed
+-- for pdf backend code.
+--
+--   1.23 % 1 : 0.23
+-- - 1.23 % 1 : 0.77
+--
+-- We could probably use just %s with integers but who knows what Lua 5.3 will do? So let's
+-- for the moment use %i.
+
+local format_F = function() -- beware, no cast to number
     n = n + 1
-    return format("((a%s == 0 and '0') or (a%s == 1 and '1') or format('%%%sf',a%s))",n,n,f,n)
+    if not f or f == "" then
+        return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or format((a%s %% 1 == 0) and '%%i' or '%%.9f',a%s))",n,n,n,n)
+    else
+        return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n)
+    end
 end
 
 local format_g = function(f)
@@ -1077,3 +1094,23 @@ end
 -- string.formatteds = formatteds
 --
 -- setmetatable(formatteds, { __index = make, __call = use })
+
+-- This is a somewhat silly one used in commandline reconstruction but the older
+-- method, using a combination of fine, gsub, quoted and unquoted was not that
+-- reliable.
+--
+-- '"foo"bar \"and " whatever"' => "foo\"bar \"and \" whatever"
+-- 'foo"bar \"and " whatever'   => "foo\"bar \"and \" whatever"
+
+local dquote = patterns.dquote -- P('"')
+local equote = patterns.escaped + dquote / '\\"' + 1
+local space  = patterns.space
+local cquote = Cc('"')
+
+local pattern =
+    Cs(dquote * (equote - P(-2))^0 * dquote)                    -- we keep the outer but escape unescaped ones
+  + Cs(cquote * (equote - space)^0 * space * equote^0 * cquote) -- we escape unescaped ones
+
+function string.optionalquoted(str)
+    return lpegmatch(pattern,str) or str
+end
diff --git a/whatsnew.lua b/whatsnew.lua
index 0c0ca9b..e9cc378 100644
--- a/whatsnew.lua
+++ b/whatsnew.lua
@@ -67,7 +67,6 @@ local filenames = {
   ["util"] = {
     "deb",
     "dim",
-    "env",
     "jsn",
     "lua",
     "prs",