sync with Context as of 2014-03-16

9 files changed, 195 insertions, 55 deletions

diff --git a/lualibs-dir.lua b/lualibs-dir.lua
index b658b7c..2572120 100644
--- a/lualibs-dir.lua
+++ b/lualibs-dir.lua
@@ -27,7 +27,7 @@ local currentdir = lfs.currentdir
 local chdir      = lfs.chdir
 local mkdir      = lfs.mkdir
 
-local onwindows  = os.type == "windows" or find(os.getenv("PATH"),";")
+local onwindows  = os.type == "windows" or find(os.getenv("PATH"),";",1,true)
 
 -- in case we load outside luatex
 
@@ -189,7 +189,7 @@ local function glob(str,t)
             local split = lpegmatch(pattern,str) -- we could use the file splitter
             if split then
                 local root, path, base = split[1], split[2], split[3]
-                local recurse = find(base,"%*%*")
+                local recurse = find(base,"**",1,true) -- find(base,"%*%*")
                 local start = root .. path
                 local result = lpegmatch(filter,start .. base)
                 globpattern(start,result,recurse,t)
@@ -215,7 +215,7 @@ local function glob(str,t)
                 local t = t or { }
                 local action = action or function(name) t[#t+1] = name end
                 local root, path, base = split[1], split[2], split[3]
-                local recurse = find(base,"%*%*")
+                local recurse =  find(base,"**",1,true) -- find(base,"%*%*")
                 local start = root .. path
                 local result = lpegmatch(filter,start .. base)
                 globpattern(start,result,recurse,action)
@@ -296,7 +296,6 @@ if onwindows then
             str = ""
             for i=1,n do
                 local s = select(i,...)
-                local s = select(i,...)
                 if s == "" then
                     -- skip
                 elseif str == "" then
diff --git a/lualibs-io.lua b/lualibs-io.lua
index 52f166a..020e811 100644
--- a/lualibs-io.lua
+++ b/lualibs-io.lua
@@ -12,7 +12,7 @@ local concat = table.concat
 local floor = math.floor
 local type = type
 
-if string.find(os.getenv("PATH"),";") then
+if string.find(os.getenv("PATH"),";",1,true) then
     io.fileseparator, io.pathseparator = "\\", ";"
 else
     io.fileseparator, io.pathseparator = "/" , ":"
diff --git a/lualibs-os.lua b/lualibs-os.lua
index bfafa4f..1dff79c 100644
--- a/lualibs-os.lua
+++ b/lualibs-os.lua
@@ -137,7 +137,7 @@ function os.resultof(command)
 end
 
 if not io.fileseparator then
-    if find(os.getenv("PATH"),";") then
+    if find(os.getenv("PATH"),";",1,true) then
         io.fileseparator, io.pathseparator, os.type = "\\", ";", os.type or "mswin"
     else
         io.fileseparator, io.pathseparator, os.type = "/" , ":", os.type or "unix"
@@ -236,7 +236,7 @@ elseif os.type == "windows" then
 
     function resolvers.platform(t,k)
         local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or ""
-        if find(architecture,"AMD64") then
+        if find(architecture,"AMD64",1,true) then
          -- platform = "mswin-64"
             platform = "win64"
         else
@@ -252,9 +252,9 @@ elseif name == "linux" then
     function resolvers.platform(t,k)
         -- we sometimes have HOSTTYPE set so let's check that first
         local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or ""
-        if find(architecture,"x86_64") then
+        if find(architecture,"x86_64",1,true) then
             platform = "linux-64"
-        elseif find(architecture,"ppc") then
+        elseif find(architecture,"ppc",1,true) then
             platform = "linux-ppc"
         else
             platform = "linux"
@@ -285,9 +285,9 @@ elseif name == "macosx" then
         if architecture == "" then
          -- print("\nI have no clue what kind of OSX you're running so let's assume an 32 bit intel.\n")
             platform = "osx-intel"
-        elseif find(architecture,"i386") then
+        elseif find(architecture,"i386",1,true) then
             platform = "osx-intel"
-        elseif find(architecture,"x86_64") then
+        elseif find(architecture,"x86_64",1,true) then
             platform = "osx-64"
         else
             platform = "osx-ppc"
@@ -301,7 +301,7 @@ elseif name == "sunos" then
 
     function resolvers.platform(t,k)
         local platform, architecture = "", os.resultof("uname -m") or ""
-        if find(architecture,"sparc") then
+        if find(architecture,"sparc",1,true) then
             platform = "solaris-sparc"
         else -- if architecture == 'i86pc'
             platform = "solaris-intel"
@@ -315,7 +315,7 @@ elseif name == "freebsd" then
 
     function resolvers.platform(t,k)
         local platform, architecture = "", os.resultof("uname -m") or ""
-        if find(architecture,"amd64") then
+        if find(architecture,"amd64",1,true) then
             platform = "freebsd-amd64"
         else
             platform = "freebsd"
@@ -330,7 +330,7 @@ elseif name == "kfreebsd" then
     function resolvers.platform(t,k)
         -- we sometimes have HOSTTYPE set so let's check that first
         local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or ""
-        if find(architecture,"x86_64") then
+        if find(architecture,"x86_64",1,true) then
             platform = "kfreebsd-amd64"
         else
             platform = "kfreebsd-i386"
@@ -356,7 +356,7 @@ else
 end
 
 function resolvers.bits(t,k)
-    local bits = find(os.platform,"64") and 64 or 32
+    local bits = find(os.platform,"64",1,true) and 64 or 32
     os.bits = bits
     return bits
 end
diff --git a/lualibs-table.lua b/lualibs-table.lua
index c318c57..d231830 100644
--- a/lualibs-table.lua
+++ b/lualibs-table.lua
@@ -88,6 +88,38 @@ local function sortedkeys(tab)
     end
 end
 
+local function sortedhashonly(tab)
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if type(key) == "string" then
+                s = s + 1
+                srt[s] = key
+            end
+        end
+        sort(srt)
+        return srt
+    else
+        return { }
+    end
+end
+
+local function sortedindexonly(tab)
+    if tab then
+        local srt, s = { }, 0
+        for key,_ in next, tab do
+            if type(key) == "number" then
+                s = s + 1
+                srt[s] = key
+            end
+        end
+        sort(srt)
+        return srt
+    else
+        return { }
+    end
+end
+
 local function sortedhashkeys(tab,cmp) -- fast one
     if tab then
         local srt, s = { }, 0
@@ -114,8 +146,10 @@ function table.allkeys(t)
     return sortedkeys(keys)
 end
 
-table.sortedkeys     = sortedkeys
-table.sortedhashkeys = sortedhashkeys
+table.sortedkeys      = sortedkeys
+table.sortedhashonly  = sortedhashonly
+table.sortedindexonly = sortedindexonly
+table.sortedhashkeys  = sortedhashkeys
 
 local function nothing() end
 
diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua
index 067cff2..034726f 100644
--- a/lualibs-trac-inf.lua
+++ b/lualibs-trac-inf.lua
@@ -12,7 +12,7 @@ if not modules then modules = { } end modules ['trac-inf'] = {
 -- and rawget.
 
 local type, tonumber, select = type, tonumber, select
-local format, lower = string.format, string.lower
+local format, lower, find = string.format, string.lower, string.find
 local concat = table.concat
 local clock = os.gettimeofday or os.clock -- should go in environment
 
@@ -123,10 +123,8 @@ function statistics.show()
         -- this code will move
         local register = statistics.register
         register("used platform", function()
-            local mask = lua.mask or "ascii"
-            return format("%s, type: %s, binary subtree: %s, symbol mask: %s (%s)",
-                os.platform or "unknown",os.type or "unknown", environment.texos or "unknown",
-                mask,mask == "utf" and "τεχ" or "tex")
+            return format("%s, type: %s, binary subtree: %s",
+                os.platform or "unknown",os.type or "unknown", environment.texos or "unknown")
         end)
         register("luatex banner", function()
             return lower(status.banner)
@@ -139,16 +137,25 @@ function statistics.show()
             return format("%s direct, %s indirect, %s total", total-indirect, indirect, total)
         end)
         if jit then
-            local status = { jit.status() }
-            if status[1] then
-                register("luajit status", function()
-                    return concat(status," ",2)
-                end)
+            local jitstatus = { jit.status() }
+            if jitstatus[1] then
+                register("luajit options", concat(jitstatus," ",2))
             end
         end
         -- so far
      -- collectgarbage("collect")
-        register("current memory usage",statistics.memused)
+        register("lua properties",function()
+            local list = status.list()
+            local hashchar = tonumber(list.luatex_hashchars)
+            local mask = lua.mask or "ascii"
+            return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)",
+                jit and "luajit" or "lua",
+                statistics.memused(),
+                list.luatex_hashtype or "default",
+                hashchar and 2^hashchar or "unknown",
+                mask,
+                mask == "utf" and "τεχ" or "tex")
+        end)
         register("runtime",statistics.runtime)
         logs.newline() -- initial newline
         for i=1,#statusinfo do
diff --git a/lualibs-util-env.lua b/lualibs-util-env.lua
index 0a708ea..e96a464 100644
--- a/lualibs-util-env.lua
+++ b/lualibs-util-env.lua
@@ -197,7 +197,7 @@ function environment.reconstructcommandline(arg,noquote)
             a = resolvers.resolve(a)
             a = unquoted(a)
             a = gsub(a,'"','\\"') -- tricky
-            if find(a," ") then
+            if find(a," ",1,true) then
                 result[#result+1] = quoted(a)
             else
                 result[#result+1] = a
diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua
index e5b35a7..2cede91 100644
--- a/lualibs-util-prs.lua
+++ b/lualibs-util-prs.lua
@@ -179,12 +179,12 @@ function parsers.settings_to_array(str,strict)
     elseif not str or str == "" then
         return { }
     elseif strict then
-        if find(str,"{") then
+        if find(str,"{",1,true) then
             return lpegmatch(pattern,str)
         else
             return { str }
         end
-    elseif find(str,",") then
+    elseif find(str,",",1,true) then
         return lpegmatch(pattern,str)
     else
         return { str }
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index 4ecaed7..578f4d0 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -152,17 +152,105 @@ end
 --     print(strings.tabtospace(t[k]))
 -- end
 
-function strings.striplong(str) -- strips all leading spaces
-    str = gsub(str,"^%s*","")
-    str = gsub(str,"[\n\r]+ *","\n")
-    return str
+-- todo: lpeg
+
+-- function strings.striplong(str) -- strips all leading spaces
+--     str = gsub(str,"^%s*","")
+--     str = gsub(str,"[\n\r]+ *","\n")
+--     return str
+-- end
+
+local newline     = patterns.newline
+local endofstring = patterns.endofstring
+local whitespace  = patterns.whitespace
+local spacer      = patterns.spacer
+
+local space       = spacer^0
+local nospace     = space/""
+local endofline   = nospace * newline
+
+local stripend    = (whitespace^1 * endofstring)/""
+
+local normalline  = (nospace * ((1-space*(newline+endofstring))^1) * nospace)
+
+local stripempty  = endofline^1/""
+local normalempty = endofline^1
+local singleempty = endofline * (endofline^0/"")
+local doubleempty = endofline * endofline^-1 * (endofline^0/"")
+
+local stripstart  = stripempty^0
+
+local p_prune_normal    = Cs ( stripstart * ( stripend + normalline + normalempty )^0 )
+local p_prune_collapse  = Cs ( stripstart * ( stripend + normalline + doubleempty )^0 )
+local p_prune_noempty   = Cs ( stripstart * ( stripend + normalline + singleempty )^0 )
+local p_retain_normal   = Cs (              (            normalline + normalempty )^0 )
+local p_retain_collapse = Cs (              (            normalline + doubleempty )^0 )
+local p_retain_noempty  = Cs (              (            normalline + singleempty )^0 )
+
+-- function striplines(str,prune,collapse,noempty)
+--     if prune then
+--         if noempty then
+--             return lpegmatch(p_prune_noempty,str) or str
+--         elseif collapse then
+--             return lpegmatch(p_prune_collapse,str) or str
+--         else
+--             return lpegmatch(p_prune_normal,str) or str
+--         end
+--     else
+--         if noempty then
+--             return lpegmatch(p_retain_noempty,str) or str
+--         elseif collapse then
+--             return lpegmatch(p_retain_collapse,str) or str
+--         else
+--             return lpegmatch(p_retain_normal,str) or str
+--         end
+--     end
+-- end
+
+local striplinepatterns = {
+    ["prune"]               = p_prune_normal,
+    ["prune and collapse"]  = p_prune_collapse, -- default
+    ["prune and no empty"]  = p_prune_noempty,
+    ["retain"]              = p_retain_normal,
+    ["retain and collapse"] = p_retain_collapse,
+    ["retain and no empty"] = p_retain_noempty,
+}
+
+strings.striplinepatterns = striplinepatterns
+
+function strings.striplines(str,how)
+    return str and lpegmatch(how and striplinepatterns[how] or p_prune_collapse,str) or str
 end
 
--- local template = string.striplong([[
+strings.striplong = strings.striplines -- for old times sake
+
+-- local str = table.concat( {
+-- "  ",
+-- "    aap",
+-- "  noot mies",
+-- "  ",
+-- "    ",
+-- " zus    wim jet",
+-- "zus    wim jet",
+-- "       zus    wim jet",
+-- "    ",
+-- }, "\n")
+
+-- local str = table.concat( {
+-- "  aaaa",
+-- "  bb",
+-- "  cccccc",
+-- }, "\n")
+
+-- for k, v in table.sortedhash(utilities.strings.striplinepatterns) do
+--     logs.report("stripper","method: %s, result: [[%s]]",k,utilities.strings.striplines(str,k))
+-- end
+
+-- inspect(strings.striplong([[
 --   aaaa
 --   bb
 --   cccccc
--- ]])
+-- ]]))
 
 function strings.nice(str)
     str = gsub(str,"[:%-+_]+"," ") -- maybe more
@@ -418,7 +506,7 @@ local format_i = function(f)
     if f and f ~= "" then
         return format("format('%%%si',a%s)",f,n)
     else
-        return format("format('%%i',a%s)",n)
+        return format("format('%%i',a%s)",n) -- why not just tostring()
     end
 end
 
@@ -434,6 +522,11 @@ local format_f = function(f)
     return format("format('%%%sf',a%s)",f,n)
 end
 
+local format_F = function(f)
+    n = n + 1
+    return format("((a%s == 0 and '0') or (a%s == 1 and '1') or format('%%%sf',a%s))",n,n,f,n)
+end
+
 local format_g = function(f)
     n = n + 1
     return format("format('%%%sg',a%s)",f,n)
@@ -707,7 +800,7 @@ local builder = Cs { "start",
                 V("!") -- new
               + V("s") + V("q")
               + V("i") + V("d")
-              + V("f") + V("g") + V("G") + V("e") + V("E")
+              + V("f") + V("F") + V("g") + V("G") + V("e") + V("E")
               + V("x") + V("X") + V("o")
               --
               + V("c")
@@ -742,6 +835,7 @@ local builder = Cs { "start",
     ["i"] = (prefix_any * P("i")) / format_i, -- %i => regular %i (integer)
     ["d"] = (prefix_any * P("d")) / format_d, -- %d => regular %d (integer)
     ["f"] = (prefix_any * P("f")) / format_f, -- %f => regular %f (float)
+    ["F"] = (prefix_any * P("F")) / format_F, -- %F => regular %f (float) but 0/1 check
     ["g"] = (prefix_any * P("g")) / format_g, -- %g => regular %g (float)
     ["G"] = (prefix_any * P("G")) / format_G, -- %G => regular %G (float)
     ["e"] = (prefix_any * P("e")) / format_e, -- %e => regular %e (float)
@@ -816,7 +910,8 @@ local function make(t,str)
         f = loadstripped(p)()
     else
         n = 0
-        p = lpegmatch(builder,str,1,"..",t._extensions_) -- after this we know n
+     -- p = lpegmatch(builder,str,1,"..",t._extensions_) -- after this we know n
+        p = lpegmatch(builder,str,1,t._connector_,t._extensions_) -- after this we know n
         if n > 0 then
             p = format(template,preamble,t._preamble_,arguments[n],p)
          -- print("builder 2 >",p)
@@ -875,22 +970,24 @@ strings.formatters = { }
 -- table (metatable) in which case we could better keep a count and
 -- clear that table when a threshold is reached
 
+-- _connector_ is an experiment
+
 if _LUAVERSION < 5.2  then
 
-    function strings.formatters.new()
-        local t = { _extensions_ = { }, _preamble_ = preamble, _environment_ = { }, _type_ = "formatter" }
+    function strings.formatters.new(noconcat)
+        local t = { _type_ = "formatter", _connector_ = noconcat and "," or "..", _extensions_ = { }, _preamble_ = preamble, _environment_ = { } }
         setmetatable(t, { __index = make, __call = use })
         return t
     end
 
 else
 
-    function strings.formatters.new()
+    function strings.formatters.new(noconcat)
         local e = { } -- better make a copy as we can overload
         for k, v in next, environment do
             e[k] = v
         end
-        local t = { _extensions_ = { }, _preamble_ = "", _environment_ = e, _type_ = "formatter" }
+        local t = { _type_ = "formatter", _connector_ = noconcat and "," or "..", _extensions_ = { }, _preamble_ = "", _environment_ = e }
         setmetatable(t, { __index = make, __call = use })
         return t
     end
diff --git a/lualibs-util-tab.lua b/lualibs-util-tab.lua
index d235520..f9e9b31 100644
--- a/lualibs-util-tab.lua
+++ b/lualibs-util-tab.lua
@@ -21,27 +21,29 @@ local utftoeight = utf.toeight
 
 local splitter = lpeg.tsplitat(".")
 
-function tables.definetable(target,nofirst,nolast) -- defines undefined tables
-    local composed, shortcut, t = nil, nil, { }
+function utilities.tables.definetable(target,nofirst,nolast) -- defines undefined tables
+    local composed, t = nil, {  }
     local snippets = lpegmatch(splitter,target)
     for i=1,#snippets - (nolast and 1 or 0) do
         local name = snippets[i]
         if composed then
-            composed = shortcut .. "." .. name
-            shortcut = shortcut .. "_" .. name
-            t[#t+1] = formatters["local %s = %s if not %s then %s = { } %s = %s end"](shortcut,composed,shortcut,shortcut,composed,shortcut)
+            composed = composed .. "." .. name
+                t[#t+1] = formatters["if not %s then %s = { } end"](composed,composed)
         else
             composed = name
-            shortcut = name
             if not nofirst then
                 t[#t+1] = formatters["%s = %s or { }"](composed,composed)
             end
         end
     end
-    if nolast then
-        composed = shortcut .. "." .. snippets[#snippets]
+    if composed then
+        if nolast then
+            composed = composed .. "." .. snippets[#snippets]
+        end
+        return concat(t,"\n"), composed -- could be shortcut
+    else
+        return "", target
     end
-    return concat(t,"\n"), composed
 end
 
 -- local t = tables.definedtable("a","b","c","d")
@@ -73,7 +75,7 @@ end
 
 function tables.migratetable(target,v,root)
     local t = root or _G
-    local names = string.split(target,".")
+    local names = lpegmatch(splitter,target)
     for i=1,#names-1 do
         local name = names[i]
         t[name] = t[name] or { }
@@ -493,7 +495,8 @@ end
 
 -- The next version is somewhat faster, although in practice one will seldom
 -- serialize a lot using this one. Often the above variants are more efficient.
--- If we would really need this a lot, we could hash q keys.
+-- If we would really need this a lot, we could hash q keys, or just not used
+-- indented code.
 
 -- char-def.lua : 0.53 -> 0.38
 -- husayni.tma  : 0.28 -> 0.19