sync with Context as of 2013-07-14

12 files changed, 122 insertions, 45 deletions

diff --git a/lualibs-file.lua b/lualibs-file.lua
index a64ee86..ebb2b39 100644
--- a/lualibs-file.lua
+++ b/lualibs-file.lua
@@ -368,11 +368,14 @@ function file.joinpath(tab,separator) -- table
     return tab and concat(tab,separator or io.pathseparator) -- can have trailing //
 end
 
+local someslash = S("\\/")
 local stripper  = Cs(P(fwslash)^0/"" * reslasher)
-local isnetwork = fwslash * fwslash * (1-fwslash) + (1-fwslash-colon)^1 * colon
+local isnetwork = someslash * someslash * (1-someslash)
+                + (1-fwslash-colon)^1 * colon
 local isroot    = fwslash^1 * -1
 local hasroot   = fwslash^1
 
+local reslasher = lpeg.replacer(S("\\/"),"/")
 local deslasher = lpeg.replacer(S("\\/")^1,"/")
 
 -- If we have a network or prefix then there is a change that we end up with two
@@ -386,8 +389,13 @@ function file.join(...)
     local lst = { ... }
     local one = lst[1]
     if lpegmatch(isnetwork,one) then
+        local one = lpegmatch(reslasher,one)
         local two = lpegmatch(deslasher,concat(lst,"/",2))
-        return one .. "/" .. two
+        if lpegmatch(hasroot,two) then
+            return one .. two
+        else
+            return one .. "/" .. two
+        end
     elseif lpegmatch(isroot,one) then
         local two = lpegmatch(deslasher,concat(lst,"/",2))
         if lpegmatch(hasroot,two) then
@@ -412,6 +420,8 @@ end
 -- print(file.join("http://a","/y"))
 -- print(file.join("http:///a","/y"))
 -- print(file.join("//nas-1","/y"))
+-- print(file.join("//nas-1/a/b/c","/y"))
+-- print(file.join("\\\\nas-1\\a\\b\\c","\\y"))
 
 -- The previous one fails on "a.b/c"  so Taco came up with a split based
 -- variant. After some skyping we got it sort of compatible with the old
@@ -421,9 +431,14 @@ end
 -- finds were replaced by lpegs.
 
 local drivespec    = R("az","AZ")^1 * colon
-local anchors      = fwslash + drivespec
-local untouched    = periods + (1-period)^1 * P(-1)
-local splitstarter = (Cs(drivespec * (bwslash/"/" + fwslash)^0) + Cc(false)) * Ct(lpeg.splitat(S("/\\")^1))
+local anchors      = fwslash
+                   + drivespec
+local untouched    = periods
+                   + (1-period)^1 * P(-1)
+local mswindrive   = Cs(drivespec * (bwslash/"/" + fwslash)^0)
+local mswinuncpath = (bwslash + fwslash) * (bwslash + fwslash) * Cc("//")
+local splitstarter = (mswindrive + mswinuncpath + Cc(false))
+                   * Ct(lpeg.splitat(S("/\\")^1))
 local absolute     = fwslash
 
 function file.collapsepath(str,anchor) -- anchor: false|nil, true, "."
@@ -490,6 +505,7 @@ end
 -- test("a/./b/..") test("a/aa/../b/bb") test("a/.././././b/..") test("a/./././b/..")
 -- test("a/b/c/../..") test("./a/b/c/../..") test("a/b/c/../..")
 -- test("./a")
+-- test([[\\a.b.c\d\e]])
 
 local validchars = R("az","09","AZ","--","..")
 local pattern_a  = lpeg.replacer(1-validchars)
diff --git a/lualibs-io.lua b/lualibs-io.lua
index 06e1fb5..e3a443b 100644
--- a/lualibs-io.lua
+++ b/lualibs-io.lua
@@ -35,6 +35,7 @@ local function readall(f)
         return f:read('*all')
     else
         local done = f:seek("set",0)
+        local step
         if size < 1024*1024 then
             step = 1024 * 1024
         elseif size > 16*1024*1024 then
diff --git a/lualibs-lpeg.lua b/lualibs-lpeg.lua
index 7be86d3..b33df96 100644
--- a/lualibs-lpeg.lua
+++ b/lualibs-lpeg.lua
@@ -13,6 +13,19 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
 
 lpeg = require("lpeg")
 
+-- The latest lpeg doesn't have print any more, and even the new ones are not
+-- available by default (only when debug mode is enabled), which is a pitty as
+-- as it helps bailign down bottlenecks. Performance seems comparable, although
+--
+-- local p = lpeg.C(lpeg.P(1)^0 * lpeg.P(-1))
+-- local a = string.rep("123",10)
+-- lpeg.match(p,a)
+--
+-- is nearly 20% slower and also still suboptimal (i.e. a match that runs from
+-- begin to end, one of the cases where string matchers win).
+
+if not lpeg.print then function lpeg.print(...) print(lpeg.pcode(...)) end end
+
 -- tracing (only used when we encounter a problem in integration of lpeg in luatex)
 
 -- some code will move to unicode and string
@@ -212,7 +225,7 @@ patterns.propername    = (uppercase + lowercase + underscore) * (uppercase + low
 patterns.somecontent   = (anything - newline - space)^1 -- (utf8char - newline - space)^1
 patterns.beginline     = #(1-newline)
 
-patterns.longtostring  = Cs(whitespace^0/"" * nonwhitespace^0 * ((whitespace^0/" " * (patterns.quoted + nonwhitespace)^1)^0))
+patterns.longtostring  = Cs(whitespace^0/"" * ((patterns.quoted + nonwhitespace^1 + whitespace^1/"" * (P(-1) + Cc(" ")))^0))
 
 local function anywhere(pattern) --slightly adapted from website
     return P { P(pattern) + 1 * V(1) }
diff --git a/lualibs-table.lua b/lualibs-table.lua
index 4f6b9b4..11cb66b 100644
--- a/lualibs-table.lua
+++ b/lualibs-table.lua
@@ -346,6 +346,7 @@ local noquotes, hexify, handle, reduce, compact, inline, functions
 local reserved = table.tohash { -- intercept a language inconvenience: no reserved words as key
     'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if',
     'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while',
+    'NaN', 'goto',
 }
 
 local function simple_table(t)
diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua
index eefc15a..79cbdba 100644
--- a/lualibs-trac-inf.lua
+++ b/lualibs-trac-inf.lua
@@ -11,20 +11,24 @@ if not modules then modules = { } end modules ['trac-inf'] = {
 -- get warnings about assignments. This is more efficient than using rawset
 -- and rawget.
 
-local type, tonumber = type, tonumber
+local type, tonumber, select = type, tonumber, select
 local format, lower = string.format, string.lower
 local concat = table.concat
 local clock = os.gettimeofday or os.clock -- should go in environment
 
-statistics       = statistics or { }
-local statistics = statistics
+local setmetatableindex = table.setmetatableindex
+local serialize         = table.serialize
+local formatters        = string.formatters
 
-statistics.enable    = true
-statistics.threshold = 0.01
+statistics              = statistics or { }
+local statistics        = statistics
+
+statistics.enable       = true
+statistics.threshold    = 0.01
 
 local statusinfo, n, registered, timers = { }, 0, { }, { }
 
-table.setmetatableindex(timers,function(t,k)
+setmetatableindex(timers,function(t,k)
     local v = { timing = 0, loadtime = 0 }
     t[k] = v
     return v
@@ -178,6 +182,19 @@ function statistics.timed(action)
     report("total runtime: %s",elapsedtime("run"))
 end
 
+-- goodie
+
+function statistics.tracefunction(base,tag,...)
+    for i=1,select("#",...) do
+        local name = select(i,...)
+        local stat = { }
+        local func = base[name]
+        setmetatableindex(stat,function(t,k) t[k] = 0 return 0 end)
+        base[name] = function(n,k,v) stat[k] = stat[k] + 1 return func(n,k,v) end
+        statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end)
+    end
+end
+
 -- where, not really the best spot for this:
 
 commands = commands or { }
diff --git a/lualibs-util-dim.lua b/lualibs-util-dim.lua
index 47b2706..6906149 100644
--- a/lualibs-util-dim.lua
+++ b/lualibs-util-dim.lua
@@ -22,6 +22,8 @@ local allocate          = utilities.storage.allocate
 local setmetatableindex = table.setmetatableindex
 local formatters        = string.formatters
 
+local texget            = tex and tex.get or function() return 65536*10*100 end
+
 --this might become another namespace
 
 number = number or { }
@@ -137,7 +139,7 @@ capture takes place.</p>
 --ldx]]--
 
 local amount = (S("+-")^0 * R("09")^0 * P(".")^0 * R("09")^0) + Cc("0")
-local unit   = R("az")^1
+local unit   = R("az")^1 + P("%")
 
 local dimenpair = amount/tonumber * (unit^1/dimenfactors + Cc(1)) -- tonumber is new
 
@@ -376,10 +378,10 @@ function dimen(a)
                 a = k
             else
                 local value, unit = lpegmatch(dimenpair,a)
-                if type(unit) == "function" then
-                    k = value/unit()
+                if value and unit then
+                    k = value/unit -- to be considered: round
                 else
-                    k = value/unit
+                    k = 0
                 end
                 known[a] = k
                 a = k
@@ -412,16 +414,16 @@ function string.todimen(str) -- maybe use tex.sp when available
     end
 end
 
---~ local known = { }
-
---~ function string.todimen(str) -- maybe use tex.sp
---~     local k = known[str]
---~     if not k then
---~         k = tex.sp(str)
---~         known[str] = k
---~     end
---~     return k
---~ end
+-- local known = { }
+--
+-- function string.todimen(str) -- maybe use tex.sp
+--     local k = known[str]
+--     if not k then
+--         k = tex.sp(str)
+--         known[str] = k
+--     end
+--     return k
+-- end
 
 stringtodimen = string.todimen -- local variable defined earlier
 
@@ -439,7 +441,7 @@ probably use a hash instead of a one-element table.</p>
 --ldx]]--
 
 function number.percent(n,d) -- will be cleaned up once luatex 0.30 is out
-    d = d or tex.hsize
+    d = d or texget("hsize")
     if type(d) == "string" then
         d = stringtodimen(d)
     end
diff --git a/lualibs-util-jsn.lua b/lualibs-util-jsn.lua
index 29587cd..bbe25d8 100644
--- a/lualibs-util-jsn.lua
+++ b/lualibs-util-jsn.lua
@@ -42,8 +42,20 @@ local dquote     = P('"')
 local whitespace = lpeg.patterns.whitespace
 local optionalws = whitespace^0
 
-local escape     = C(P("\\u") / "0x" * S("09","AF","af")) / function(s) return utfchar(tonumber(s)) end
-local jstring    = dquote * Cs((escape + (1-dquote))^0) * dquote
+local escapes    = {
+ -- ["\\"] = "\\",  -- lua will escape these
+ -- ["/"]  = "/",   -- no need to escape this one
+    ["b"]  = "\010",
+    ["f"]  = "\014",
+    ["n"]  = "\n",
+    ["r"]  = "\r",
+    ["t"]  = "\t",
+}
+
+local escape_un  = C(P("\\u") / "0x" * S("09","AF","af")) / function(s) return utfchar(tonumber(s)) end
+local escape_bs  = P([[\]]) / "" * (P(1) / escapes) -- if not found then P(1) is returned i.e. the to be escaped char
+
+local jstring    = dquote * Cs((escape_un + escape_bs + (1-dquote))^0) * dquote
 local jtrue      = P("true")  * Cc(true)
 local jfalse     = P("false") * Cc(false)
 local jnull      = P("null")  * Cc(nil)
diff --git a/lualibs-util-lua.lua b/lualibs-util-lua.lua
index 61d1190..e1dcdc9 100644
--- a/lualibs-util-lua.lua
+++ b/lualibs-util-lua.lua
@@ -74,9 +74,16 @@ end
 function luautilities.loadedluacode(fullname,forcestrip,name)
     -- quite subtle ... doing this wrong incidentally can give more bytes
     name = name or fullname
-    local code = environment.loadpreprocessedfile and environment.loadpreprocessedfile(fullname) or loadfile(fullname)
+    local code, message
+    if environment.loadpreprocessedfile then
+        code, message = environment.loadpreprocessedfile(fullname)
+    else
+        code, message = loadfile(fullname)
+    end
     if code then
         code()
+    else
+        report_lua("loading of file %a failed:\n\t%s",fullname,message or "no message")
     end
     if forcestrip and luautilities.stripcode then
         if type(forcestrip) == "function" then
@@ -97,15 +104,16 @@ function luautilities.loadedluacode(fullname,forcestrip,name)
 end
 
 function luautilities.strippedloadstring(code,forcestrip,name) -- not executed
+    local code, message = load(code)
+    if not code then
+        report_lua("loading of file %a failed:\n\t%s",name,message or "no message")
+    end
     if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then
-        code = load(code)
-        if not code then
-            report_lua("fatal error %a in file %a",3,name)
-        end
         register(name)
-        code = dump(code,true)
+        return load(dump(code,true)), 0 -- not yet executes
+    else
+        return code, 0
     end
-    return load(code), 0
 end
 
 function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true
diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua
index 9b2a2b0..7fe1e70 100644
--- a/lualibs-util-prs.lua
+++ b/lualibs-util-prs.lua
@@ -410,7 +410,7 @@ function parsers.csvsplitter(specification)
         end
         whatever = quotedata + whatever
     end
-    local parser = Ct((Ct(whatever * (separator * whatever)^0) * S("\n\r"))^0 )
+    local parser = Ct((Ct(whatever * (separator * whatever)^0) * S("\n\r")^1)^0 )
     return function(data)
         return lpegmatch(parser,data)
     end
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index 0c8c0e2..13e1e09 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -339,7 +339,7 @@ local format_i = function(f)
     if f and f ~= "" then
         return format("format('%%%si',a%s)",f,n)
     else
-        return format("a%s",n)
+        return format("format('%%i',a%s)",n)
     end
 end
 
diff --git a/lualibs-util-tab.lua b/lualibs-util-tab.lua
index b7db4fa..f18c719 100644
--- a/lualibs-util-tab.lua
+++ b/lualibs-util-tab.lua
@@ -293,13 +293,12 @@ function tables.encapsulate(core,capsule,protect)
     end
 end
 
--- we no longer have %q in keys as for this kind of tables we can
--- assume sane keys
+-- best keep [%q] keys (as we have some in older applications i.e. saving user data
 
-local f_hashed_string   = formatters["[%s]=%q,"]
-local f_hashed_number   = formatters["[%s]=%s,"]
-local f_hashed_boolean  = formatters["[%s]=%l,"]
-local f_hashed_table    = formatters["[%s]="]
+local f_hashed_string   = formatters["[%q]=%q,"]
+local f_hashed_number   = formatters["[%q]=%s,"]
+local f_hashed_boolean  = formatters["[%q]=%l,"]
+local f_hashed_table    = formatters["[%q]="]
 
 local f_indexed_string  = formatters["%q,"]
 local f_indexed_number  = formatters["%s,"]
@@ -521,7 +520,7 @@ local serialize = table.serialize -- the extensive one, the one we started with
 
 function table.serialize(root,name,specification)
 
-    if specification then
+    if type(specification) == "table" then
         return serialize(root,name,specification) -- the original one
     end
 
diff --git a/lualibs-util-tpl.lua b/lualibs-util-tpl.lua
index dcc4c12..e0c405a 100644
--- a/lualibs-util-tpl.lua
+++ b/lualibs-util-tpl.lua
@@ -159,6 +159,14 @@ end
 
 templates.replace = replace
 
+function templates.replacer(str,how,recurse) -- reads nicer
+    return function(mapping)
+        return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str
+    end
+end
+
+-- local cmd = templates.replacer([[foo %bar%]]) print(cmd { bar = "foo" })
+
 function templates.load(filename,mapping,how,recurse)
     local data = io.loaddata(filename) or ""
     if mapping and next(mapping) then