Merge pull request #23 from phi-gamma/master

sync with Context

12 files changed, 225 insertions, 76 deletions

diff --git a/.gitignore b/.gitignore
index 1fcf771..778d660 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 testing/*
 *-merged.lua
+*.aux
+*.sh
diff --git a/NEWS b/NEWS
index 9b069f6..315fb04 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,7 @@
                         History of the lualibs package
+2013/31/03 v2.1a/
+    * sync with Context beta as of 2013-12-30
+
 2013/11/03 v2.1/
     * sync with Context beta as of 2013-11-01
     * add l-gzip
diff --git a/lualibs-lpeg.lua b/lualibs-lpeg.lua
index 399b3ad..6d3acd7 100644
--- a/lualibs-lpeg.lua
+++ b/lualibs-lpeg.lua
@@ -6,6 +6,10 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+-- lpeg 12 vs lpeg 10: slower compilation, similar parsing speed (i need to check
+-- if i can use new features like capture / 2 and .B (at first sight the xml
+-- parser is some 5% slower)
+
 -- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1)
 
 -- move utf    -> l-unicode
@@ -15,14 +19,15 @@ lpeg = require("lpeg")
 
 -- The latest lpeg doesn't have print any more, and even the new ones are not
 -- available by default (only when debug mode is enabled), which is a pitty as
--- as it helps bailign down bottlenecks. Performance seems comparable, although
+-- as it helps nailign down bottlenecks. Performance seems comparable: some 10%
+-- slower pattern compilation, same parsing speed, although,
 --
 -- local p = lpeg.C(lpeg.P(1)^0 * lpeg.P(-1))
--- local a = string.rep("123",10)
+-- local a = string.rep("123",100)
 -- lpeg.match(p,a)
 --
--- is nearly 20% slower and also still suboptimal (i.e. a match that runs from
--- begin to end, one of the cases where string matchers win).
+-- seems slower and is also still suboptimal (i.e. a match that runs from begin
+-- to end, one of the cases where string matchers win).
 
 if not lpeg.print then function lpeg.print(...) print(lpeg.pcode(...)) end end
 
@@ -74,7 +79,9 @@ local lpegtype, lpegmatch, lpegprint = lpeg.type, lpeg.match, lpeg.print
 
 -- let's start with an inspector:
 
-setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end)
+if setinspector then
+    setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end)
+end
 
 -- Beware, we predefine a bunch of patterns here and one reason for doing so
 -- is that we get consistent behaviour in some of the visualizers.
@@ -469,7 +476,7 @@ end
 -- local pattern1 = P(1-P(pattern))^0 * P(pattern)   : test for not nil
 -- local pattern2 = (P(pattern) * Cc(true) + P(1))^0 : test for true (could be faster, but not much)
 
-function lpeg.finder(lst,makefunction) -- beware: slower than find with 'patternless finds'
+function lpeg.finder(lst,makefunction,isutf) -- beware: slower than find with 'patternless finds'
     local pattern
     if type(lst) == "table" then
         pattern = P(false)
@@ -485,7 +492,12 @@ function lpeg.finder(lst,makefunction) -- beware: slower than find with 'pattern
     else
         pattern = P(lst)
     end
-    pattern = (1-pattern)^0 * pattern
+    if isutf then
+--         pattern = ((utf8char or 1)-pattern)^0 * pattern
+        pattern = ((utf8char or 1)-pattern)^0 * pattern
+    else
+        pattern = (1-pattern)^0 * pattern
+    end
     if makefunction then
         return function(str)
             return lpegmatch(pattern,str)
diff --git a/lualibs-os.lua b/lualibs-os.lua
index 7f3fd7c..bfafa4f 100644
--- a/lualibs-os.lua
+++ b/lualibs-os.lua
@@ -197,9 +197,8 @@ end
 -- no need for function anymore as we have more clever code and helpers now
 -- this metatable trickery might as well disappear
 
-os.resolvers = os.resolvers or { } -- will become private
-
-local resolvers = os.resolvers
+local resolvers = os.resolvers or { }
+os.resolvers    = resolvers
 
 setmetatable(os, { __index = function(t,k)
     local r = resolvers[k]
@@ -222,6 +221,8 @@ local function guess()
     return os.resultof("echo $HOSTTYPE") or ""
 end
 
+-- os.bits = 32 | 64
+
 if platform ~= "" then
 
     os.platform = platform
@@ -230,10 +231,14 @@ elseif os.type == "windows" then
 
     -- we could set the variable directly, no function needed here
 
-    function os.resolvers.platform(t,k)
+    -- PROCESSOR_ARCHITECTURE : binary platform
+    -- PROCESSOR_ARCHITEW6432 : OS platform
+
+    function resolvers.platform(t,k)
         local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or ""
         if find(architecture,"AMD64") then
-            platform = "mswin-64"
+         -- platform = "mswin-64"
+            platform = "win64"
         else
             platform = "mswin"
         end
@@ -244,7 +249,7 @@ elseif os.type == "windows" then
 
 elseif name == "linux" then
 
-    function os.resolvers.platform(t,k)
+    function resolvers.platform(t,k)
         -- we sometimes have HOSTTYPE set so let's check that first
         local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or ""
         if find(architecture,"x86_64") then
@@ -271,7 +276,7 @@ elseif name == "macosx" then
         therefore not permitted to run the 64 bit kernel.
       ]]--
 
-    function os.resolvers.platform(t,k)
+    function resolvers.platform(t,k)
      -- local platform, architecture = "", os.getenv("HOSTTYPE") or ""
      -- if architecture == "" then
      --     architecture = os.resultof("echo $HOSTTYPE") or ""
@@ -294,7 +299,7 @@ elseif name == "macosx" then
 
 elseif name == "sunos" then
 
-    function os.resolvers.platform(t,k)
+    function resolvers.platform(t,k)
         local platform, architecture = "", os.resultof("uname -m") or ""
         if find(architecture,"sparc") then
             platform = "solaris-sparc"
@@ -308,7 +313,7 @@ elseif name == "sunos" then
 
 elseif name == "freebsd" then
 
-    function os.resolvers.platform(t,k)
+    function resolvers.platform(t,k)
         local platform, architecture = "", os.resultof("uname -m") or ""
         if find(architecture,"amd64") then
             platform = "freebsd-amd64"
@@ -322,7 +327,7 @@ elseif name == "freebsd" then
 
 elseif name == "kfreebsd" then
 
-    function os.resolvers.platform(t,k)
+    function resolvers.platform(t,k)
         -- we sometimes have HOSTTYPE set so let's check that first
         local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or ""
         if find(architecture,"x86_64") then
@@ -341,7 +346,7 @@ else
     -- os.setenv("MTX_PLATFORM",platform)
     -- os.platform = platform
 
-    function os.resolvers.platform(t,k)
+    function resolvers.platform(t,k)
         local platform = "linux"
         os.setenv("MTX_PLATFORM",platform)
         os.platform = platform
@@ -350,6 +355,12 @@ else
 
 end
 
+function resolvers.bits(t,k)
+    local bits = find(os.platform,"64") and 64 or 32
+    os.bits = bits
+    return bits
+end
+
 -- beware, we set the randomseed
 
 -- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the
diff --git a/lualibs-package.lua b/lualibs-package.lua
index 0dbff7c..075fcde 100644
--- a/lualibs-package.lua
+++ b/lualibs-package.lua
@@ -274,7 +274,7 @@ methods["qualified path"]=function(name)
 end
 
 methods["lua extra list"] = function(name)
-    return loadedbypath(addsuffix(lualibfile(name),"lua"       ),name,getextraluapaths(),false,"lua")
+    return loadedbypath(addsuffix(lualibfile(name),"lua"),name,getextraluapaths(),false,"lua")
 end
 
 methods["lib extra list"] = function(name)
diff --git a/lualibs-table.lua b/lualibs-table.lua
index 11cb66b..c318c57 100644
--- a/lualibs-table.lua
+++ b/lualibs-table.lua
@@ -129,10 +129,13 @@ local function sortedhash(t,cmp)
             s = sortedkeys(t) -- the robust one
         end
         local n = 0
+        local m = #s
         local function kv(s)
-            n = n + 1
-            local k = s[n]
-            return k, t[k]
+            if n < m then
+                n = n + 1
+                local k = s[n]
+                return k, t[k]
+            end
         end
         return kv, s
     else
@@ -1003,7 +1006,9 @@ function table.print(t,...)
     end
 end
 
-setinspector(function(v) if type(v) == "table" then serialize(print,v,"table") return true end end)
+if setinspector then
+    setinspector(function(v) if type(v) == "table" then serialize(print,v,"table") return true end end)
+end
 
 -- -- -- obsolete but we keep them for a while and might comment them later -- -- --
 
@@ -1054,3 +1059,24 @@ function table.sorted(t,...)
     sort(t,...)
     return t -- still sorts in-place
 end
+
+--
+
+function table.values(t,s) -- optional sort flag
+    if t then
+        local values, keys, v = { }, { }, 0
+        for key, value in next, t do
+            if not keys[value] then
+                v = v + 1
+                values[v] = value
+                keys[k] = key
+            end
+        end
+        if s then
+            sort(values)
+        end
+        return values
+    else
+        return { }
+    end
+end
diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua
index 79cbdba..802f2e6 100644
--- a/lualibs-trac-inf.lua
+++ b/lualibs-trac-inf.lua
@@ -122,6 +122,9 @@ function statistics.show()
     if statistics.enable then
         -- this code will move
         local register = statistics.register
+        register("used platform", function()
+            return format("%s, type: %s, binary subtree: %s",os.platform or "unknown",os.type or "unknown", environment.texos or "unknown")
+        end)
         register("luatex banner", function()
             return lower(status.banner)
         end)
@@ -179,7 +182,7 @@ function statistics.timed(action)
     starttiming("run")
     action()
     stoptiming("run")
-    report("total runtime: %s",elapsedtime("run"))
+    report("total runtime: %s seconds",elapsedtime("run"))
 end
 
 -- goodie
diff --git a/lualibs-unicode.lua b/lualibs-unicode.lua
index 7ada394..6601a4c 100644
--- a/lualibs-unicode.lua
+++ b/lualibs-unicode.lua
@@ -9,7 +9,6 @@ if not modules then modules = { } end modules ['l-unicode'] = {
 -- this module will be reorganized
 
 -- todo: utf.sub replacement (used in syst-aux)
-
 -- we put these in the utf namespace:
 
 utf = utf or (unicode and unicode.utf8) or { }
@@ -935,19 +934,27 @@ end
 local _, l_remap = utf.remapper(little)
 local _, b_remap = utf.remapper(big)
 
-function utf.utf8_to_utf16_be(str)
-    return char(254,255) .. lpegmatch(b_remap,str)
+function utf.utf8_to_utf16_be(str,nobom)
+    if nobom then
+        return lpegmatch(b_remap,str)
+    else
+        return char(254,255) .. lpegmatch(b_remap,str)
+    end
 end
 
-function utf.utf8_to_utf16_le(str)
-    return char(255,254) .. lpegmatch(l_remap,str)
+function utf.utf8_to_utf16_le(str,nobom)
+    if nobom then
+        return lpegmatch(l_remap,str)
+    else
+        return char(255,254) .. lpegmatch(l_remap,str)
+    end
 end
 
-function utf.utf8_to_utf16(str,littleendian)
+function utf.utf8_to_utf16(str,littleendian,nobom)
     if littleendian then
-        return utf.utf8_to_utf16_le(str)
+        return utf.utf8_to_utf16_le(str,nobom)
     else
-        return utf.utf8_to_utf16_be(str)
+        return utf.utf8_to_utf16_be(str,nobom)
     end
 end
 
@@ -1122,3 +1129,13 @@ if not utf.values then
     string.utfvalues = utf.values
 
 end
+
+function utf.chrlen(u) -- u is number
+    return
+        (u < 0x80 and 1) or
+        (u < 0xE0 and 2) or
+        (u < 0xF0 and 3) or
+        (u < 0xF8 and 4) or
+        (u < 0xFC and 5) or
+        (u < 0xFE and 6) or 0
+end
diff --git a/lualibs-url.lua b/lualibs-url.lua
index 7b7910f..7bb7312 100644
--- a/lualibs-url.lua
+++ b/lualibs-url.lua
@@ -26,6 +26,8 @@ local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replac
 --    |   ___________|____________                              |
 --   / \ /                        \                             |
 --   urn:example:animal:ferret:nose               interpretable as extension
+--
+-- also nice: http://url.spec.whatwg.org/ (maybe some day ...)
 
 url       = url or { }
 local url = url
@@ -43,7 +45,7 @@ local hexdigit    = R("09","AF","af")
 local plus        = P("+")
 local nothing     = Cc("")
 local escapedchar = (percent * C(hexdigit * hexdigit)) / tochar
-local escaped     = (plus / " ") + escapedchar
+local escaped     = (plus / " ") + escapedchar -- so no loc://foo++.tex
 
 local noslash     = P("/") / ""
 
@@ -189,7 +191,11 @@ local function hashed(str) -- not yet ok (/test?test)
     return s
 end
 
--- inspect(hashed("template://test"))
+-- inspect(hashed("template:///test"))
+-- inspect(hashed("template:///test++.whatever"))
+-- inspect(hashed("template:///test%2B%2B.whatever"))
+-- inspect(hashed("template:///test%x.whatever"))
+-- inspect(hashed("tem%2Bplate:///test%x.whatever"))
 
 -- Here we assume:
 --
diff --git a/lualibs-util-env.lua b/lualibs-util-env.lua
index 1184c1d..0a708ea 100644
--- a/lualibs-util-env.lua
+++ b/lualibs-util-env.lua
@@ -42,7 +42,7 @@ local basicengines = allocate {
  -- ["texluajit.exe"] = "luajittex",
 }
 
-local luaengines=allocate {
+local luaengines = allocate {
     ["lua"]    = true,
     ["luajit"] = true,
 }
@@ -66,6 +66,8 @@ elseif luaengines[file.removesuffix(arg[-1])] then
 --         arg[k-1] = arg[k]
 --     end
 --     remove(arg) -- last
+--
+--    environment.used_as_library = true
 elseif validengines[file.removesuffix(arg[0])] then
     if arg[1] == "--luaonly" then
         arg[-1] = arg[0]
diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua
index 29a57e0..e5b35a7 100644
--- a/lualibs-util-prs.lua
+++ b/lualibs-util-prs.lua
@@ -184,8 +184,10 @@ function parsers.settings_to_array(str,strict)
         else
             return { str }
         end
-    else
+    elseif find(str,",") then
         return lpegmatch(pattern,str)
+    else
+        return { str }
     end
 end
 
@@ -483,8 +485,8 @@ function parsers.rfc4180splitter(specification)
                       * Cs((dquotechar + (1 - quotechar))^0)
                       * quotechar
     local non_escaped = C((1 - quotechar - newline - separator)^1)
-    local field       = escaped + non_escaped
-    local record      = Ct((field * separator^-1)^1)
+    local field       = escaped + non_escaped + Cc("")
+    local record      = Ct(field * (separator * field)^1)
     local headerline  = record * Cp()
     local wholeblob   = Ct((newline^-1 * record)^0)
     return function(data,getheader)
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index 24a3f6e..f04f0e5 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -20,8 +20,16 @@ local utfchar, utfbyte = utf.char, utf.byte
 ----- loadstripped = utilities.lua.loadstripped
 ----- setmetatableindex = table.setmetatableindex
 
-local loadstripped = _LUAVERSION < 5.2 and load or function(str)
-    return load(dump(load(str),true)) -- it only makes sense in luajit and luatex where we have a stipped load
+-- local loadstripped = _LUAVERSION < 5.2 and load or function(str)
+--     return load(dump(load(str),true)) -- it only makes sense in luajit and luatex where we have a stipped load
+-- end
+
+local loadstripped = function(str,shortcuts)
+    if shortcuts then
+        return load(dump(load(str),true),nil,nil,shortcuts)
+    else
+        return load(dump(load(str),true))
+    end
 end
 
 -- todo: make a special namespace for the formatter
@@ -291,33 +299,69 @@ function number.sparseexponent(f,n)
     return tostring(n)
 end
 
-local preamble = [[
-local type = type
-local tostring = tostring
-local tonumber = tonumber
-local format = string.format
-local concat = table.concat
-local signed = number.signed
-local points = number.points
-local basepoints = number.basepoints
-local utfchar = utf.char
-local utfbyte = utf.byte
-local lpegmatch = lpeg.match
-local nspaces = string.nspaces
-local tracedchar = string.tracedchar
-local autosingle = string.autosingle
-local autodouble = string.autodouble
-local sequenced = table.sequenced
-local formattednumber = number.formatted
-local sparseexponent = number.sparseexponent
-]]
-
 local template = [[
 %s
 %s
 return function(%s) return %s end
 ]]
 
+-- local environment = {
+--     lpeg   = lpeg,
+--     type   = type,
+--     string = string,
+--     number = number,
+--     table  = table,
+--     utf    = utf,
+-- }
+--
+-- local preamble = [[
+-- local type            = type
+-- local tostring        = tostring
+-- local tonumber        = tonumber
+-- local format          = string.format
+-- local concat          = table.concat
+-- local signed          = number.signed
+-- local points          = number.points
+-- local basepoints      = number.basepoints
+-- local utfchar         = utf.char
+-- local utfbyte         = utf.byte
+-- local lpegmatch       = lpeg.match
+-- local nspaces         = string.nspaces
+-- local tracedchar      = string.tracedchar
+-- local autosingle      = string.autosingle
+-- local autodouble      = string.autodouble
+-- local sequenced       = table.sequenced
+-- local formattednumber = number.formatted
+-- local sparseexponent  = number.sparseexponent
+-- ]]
+
+local environment = {
+    global          = global or _G,
+    lpeg            = lpeg,
+    type            = type,
+    tostring        = tostring,
+    tonumber        = tonumber,
+    format          = string.format,
+    concat          = table.concat,
+    signed          = number.signed,
+    points          = number.points,
+    basepoints      = number.basepoints,
+    utfchar         = utf.char,
+    utfbyte         = utf.byte,
+    lpegmatch       = lpeg.match,
+    nspaces         = string.nspaces,
+    tracedchar      = string.tracedchar,
+    autosingle      = string.autosingle,
+    autodouble      = string.autodouble,
+    sequenced       = table.sequenced,
+    formattednumber = number.formatted,
+    sparseexponent  = number.sparseexponent,
+}
+
+local preamble = ""
+
+-- -- --
+
 local arguments = { "a1" } -- faster than previously used (select(n,...))
 
 setmetatable(arguments, { __index =
@@ -647,7 +691,7 @@ local format_extension = function(extensions,f,name)
     end
 end
 
--- aA b cC d eE f gG hH iI jJ lL mM N o p qQ r sS tT uU wW xX
+-- aA b cC d eE f gG hH iI jJ lL mM N o p qQ r sS tT uU wW xX z
 
 local builder = Cs { "start",
     start = (
@@ -740,28 +784,37 @@ local builder = Cs { "start",
 
 -- we can be clever and only alias what is needed
 
+-- local direct = Cs (
+--         P("%")/""
+--       * Cc([[local format = string.format return function(str) return format("%]])
+--       * (S("+- .") + R("09"))^0
+--       * S("sqidfgGeExXo")
+--       * Cc([[",str) end]])
+--       * P(-1)
+--     )
+
 local direct = Cs (
-        P("%")/""
-      * Cc([[local format = string.format return function(str) return format("%]])
-      * (S("+- .") + R("09"))^0
-      * S("sqidfgGeExXo")
-      * Cc([[",str) end]])
-      * P(-1)
-    )
+    P("%")
+  * (S("+- .") + R("09"))^0
+  * S("sqidfgGeExXo")
+  * P(-1) / [[local format = string.format return function(str) return format("%0",str) end]]
+)
 
 local function make(t,str)
     local f
     local p
     local p = lpegmatch(direct,str)
     if p then
+     -- f = loadstripped(p)()
+     -- print("builder 1 >",p)
         f = loadstripped(p)()
     else
         n = 0
         p = lpegmatch(builder,str,1,"..",t._extensions_) -- after this we know n
         if n > 0 then
             p = format(template,preamble,t._preamble_,arguments[n],p)
---         print("builder>",p)
-            f = loadstripped(p)()
+         -- print("builder 2 >",p)
+            f = loadstripped(p,t._environment_)() -- t._environment is not populated (was experiment)
         else
             f = function() return str end
         end
@@ -817,7 +870,11 @@ strings.formatters = { }
 -- clear that table when a threshold is reached
 
 function strings.formatters.new()
-    local t = { _extensions_ = { }, _preamble_ = "", _type_ = "formatter" }
+    local e = { } -- better make a copy as we can overload
+    for k, v in next, environment do
+        e[k] = v
+    end
+    local t = { _extensions_ = { }, _preamble_ = "", _environment_ = e, _type_ = "formatter" }
     setmetatable(t, { __index = make, __call = use })
     return t
 end
@@ -838,8 +895,12 @@ string.formatter   = function(str,...) return formatters[str](...) end -- someti
 local function add(t,name,template,preamble)
     if type(t) == "table" and t._type_ == "formatter" then
         t._extensions_[name] = template or "%s"
-        if preamble then
+        if type(preamble) == "string" then
             t._preamble_ = preamble .. "\n" .. t._preamble_ -- so no overload !
+        elseif type(preamble) == "table" then
+            for k, v in next, preamble do
+                t._environment_[k] = v
+            end
         end
     end
 end
@@ -856,9 +917,13 @@ patterns.luaquoted = Cs(Cc('"') * ((1-S('"\n'))^1 + P('"')/'\\"' + P('\n')/'\\n"
 -- escaping by lpeg is faster for strings without quotes, slower on a string with quotes, but
 -- faster again when other q-escapables are found (the ones we don't need to escape)
 
-add(formatters,"xml", [[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]])
-add(formatters,"tex", [[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]])
-add(formatters,"lua", [[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]])
+-- add(formatters,"xml", [[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]])
+-- add(formatters,"tex", [[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]])
+-- add(formatters,"lua", [[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]])
+
+add(formatters,"xml", [[lpegmatch(xmlescape,%s)]],{ xmlescape = lpeg.patterns.xmlescape })
+add(formatters,"tex", [[lpegmatch(texescape,%s)]],{ texescape = lpeg.patterns.texescape })
+add(formatters,"lua", [[lpegmatch(luaescape,%s)]],{ luaescape = lpeg.patterns.luaescape })
 
 -- -- yes or no:
 --