sync with Context as of 2015-05-04

9 files changed, 209 insertions, 122 deletions

diff --git a/lualibs-lpeg.lua b/lualibs-lpeg.lua
index 192e32f..55a0d89 100644
--- a/lualibs-lpeg.lua
+++ b/lualibs-lpeg.lua
@@ -10,6 +10,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
 -- if i can use new features like capture / 2 and .B (at first sight the xml
 -- parser is some 5% slower)
 
+-- lpeg.P("abc") is faster than lpeg.P("a") * lpeg.P("b") * lpeg.P("c")
+
 -- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1)
 
 -- move utf    -> l-unicode
@@ -19,7 +21,7 @@ lpeg = require("lpeg")
 
 -- The latest lpeg doesn't have print any more, and even the new ones are not
 -- available by default (only when debug mode is enabled), which is a pitty as
--- as it helps nailign down bottlenecks. Performance seems comparable: some 10%
+-- as it helps nailing down bottlenecks. Performance seems comparable: some 10%
 -- slower pattern compilation, same parsing speed, although,
 --
 -- local p = lpeg.C(lpeg.P(1)^0 * lpeg.P(-1))
@@ -834,121 +836,185 @@ end
 
 -- experiment:
 
--- local function make(t)
---     local p
---     local keys = sortedkeys(t)
---     for i=1,#keys do
---         local k = keys[i]
---         local v = t[k]
---         if not p then
---             if next(v) then
---                 p = P(k) * make(v)
---             else
---                 p = P(k)
---             end
---         else
---             if next(v) then
---                 p = p + P(k) * make(v)
---             else
---                 p = p + P(k)
---             end
---         end
---     end
---     return p
--- end
-
--- local function make(t)
---     local p = P(false)
---     local keys = sortedkeys(t)
---     for i=1,#keys do
---         local k = keys[i]
---         local v = t[k]
---         if next(v) then
---             p = p + P(k) * make(v)
---         else
---             p = p + P(k)
---         end
---     end
---     return p
--- end
-
--- function lpeg.utfchartabletopattern(list) -- goes to util-lpg
---     local tree = { }
---     for i=1,#list do
---         local t = tree
---         for c in gmatch(list[i],".") do
---             local tc = t[c]
---             if not tc then
---                 tc = { }
---                 t[c] = tc
---             end
---             t = tc
---         end
---     end
---     return make(tree)
--- end
+local p_false = P(false)
+local p_true  = P(true)
 
-local function make(t,hash)
-    local p = P(false)
+local function make(t)
+    local function making(t)
+        local p    = p_false
+        local keys = sortedkeys(t)
+        for i=1,#keys do
+            local k = keys[i]
+            if k ~= "" then
+                local v = t[k]
+                if v == true then
+                    p = p + P(k) * p_true
+                elseif v == false then
+                    -- can't happen
+                else
+                    p = p + P(k) * making(v)
+                end
+            end
+        end
+        if t[""] then
+            p = p + p_true
+        end
+        return p
+    end
+    local p    = p_false
     local keys = sortedkeys(t)
     for i=1,#keys do
         local k = keys[i]
-        local v = t[k]
-        local h = hash[v]
-        if h then
-            if next(v) then
-                p = p + P(k) * (make(v,hash) + P(true))
+        if k ~= "" then
+            local v = t[k]
+            if v == true then
+                p = p + P(k) * p_true
+            elseif v == false then
+                -- can't happen
             else
-                p = p + P(k) * P(true)
+                p = p + P(k) * making(v)
             end
-        else
-            if next(v) then
-                p = p + P(k) * make(v,hash)
+        end
+    end
+    return p
+end
+
+local function collapse(t,x)
+    if type(t) ~= "table" then
+        return t, x
+    else
+        local n = next(t)
+        if n == nil then
+            return t, x
+        elseif next(t,n) == nil then
+            -- one entry
+            local k = n
+            local v = t[k]
+            if type(v) == "table" then
+                return collapse(v,x..k)
             else
-                p = p + P(k)
+                return v, x .. k
+            end
+        else
+            local tt = { }
+            for k, v in next, t do
+                local vv, kk = collapse(v,k)
+                tt[kk] = vv
             end
+            return tt, x
         end
     end
-    return p
 end
 
 function lpeg.utfchartabletopattern(list) -- goes to util-lpg
     local tree = { }
-    local hash = { }
     local n = #list
     if n == 0 then
-        -- we could always use this branch
         for s in next, list do
             local t = tree
+            local p, pk
             for c in gmatch(s,".") do
-                local tc = t[c]
-                if not tc then
-                    tc = { }
-                    t[c] = tc
+                if t == true then
+                    t = { [c] = true, [""] = true }
+                    p[pk] = t
+                    p = t
+                    t = false
+                elseif t == false then
+                    t = { [c] = false }
+                    p[pk] = t
+                    p = t
+                    t = false
+                else
+                    local tc = t[c]
+                    if not tc then
+                        tc = false
+                        t[c] = false
+                    end
+                    p = t
+                    t = tc
                 end
-                t = tc
+                pk = c
+            end
+            if t == false then
+                p[pk] = true
+            elseif t == true then
+                -- okay
+            else
+                t[""] = true
             end
-            hash[t] = s
         end
     else
         for i=1,n do
-            local t = tree
             local s = list[i]
+            local t = tree
+            local p, pk
             for c in gmatch(s,".") do
-                local tc = t[c]
-                if not tc then
-                    tc = { }
-                    t[c] = tc
+                if t == true then
+                    t = { [c] = true, [""] = true }
+                    p[pk] = t
+                    p = t
+                    t = false
+                elseif t == false then
+                    t = { [c] = false }
+                    p[pk] = t
+                    p = t
+                    t = false
+                else
+                    local tc = t[c]
+                    if not tc then
+                        tc = false
+                        t[c] = false
+                    end
+                    p = t
+                    t = tc
                 end
-                t = tc
+                pk = c
+            end
+            if t == false then
+                p[pk] = true
+            elseif t == true then
+                -- okay
+            else
+                t[""] = true
             end
-            hash[t] = s
         end
     end
-    return make(tree,hash)
+--     collapse(tree,"") -- needs testing, maybe optional, slightly faster because P("x")*P("X") seems slower than P"(xX") (why)
+--     inspect(tree)
+    return make(tree)
 end
 
--- inspect ( lpeg.utfchartabletopattern {
+-- local t = { "start", "stoep", "staart", "paard" }
+-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/string.upper + 1)^1)
+
+-- local t = { "a", "abc", "ac", "abe", "abxyz", "xy", "bef","aa" }
+-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/string.upper + 1)^1)
+
+-- inspect(lpegmatch(p,"a"))
+-- inspect(lpegmatch(p,"aa"))
+-- inspect(lpegmatch(p,"aaaa"))
+-- inspect(lpegmatch(p,"ac"))
+-- inspect(lpegmatch(p,"bc"))
+-- inspect(lpegmatch(p,"zzbczz"))
+-- inspect(lpegmatch(p,"zzabezz"))
+-- inspect(lpegmatch(p,"ab"))
+-- inspect(lpegmatch(p,"abc"))
+-- inspect(lpegmatch(p,"abe"))
+-- inspect(lpegmatch(p,"xa"))
+-- inspect(lpegmatch(p,"bx"))
+-- inspect(lpegmatch(p,"bax"))
+-- inspect(lpegmatch(p,"abxyz"))
+-- inspect(lpegmatch(p,"foobarbefcrap"))
+
+-- local t = { ["^"] = 1, ["^^"] = 2, ["^^^"] = 3, ["^^^^"] = 4 }
+-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/t + 1)^1)
+-- inspect(lpegmatch(p," ^ ^^ ^^^ ^^^^ ^^^^^ ^^^^^^ ^^^^^^^ "))
+
+-- local t = { ["^^"] = 2, ["^^^"] = 3, ["^^^^"] = 4 }
+-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/t + 1)^1)
+-- inspect(lpegmatch(p," ^ ^^ ^^^ ^^^^ ^^^^^ ^^^^^^ ^^^^^^^ "))
+
+-- lpeg.utfchartabletopattern {
 --     utfchar(0x00A0), -- nbsp
 --     utfchar(0x2000), -- enquad
 --     utfchar(0x2001), -- emquad
@@ -964,7 +1030,7 @@ end
 --     utfchar(0x200B), -- zerowidthspace
 --     utfchar(0x202F), -- narrownobreakspace
 --     utfchar(0x205F), -- math thinspace
--- } )
+-- }
 
 -- a few handy ones:
 --
diff --git a/lualibs-math.lua b/lualibs-math.lua
index 43f60b5..ec62919 100644
--- a/lualibs-math.lua
+++ b/lualibs-math.lua
@@ -8,6 +8,10 @@ if not modules then modules = { } end modules ['l-math'] = {
 
 local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan
 
+if not math.ceiling then
+    math.ceiling = math.ceil
+end
+
 if not math.round then
     function math.round(x) return floor(x + 0.5) end
 end
diff --git a/lualibs-os.lua b/lualibs-os.lua
index f44b316..0a86ea6 100644
--- a/lualibs-os.lua
+++ b/lualibs-os.lua
@@ -349,6 +349,8 @@ else
 
 end
 
+os.newline = name == "windows" and "\013\010" or "\010" -- crlf or lf
+
 function resolvers.bits(t,k)
     local bits = find(os.platform,"64",1,true) and 64 or 32
     os.bits = bits
diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua
index 034726f..5497e54 100644
--- a/lualibs-trac-inf.lua
+++ b/lualibs-trac-inf.lua
@@ -207,17 +207,3 @@ function statistics.tracefunction(base,tag,...)
         statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end)
     end
 end
-
--- where, not really the best spot for this:
-
-commands = commands or { }
-
-function commands.resettimer(name)
-    resettiming(name or "whatever")
-    starttiming(name or "whatever")
-end
-
-function commands.elapsedtime(name)
-    stoptiming(name or "whatever")
-    context(elapsedtime(name or "whatever"))
-end
diff --git a/lualibs-unicode.lua b/lualibs-unicode.lua
index 02dd1a0..70b6032 100644
--- a/lualibs-unicode.lua
+++ b/lualibs-unicode.lua
@@ -418,9 +418,11 @@ if not utf.sub then
         end
     end
 
-    local pattern_zero = Cmt(p_utf8char,slide_zero)^0
-    local pattern_one  = Cmt(p_utf8char,slide_one )^0
-    local pattern_two  = Cmt(p_utf8char,slide_two )^0
+    local pattern_zero  = Cmt(p_utf8char,slide_zero)^0
+    local pattern_one   = Cmt(p_utf8char,slide_one )^0
+    local pattern_two   = Cmt(p_utf8char,slide_two )^0
+
+    local pattern_first = C(patterns.utf8character)
 
     function utf.sub(str,start,stop)
         if not start then
@@ -463,7 +465,9 @@ if not utf.sub then
                 end
             end
         end
-        if start > stop then
+        if start == 1 and stop == 1 then
+            return lpegmatch(pattern_first,str) or ""
+        elseif start > stop then
             return ""
         elseif start > 1 then
             b, e, n, first, last = 0, 0, 0, start - 1, stop
diff --git a/lualibs-util-dim.lua b/lualibs-util-dim.lua
index bfffb10..2bdb870 100644
--- a/lualibs-util-dim.lua
+++ b/lualibs-util-dim.lua
@@ -92,9 +92,9 @@ format (string) is implemented using this table.</p>
 --ldx]]--
 
 local f_none = formatters["%s%s"]
-local f_true = formatters["%0.5f%s"]
+local f_true = formatters["%0.5F%s"]
 
-local function numbertodimen(n,unit,fmt)
+local function numbertodimen(n,unit,fmt) -- will be redefined later !
     if type(n) == 'string' then
         return n
     else
diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua
index 302b984..ea34c2c 100644
--- a/lualibs-util-prs.lua
+++ b/lualibs-util-prs.lua
@@ -333,7 +333,7 @@ end
 
 -- for mtx-context etc: aaaa bbbb cccc=dddd eeee=ffff
 
-local str      = C((1-whitespace-equal)^1)
+local str      = Cs(lpegpatterns.unquoted) + C((1-whitespace-equal)^1)
 local setting  = Cf( Carg(1) * (whitespace^0 * Cg(str * whitespace^0 * (equal * whitespace^0 * str + Cc(""))))^1,rawset)
 local splitter = setting^1
 
@@ -341,6 +341,12 @@ function utilities.parsers.options_to_hash(str,target)
     return str and lpegmatch(splitter,str,1,target or { }) or { }
 end
 
+local splitter = lpeg.tsplitat(" ")
+
+function utilities.parsers.options_to_array(str)
+    return str and lpegmatch(splitter,str) or { }
+end
+
 -- for chem (currently one level)
 
 local value     = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace)
@@ -524,7 +530,7 @@ function parsers.rfc4180splitter(specification)
     local field       = escaped + non_escaped + Cc("")
     local record      = Ct(field * (separator * field)^1)
     local headerline  = record * Cp()
-    local wholeblob   = Ct((newline^-1 * record)^0)
+    local wholeblob   = Ct((newline^(specification.strict and -1 or 1) * record)^0)
     return function(data,getheader)
         if getheader then
             local header, position = lpegmatch(headerline,data)
diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua
index a677a82..de4a87e 100644
--- a/lualibs-util-str.lua
+++ b/lualibs-util-str.lua
@@ -44,7 +44,12 @@ end
 
 if not number then number = { } end -- temp hack for luatex-fonts
 
-local stripper = patterns.stripzeros
+local stripper    = patterns.stripzeros
+local newline     = patterns.newline
+local endofstring = patterns.endofstring
+local whitespace  = patterns.whitespace
+local spacer      = patterns.spacer
+local spaceortab  = patterns.spaceortab
 
 local function points(n)
     n = tonumber(n)
@@ -62,12 +67,12 @@ number.basepoints = basepoints
 -- str = " \n \ntest  \n test\ntest "
 -- print("["..string.gsub(string.collapsecrlf(str),"\n","+").."]")
 
-local rubish     = patterns.spaceortab^0 * patterns.newline
-local anyrubish  = patterns.spaceortab + patterns.newline
+local rubish     = spaceortab^0 * newline
+local anyrubish  = spaceortab + newline
 local anything   = patterns.anything
-local stripped   = (patterns.spaceortab^1 / "") * patterns.newline
+local stripped   = (spaceortab^1 / "") * newline
 local leading    = rubish^0 / ""
-local trailing   = (anyrubish^1 * patterns.endofstring) / ""
+local trailing   = (anyrubish^1 * endofstring) / ""
 local redundant  = rubish^3 / "\n"
 
 local pattern = Cs(leading * (trailing + redundant + stripped + anything)^0)
@@ -129,7 +134,7 @@ local pattern =
               return ""
           end
       end
-    + patterns.newline * Cp() / function(position)
+    + newline * Cp() / function(position)
           extra, start = 0, position
       end
     + patterns.anything
@@ -162,11 +167,6 @@ end
 --     return str
 -- end
 
-local newline     = patterns.newline
-local endofstring = patterns.endofstring
-local whitespace  = patterns.whitespace
-local spacer      = patterns.spacer
-
 local space       = spacer^0
 local nospace     = space/""
 local endofline   = nospace * newline
@@ -1117,3 +1117,9 @@ local pattern =
 function string.optionalquoted(str)
     return lpegmatch(pattern,str) or str
 end
+
+local pattern = Cs((newline / os.newline + 1)^0)
+
+function string.replacenewlines(str)
+    return lpegmatch(pattern,str)
+end
diff --git a/lualibs-util-tpl.lua b/lualibs-util-tpl.lua
index bd0e261..468dd42 100644
--- a/lualibs-util-tpl.lua
+++ b/lualibs-util-tpl.lua
@@ -128,6 +128,11 @@ local function replacekeyquoted(s,t,how,recurse) -- ".. \" "
     end
 end
 
+local function replaceoptional(l,m,r,t,how,recurse)
+    local v = t[l]
+    return v and v ~= "" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or ""
+end
+
 local single      = P("%")  -- test %test% test     : resolves test
 local double      = P("%%") -- test 10%% test       : %% becomes %
 local lquoted     = P("%[") -- test '%[test]%' test : resolves to test with escaped "'s
@@ -143,12 +148,19 @@ local norquoted   = rquoted  / ''
 local nolquotedq  = lquotedq / ''
 local norquotedq  = rquotedq / ''
 
-local key         = nosingle   * ((C((1-nosingle  )^1) * Carg(1) * Carg(2) * Carg(3)) / replacekey        ) * nosingle
-local quoted      = nolquotedq * ((C((1-norquotedq)^1) * Carg(1) * Carg(2) * Carg(3)) / replacekeyquoted  ) * norquotedq
-local unquoted    = nolquoted  * ((C((1-norquoted )^1) * Carg(1) * Carg(2) * Carg(3)) / replacekeyunquoted) * norquoted
+local noloptional = P("%?") / ''
+local noroptional = P("?%") / ''
+local nomoptional = P(":")  / ''
+
+
+local args        = Carg(1) * Carg(2) * Carg(3)
+local key         = nosingle    * ((C((1-nosingle   )^1) * args) / replacekey        ) * nosingle
+local quoted      = nolquotedq  * ((C((1-norquotedq )^1) * args) / replacekeyquoted  ) * norquotedq
+local unquoted    = nolquoted   * ((C((1-norquoted  )^1) * args) / replacekeyunquoted) * norquoted
+local optional    = noloptional * ((C((1-nomoptional)^1) * nomoptional * C((1-noroptional)^1) * args) / replaceoptional) *  noroptional
 local any         = P(1)
 
-      replacer    = Cs((unquoted + quoted + escape + key + any)^0)
+      replacer    = Cs((unquoted + quoted + escape + optional + key + any)^0)
 
 local function replace(str,mapping,how,recurse)
     if mapping and str then
@@ -164,6 +176,7 @@ end
 -- print(replace("test '%[x]%' test",{ x = [[a '%y%'  a]], y = "oeps" },'sql',true))
 -- print(replace([[test %[x]% test]],{ x = [[a "x"  a]]}))
 -- print(replace([[test %(x)% test]],{ x = [[a "x"  a]]}))
+-- print(replace([[convert %?x: -x "%x%" ?% %?y: -y "%y%" ?%]],{ x = "yes" }))
 
 templates.replace = replace