From a79c845dba381f543d09526f77dcf2b4687b73a7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Mon, 4 May 2015 23:10:29 +0200 Subject: sync with Context as of 2015-05-04 --- lualibs-lpeg.lua | 236 ++++++++++++++++++++++++++++++++------------------- lualibs-math.lua | 4 + lualibs-os.lua | 2 + lualibs-trac-inf.lua | 14 --- lualibs-unicode.lua | 12 ++- lualibs-util-dim.lua | 4 +- lualibs-util-prs.lua | 10 ++- lualibs-util-str.lua | 28 +++--- lualibs-util-tpl.lua | 21 ++++- 9 files changed, 209 insertions(+), 122 deletions(-) diff --git a/lualibs-lpeg.lua b/lualibs-lpeg.lua index 192e32f..55a0d89 100644 --- a/lualibs-lpeg.lua +++ b/lualibs-lpeg.lua @@ -10,6 +10,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = { -- if i can use new features like capture / 2 and .B (at first sight the xml -- parser is some 5% slower) +-- lpeg.P("abc") is faster than lpeg.P("a") * lpeg.P("b") * lpeg.P("c") + -- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) -- move utf -> l-unicode @@ -19,7 +21,7 @@ lpeg = require("lpeg") -- The latest lpeg doesn't have print any more, and even the new ones are not -- available by default (only when debug mode is enabled), which is a pitty as --- as it helps nailign down bottlenecks. Performance seems comparable: some 10% +-- as it helps nailing down bottlenecks. Performance seems comparable: some 10% -- slower pattern compilation, same parsing speed, although, -- -- local p = lpeg.C(lpeg.P(1)^0 * lpeg.P(-1)) @@ -834,121 +836,185 @@ end -- experiment: --- local function make(t) --- local p --- local keys = sortedkeys(t) --- for i=1,#keys do --- local k = keys[i] --- local v = t[k] --- if not p then --- if next(v) then --- p = P(k) * make(v) --- else --- p = P(k) --- end --- else --- if next(v) then --- p = p + P(k) * make(v) --- else --- p = p + P(k) --- end --- end --- end --- return p --- end - --- local function make(t) --- local p = P(false) --- local keys = sortedkeys(t) --- for i=1,#keys do --- local k = keys[i] --- local v = t[k] --- if next(v) then --- p = p + P(k) * make(v) --- else --- p = p + P(k) --- end --- end --- return p --- end - --- function lpeg.utfchartabletopattern(list) -- goes to util-lpg --- local tree = { } --- for i=1,#list do --- local t = tree --- for c in gmatch(list[i],".") do --- local tc = t[c] --- if not tc then --- tc = { } --- t[c] = tc --- end --- t = tc --- end --- end --- return make(tree) --- end +local p_false = P(false) +local p_true = P(true) -local function make(t,hash) - local p = P(false) +local function make(t) + local function making(t) + local p = p_false + local keys = sortedkeys(t) + for i=1,#keys do + local k = keys[i] + if k ~= "" then + local v = t[k] + if v == true then + p = p + P(k) * p_true + elseif v == false then + -- can't happen + else + p = p + P(k) * making(v) + end + end + end + if t[""] then + p = p + p_true + end + return p + end + local p = p_false local keys = sortedkeys(t) for i=1,#keys do local k = keys[i] - local v = t[k] - local h = hash[v] - if h then - if next(v) then - p = p + P(k) * (make(v,hash) + P(true)) + if k ~= "" then + local v = t[k] + if v == true then + p = p + P(k) * p_true + elseif v == false then + -- can't happen else - p = p + P(k) * P(true) + p = p + P(k) * making(v) end - else - if next(v) then - p = p + P(k) * make(v,hash) + end + end + return p +end + +local function collapse(t,x) + if type(t) ~= "table" then + return t, x + else + local n = next(t) + if n == nil then + return t, x + elseif next(t,n) == nil then + -- one entry + local k = n + local v = t[k] + if type(v) == "table" then + return collapse(v,x..k) else - p = p + P(k) + return v, x .. k + end + else + local tt = { } + for k, v in next, t do + local vv, kk = collapse(v,k) + tt[kk] = vv end + return tt, x end end - return p end function lpeg.utfchartabletopattern(list) -- goes to util-lpg local tree = { } - local hash = { } local n = #list if n == 0 then - -- we could always use this branch for s in next, list do local t = tree + local p, pk for c in gmatch(s,".") do - local tc = t[c] - if not tc then - tc = { } - t[c] = tc + if t == true then + t = { [c] = true, [""] = true } + p[pk] = t + p = t + t = false + elseif t == false then + t = { [c] = false } + p[pk] = t + p = t + t = false + else + local tc = t[c] + if not tc then + tc = false + t[c] = false + end + p = t + t = tc end - t = tc + pk = c + end + if t == false then + p[pk] = true + elseif t == true then + -- okay + else + t[""] = true end - hash[t] = s end else for i=1,n do - local t = tree local s = list[i] + local t = tree + local p, pk for c in gmatch(s,".") do - local tc = t[c] - if not tc then - tc = { } - t[c] = tc + if t == true then + t = { [c] = true, [""] = true } + p[pk] = t + p = t + t = false + elseif t == false then + t = { [c] = false } + p[pk] = t + p = t + t = false + else + local tc = t[c] + if not tc then + tc = false + t[c] = false + end + p = t + t = tc end - t = tc + pk = c + end + if t == false then + p[pk] = true + elseif t == true then + -- okay + else + t[""] = true end - hash[t] = s end end - return make(tree,hash) +-- collapse(tree,"") -- needs testing, maybe optional, slightly faster because P("x")*P("X") seems slower than P"(xX") (why) +-- inspect(tree) + return make(tree) end --- inspect ( lpeg.utfchartabletopattern { +-- local t = { "start", "stoep", "staart", "paard" } +-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/string.upper + 1)^1) + +-- local t = { "a", "abc", "ac", "abe", "abxyz", "xy", "bef","aa" } +-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/string.upper + 1)^1) + +-- inspect(lpegmatch(p,"a")) +-- inspect(lpegmatch(p,"aa")) +-- inspect(lpegmatch(p,"aaaa")) +-- inspect(lpegmatch(p,"ac")) +-- inspect(lpegmatch(p,"bc")) +-- inspect(lpegmatch(p,"zzbczz")) +-- inspect(lpegmatch(p,"zzabezz")) +-- inspect(lpegmatch(p,"ab")) +-- inspect(lpegmatch(p,"abc")) +-- inspect(lpegmatch(p,"abe")) +-- inspect(lpegmatch(p,"xa")) +-- inspect(lpegmatch(p,"bx")) +-- inspect(lpegmatch(p,"bax")) +-- inspect(lpegmatch(p,"abxyz")) +-- inspect(lpegmatch(p,"foobarbefcrap")) + +-- local t = { ["^"] = 1, ["^^"] = 2, ["^^^"] = 3, ["^^^^"] = 4 } +-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/t + 1)^1) +-- inspect(lpegmatch(p," ^ ^^ ^^^ ^^^^ ^^^^^ ^^^^^^ ^^^^^^^ ")) + +-- local t = { ["^^"] = 2, ["^^^"] = 3, ["^^^^"] = 4 } +-- local p = lpeg.Cs((lpeg.utfchartabletopattern(t)/t + 1)^1) +-- inspect(lpegmatch(p," ^ ^^ ^^^ ^^^^ ^^^^^ ^^^^^^ ^^^^^^^ ")) + +-- lpeg.utfchartabletopattern { -- utfchar(0x00A0), -- nbsp -- utfchar(0x2000), -- enquad -- utfchar(0x2001), -- emquad @@ -964,7 +1030,7 @@ end -- utfchar(0x200B), -- zerowidthspace -- utfchar(0x202F), -- narrownobreakspace -- utfchar(0x205F), -- math thinspace --- } ) +-- } -- a few handy ones: -- diff --git a/lualibs-math.lua b/lualibs-math.lua index 43f60b5..ec62919 100644 --- a/lualibs-math.lua +++ b/lualibs-math.lua @@ -8,6 +8,10 @@ if not modules then modules = { } end modules ['l-math'] = { local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan +if not math.ceiling then + math.ceiling = math.ceil +end + if not math.round then function math.round(x) return floor(x + 0.5) end end diff --git a/lualibs-os.lua b/lualibs-os.lua index f44b316..0a86ea6 100644 --- a/lualibs-os.lua +++ b/lualibs-os.lua @@ -349,6 +349,8 @@ else end +os.newline = name == "windows" and "\013\010" or "\010" -- crlf or lf + function resolvers.bits(t,k) local bits = find(os.platform,"64",1,true) and 64 or 32 os.bits = bits diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua index 034726f..5497e54 100644 --- a/lualibs-trac-inf.lua +++ b/lualibs-trac-inf.lua @@ -207,17 +207,3 @@ function statistics.tracefunction(base,tag,...) statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end) end end - --- where, not really the best spot for this: - -commands = commands or { } - -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end - -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) -end diff --git a/lualibs-unicode.lua b/lualibs-unicode.lua index 02dd1a0..70b6032 100644 --- a/lualibs-unicode.lua +++ b/lualibs-unicode.lua @@ -418,9 +418,11 @@ if not utf.sub then end end - local pattern_zero = Cmt(p_utf8char,slide_zero)^0 - local pattern_one = Cmt(p_utf8char,slide_one )^0 - local pattern_two = Cmt(p_utf8char,slide_two )^0 + local pattern_zero = Cmt(p_utf8char,slide_zero)^0 + local pattern_one = Cmt(p_utf8char,slide_one )^0 + local pattern_two = Cmt(p_utf8char,slide_two )^0 + + local pattern_first = C(patterns.utf8character) function utf.sub(str,start,stop) if not start then @@ -463,7 +465,9 @@ if not utf.sub then end end end - if start > stop then + if start == 1 and stop == 1 then + return lpegmatch(pattern_first,str) or "" + elseif start > stop then return "" elseif start > 1 then b, e, n, first, last = 0, 0, 0, start - 1, stop diff --git a/lualibs-util-dim.lua b/lualibs-util-dim.lua index bfffb10..2bdb870 100644 --- a/lualibs-util-dim.lua +++ b/lualibs-util-dim.lua @@ -92,9 +92,9 @@ format (string) is implemented using this table.

--ldx]]-- local f_none = formatters["%s%s"] -local f_true = formatters["%0.5f%s"] +local f_true = formatters["%0.5F%s"] -local function numbertodimen(n,unit,fmt) +local function numbertodimen(n,unit,fmt) -- will be redefined later ! if type(n) == 'string' then return n else diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua index 302b984..ea34c2c 100644 --- a/lualibs-util-prs.lua +++ b/lualibs-util-prs.lua @@ -333,7 +333,7 @@ end -- for mtx-context etc: aaaa bbbb cccc=dddd eeee=ffff -local str = C((1-whitespace-equal)^1) +local str = Cs(lpegpatterns.unquoted) + C((1-whitespace-equal)^1) local setting = Cf( Carg(1) * (whitespace^0 * Cg(str * whitespace^0 * (equal * whitespace^0 * str + Cc(""))))^1,rawset) local splitter = setting^1 @@ -341,6 +341,12 @@ function utilities.parsers.options_to_hash(str,target) return str and lpegmatch(splitter,str,1,target or { }) or { } end +local splitter = lpeg.tsplitat(" ") + +function utilities.parsers.options_to_array(str) + return str and lpegmatch(splitter,str) or { } +end + -- for chem (currently one level) local value = P(lbrace * C((nobrace + nestedbraces)^0) * rbrace) @@ -524,7 +530,7 @@ function parsers.rfc4180splitter(specification) local field = escaped + non_escaped + Cc("") local record = Ct(field * (separator * field)^1) local headerline = record * Cp() - local wholeblob = Ct((newline^-1 * record)^0) + local wholeblob = Ct((newline^(specification.strict and -1 or 1) * record)^0) return function(data,getheader) if getheader then local header, position = lpegmatch(headerline,data) diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua index a677a82..de4a87e 100644 --- a/lualibs-util-str.lua +++ b/lualibs-util-str.lua @@ -44,7 +44,12 @@ end if not number then number = { } end -- temp hack for luatex-fonts -local stripper = patterns.stripzeros +local stripper = patterns.stripzeros +local newline = patterns.newline +local endofstring = patterns.endofstring +local whitespace = patterns.whitespace +local spacer = patterns.spacer +local spaceortab = patterns.spaceortab local function points(n) n = tonumber(n) @@ -62,12 +67,12 @@ number.basepoints = basepoints -- str = " \n \ntest \n test\ntest " -- print("["..string.gsub(string.collapsecrlf(str),"\n","+").."]") -local rubish = patterns.spaceortab^0 * patterns.newline -local anyrubish = patterns.spaceortab + patterns.newline +local rubish = spaceortab^0 * newline +local anyrubish = spaceortab + newline local anything = patterns.anything -local stripped = (patterns.spaceortab^1 / "") * patterns.newline +local stripped = (spaceortab^1 / "") * newline local leading = rubish^0 / "" -local trailing = (anyrubish^1 * patterns.endofstring) / "" +local trailing = (anyrubish^1 * endofstring) / "" local redundant = rubish^3 / "\n" local pattern = Cs(leading * (trailing + redundant + stripped + anything)^0) @@ -129,7 +134,7 @@ local pattern = return "" end end - + patterns.newline * Cp() / function(position) + + newline * Cp() / function(position) extra, start = 0, position end + patterns.anything @@ -162,11 +167,6 @@ end -- return str -- end -local newline = patterns.newline -local endofstring = patterns.endofstring -local whitespace = patterns.whitespace -local spacer = patterns.spacer - local space = spacer^0 local nospace = space/"" local endofline = nospace * newline @@ -1117,3 +1117,9 @@ local pattern = function string.optionalquoted(str) return lpegmatch(pattern,str) or str end + +local pattern = Cs((newline / os.newline + 1)^0) + +function string.replacenewlines(str) + return lpegmatch(pattern,str) +end diff --git a/lualibs-util-tpl.lua b/lualibs-util-tpl.lua index bd0e261..468dd42 100644 --- a/lualibs-util-tpl.lua +++ b/lualibs-util-tpl.lua @@ -128,6 +128,11 @@ local function replacekeyquoted(s,t,how,recurse) -- ".. \" " end end +local function replaceoptional(l,m,r,t,how,recurse) + local v = t[l] + return v and v ~= "" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end + local single = P("%") -- test %test% test : resolves test local double = P("%%") -- test 10%% test : %% becomes % local lquoted = P("%[") -- test '%[test]%' test : resolves to test with escaped "'s @@ -143,12 +148,19 @@ local norquoted = rquoted / '' local nolquotedq = lquotedq / '' local norquotedq = rquotedq / '' -local key = nosingle * ((C((1-nosingle )^1) * Carg(1) * Carg(2) * Carg(3)) / replacekey ) * nosingle -local quoted = nolquotedq * ((C((1-norquotedq)^1) * Carg(1) * Carg(2) * Carg(3)) / replacekeyquoted ) * norquotedq -local unquoted = nolquoted * ((C((1-norquoted )^1) * Carg(1) * Carg(2) * Carg(3)) / replacekeyunquoted) * norquoted +local noloptional = P("%?") / '' +local noroptional = P("?%") / '' +local nomoptional = P(":") / '' + + +local args = Carg(1) * Carg(2) * Carg(3) +local key = nosingle * ((C((1-nosingle )^1) * args) / replacekey ) * nosingle +local quoted = nolquotedq * ((C((1-norquotedq )^1) * args) / replacekeyquoted ) * norquotedq +local unquoted = nolquoted * ((C((1-norquoted )^1) * args) / replacekeyunquoted) * norquoted +local optional = noloptional * ((C((1-nomoptional)^1) * nomoptional * C((1-noroptional)^1) * args) / replaceoptional) * noroptional local any = P(1) - replacer = Cs((unquoted + quoted + escape + key + any)^0) + replacer = Cs((unquoted + quoted + escape + optional + key + any)^0) local function replace(str,mapping,how,recurse) if mapping and str then @@ -164,6 +176,7 @@ end -- print(replace("test '%[x]%' test",{ x = [[a '%y%'  a]], y = "oeps" },'sql',true)) -- print(replace([[test %[x]% test]],{ x = [[a "x"  a]]})) -- print(replace([[test %(x)% test]],{ x = [[a "x"  a]]})) +-- print(replace([[convert %?x: -x "%x%" ?% %?y: -y "%y%" ?%]],{ x = "yes" })) templates.replace = replace -- cgit v1.2.3