diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | NEWS | 3 | ||||
| -rw-r--r-- | lualibs-lpeg.lua | 26 | ||||
| -rw-r--r-- | lualibs-os.lua | 33 | ||||
| -rw-r--r-- | lualibs-package.lua | 2 | ||||
| -rw-r--r-- | lualibs-table.lua | 34 | ||||
| -rw-r--r-- | lualibs-trac-inf.lua | 5 | ||||
| -rw-r--r-- | lualibs-unicode.lua | 33 | ||||
| -rw-r--r-- | lualibs-url.lua | 10 | ||||
| -rw-r--r-- | lualibs-util-env.lua | 4 | ||||
| -rw-r--r-- | lualibs-util-prs.lua | 8 | ||||
| -rw-r--r-- | lualibs-util-str.lua | 141 |
12 files changed, 225 insertions, 76 deletions
@@ -1,2 +1,4 @@ testing/* *-merged.lua +*.aux +*.sh @@ -1,4 +1,7 @@ History of the lualibs package +2013/31/03 v2.1a/ + * sync with Context beta as of 2013-12-30 + 2013/11/03 v2.1/ * sync with Context beta as of 2013-11-01 * add l-gzip diff --git a/lualibs-lpeg.lua b/lualibs-lpeg.lua index 399b3ad..6d3acd7 100644 --- a/lualibs-lpeg.lua +++ b/lualibs-lpeg.lua @@ -6,6 +6,10 @@ if not modules then modules = { } end modules ['l-lpeg'] = { license = "see context related readme files" } +-- lpeg 12 vs lpeg 10: slower compilation, similar parsing speed (i need to check +-- if i can use new features like capture / 2 and .B (at first sight the xml +-- parser is some 5% slower) + -- a new lpeg fails on a #(1-P(":")) test and really needs a + P(-1) -- move utf -> l-unicode @@ -15,14 +19,15 @@ lpeg = require("lpeg") -- The latest lpeg doesn't have print any more, and even the new ones are not -- available by default (only when debug mode is enabled), which is a pitty as --- as it helps bailign down bottlenecks. Performance seems comparable, although +-- as it helps nailign down bottlenecks. Performance seems comparable: some 10% +-- slower pattern compilation, same parsing speed, although, -- -- local p = lpeg.C(lpeg.P(1)^0 * lpeg.P(-1)) --- local a = string.rep("123",10) +-- local a = string.rep("123",100) -- lpeg.match(p,a) -- --- is nearly 20% slower and also still suboptimal (i.e. a match that runs from --- begin to end, one of the cases where string matchers win). +-- seems slower and is also still suboptimal (i.e. a match that runs from begin +-- to end, one of the cases where string matchers win). if not lpeg.print then function lpeg.print(...) print(lpeg.pcode(...)) end end @@ -74,7 +79,9 @@ local lpegtype, lpegmatch, lpegprint = lpeg.type, lpeg.match, lpeg.print -- let's start with an inspector: -setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +if setinspector then + setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +end -- Beware, we predefine a bunch of patterns here and one reason for doing so -- is that we get consistent behaviour in some of the visualizers. @@ -469,7 +476,7 @@ end -- local pattern1 = P(1-P(pattern))^0 * P(pattern) : test for not nil -- local pattern2 = (P(pattern) * Cc(true) + P(1))^0 : test for true (could be faster, but not much) -function lpeg.finder(lst,makefunction) -- beware: slower than find with 'patternless finds' +function lpeg.finder(lst,makefunction,isutf) -- beware: slower than find with 'patternless finds' local pattern if type(lst) == "table" then pattern = P(false) @@ -485,7 +492,12 @@ function lpeg.finder(lst,makefunction) -- beware: slower than find with 'pattern else pattern = P(lst) end - pattern = (1-pattern)^0 * pattern + if isutf then +-- pattern = ((utf8char or 1)-pattern)^0 * pattern + pattern = ((utf8char or 1)-pattern)^0 * pattern + else + pattern = (1-pattern)^0 * pattern + end if makefunction then return function(str) return lpegmatch(pattern,str) diff --git a/lualibs-os.lua b/lualibs-os.lua index 7f3fd7c..bfafa4f 100644 --- a/lualibs-os.lua +++ b/lualibs-os.lua @@ -197,9 +197,8 @@ end -- no need for function anymore as we have more clever code and helpers now -- this metatable trickery might as well disappear -os.resolvers = os.resolvers or { } -- will become private - -local resolvers = os.resolvers +local resolvers = os.resolvers or { } +os.resolvers = resolvers setmetatable(os, { __index = function(t,k) local r = resolvers[k] @@ -222,6 +221,8 @@ local function guess() return os.resultof("echo $HOSTTYPE") or "" end +-- os.bits = 32 | 64 + if platform ~= "" then os.platform = platform @@ -230,10 +231,14 @@ elseif os.type == "windows" then -- we could set the variable directly, no function needed here - function os.resolvers.platform(t,k) + -- PROCESSOR_ARCHITECTURE : binary platform + -- PROCESSOR_ARCHITEW6432 : OS platform + + function resolvers.platform(t,k) local platform, architecture = "", os.getenv("PROCESSOR_ARCHITECTURE") or "" if find(architecture,"AMD64") then - platform = "mswin-64" + -- platform = "mswin-64" + platform = "win64" else platform = "mswin" end @@ -244,7 +249,7 @@ elseif os.type == "windows" then elseif name == "linux" then - function os.resolvers.platform(t,k) + function resolvers.platform(t,k) -- we sometimes have HOSTTYPE set so let's check that first local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" if find(architecture,"x86_64") then @@ -271,7 +276,7 @@ elseif name == "macosx" then therefore not permitted to run the 64 bit kernel. ]]-- - function os.resolvers.platform(t,k) + function resolvers.platform(t,k) -- local platform, architecture = "", os.getenv("HOSTTYPE") or "" -- if architecture == "" then -- architecture = os.resultof("echo $HOSTTYPE") or "" @@ -294,7 +299,7 @@ elseif name == "macosx" then elseif name == "sunos" then - function os.resolvers.platform(t,k) + function resolvers.platform(t,k) local platform, architecture = "", os.resultof("uname -m") or "" if find(architecture,"sparc") then platform = "solaris-sparc" @@ -308,7 +313,7 @@ elseif name == "sunos" then elseif name == "freebsd" then - function os.resolvers.platform(t,k) + function resolvers.platform(t,k) local platform, architecture = "", os.resultof("uname -m") or "" if find(architecture,"amd64") then platform = "freebsd-amd64" @@ -322,7 +327,7 @@ elseif name == "freebsd" then elseif name == "kfreebsd" then - function os.resolvers.platform(t,k) + function resolvers.platform(t,k) -- we sometimes have HOSTTYPE set so let's check that first local platform, architecture = "", os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" if find(architecture,"x86_64") then @@ -341,7 +346,7 @@ else -- os.setenv("MTX_PLATFORM",platform) -- os.platform = platform - function os.resolvers.platform(t,k) + function resolvers.platform(t,k) local platform = "linux" os.setenv("MTX_PLATFORM",platform) os.platform = platform @@ -350,6 +355,12 @@ else end +function resolvers.bits(t,k) + local bits = find(os.platform,"64") and 64 or 32 + os.bits = bits + return bits +end + -- beware, we set the randomseed -- from wikipedia: Version 4 UUIDs use a scheme relying only on random numbers. This algorithm sets the diff --git a/lualibs-package.lua b/lualibs-package.lua index 0dbff7c..075fcde 100644 --- a/lualibs-package.lua +++ b/lualibs-package.lua @@ -274,7 +274,7 @@ methods["qualified path"]=function(name) end methods["lua extra list"] = function(name) - return loadedbypath(addsuffix(lualibfile(name),"lua" ),name,getextraluapaths(),false,"lua") + return loadedbypath(addsuffix(lualibfile(name),"lua"),name,getextraluapaths(),false,"lua") end methods["lib extra list"] = function(name) diff --git a/lualibs-table.lua b/lualibs-table.lua index 11cb66b..c318c57 100644 --- a/lualibs-table.lua +++ b/lualibs-table.lua @@ -129,10 +129,13 @@ local function sortedhash(t,cmp) s = sortedkeys(t) -- the robust one end local n = 0 + local m = #s local function kv(s) - n = n + 1 - local k = s[n] - return k, t[k] + if n < m then + n = n + 1 + local k = s[n] + return k, t[k] + end end return kv, s else @@ -1003,7 +1006,9 @@ function table.print(t,...) end end -setinspector(function(v) if type(v) == "table" then serialize(print,v,"table") return true end end) +if setinspector then + setinspector(function(v) if type(v) == "table" then serialize(print,v,"table") return true end end) +end -- -- -- obsolete but we keep them for a while and might comment them later -- -- -- @@ -1054,3 +1059,24 @@ function table.sorted(t,...) sort(t,...) return t -- still sorts in-place end + +-- + +function table.values(t,s) -- optional sort flag + if t then + local values, keys, v = { }, { }, 0 + for key, value in next, t do + if not keys[value] then + v = v + 1 + values[v] = value + keys[k] = key + end + end + if s then + sort(values) + end + return values + else + return { } + end +end diff --git a/lualibs-trac-inf.lua b/lualibs-trac-inf.lua index 79cbdba..802f2e6 100644 --- a/lualibs-trac-inf.lua +++ b/lualibs-trac-inf.lua @@ -122,6 +122,9 @@ function statistics.show() if statistics.enable then -- this code will move local register = statistics.register + register("used platform", function() + return format("%s, type: %s, binary subtree: %s",os.platform or "unknown",os.type or "unknown", environment.texos or "unknown") + end) register("luatex banner", function() return lower(status.banner) end) @@ -179,7 +182,7 @@ function statistics.timed(action) starttiming("run") action() stoptiming("run") - report("total runtime: %s",elapsedtime("run")) + report("total runtime: %s seconds",elapsedtime("run")) end -- goodie diff --git a/lualibs-unicode.lua b/lualibs-unicode.lua index 7ada394..6601a4c 100644 --- a/lualibs-unicode.lua +++ b/lualibs-unicode.lua @@ -9,7 +9,6 @@ if not modules then modules = { } end modules ['l-unicode'] = { -- this module will be reorganized -- todo: utf.sub replacement (used in syst-aux) - -- we put these in the utf namespace: utf = utf or (unicode and unicode.utf8) or { } @@ -935,19 +934,27 @@ end local _, l_remap = utf.remapper(little) local _, b_remap = utf.remapper(big) -function utf.utf8_to_utf16_be(str) - return char(254,255) .. lpegmatch(b_remap,str) +function utf.utf8_to_utf16_be(str,nobom) + if nobom then + return lpegmatch(b_remap,str) + else + return char(254,255) .. lpegmatch(b_remap,str) + end end -function utf.utf8_to_utf16_le(str) - return char(255,254) .. lpegmatch(l_remap,str) +function utf.utf8_to_utf16_le(str,nobom) + if nobom then + return lpegmatch(l_remap,str) + else + return char(255,254) .. lpegmatch(l_remap,str) + end end -function utf.utf8_to_utf16(str,littleendian) +function utf.utf8_to_utf16(str,littleendian,nobom) if littleendian then - return utf.utf8_to_utf16_le(str) + return utf.utf8_to_utf16_le(str,nobom) else - return utf.utf8_to_utf16_be(str) + return utf.utf8_to_utf16_be(str,nobom) end end @@ -1122,3 +1129,13 @@ if not utf.values then string.utfvalues = utf.values end + +function utf.chrlen(u) -- u is number + return + (u < 0x80 and 1) or + (u < 0xE0 and 2) or + (u < 0xF0 and 3) or + (u < 0xF8 and 4) or + (u < 0xFC and 5) or + (u < 0xFE and 6) or 0 +end diff --git a/lualibs-url.lua b/lualibs-url.lua index 7b7910f..7bb7312 100644 --- a/lualibs-url.lua +++ b/lualibs-url.lua @@ -26,6 +26,8 @@ local lpegmatch, lpegpatterns, replacer = lpeg.match, lpeg.patterns, lpeg.replac -- | ___________|____________ | -- / \ / \ | -- urn:example:animal:ferret:nose interpretable as extension +-- +-- also nice: http://url.spec.whatwg.org/ (maybe some day ...) url = url or { } local url = url @@ -43,7 +45,7 @@ local hexdigit = R("09","AF","af") local plus = P("+") local nothing = Cc("") local escapedchar = (percent * C(hexdigit * hexdigit)) / tochar -local escaped = (plus / " ") + escapedchar +local escaped = (plus / " ") + escapedchar -- so no loc://foo++.tex local noslash = P("/") / "" @@ -189,7 +191,11 @@ local function hashed(str) -- not yet ok (/test?test) return s end --- inspect(hashed("template://test")) +-- inspect(hashed("template:///test")) +-- inspect(hashed("template:///test++.whatever")) +-- inspect(hashed("template:///test%2B%2B.whatever")) +-- inspect(hashed("template:///test%x.whatever")) +-- inspect(hashed("tem%2Bplate:///test%x.whatever")) -- Here we assume: -- diff --git a/lualibs-util-env.lua b/lualibs-util-env.lua index 1184c1d..0a708ea 100644 --- a/lualibs-util-env.lua +++ b/lualibs-util-env.lua @@ -42,7 +42,7 @@ local basicengines = allocate { -- ["texluajit.exe"] = "luajittex", } -local luaengines=allocate { +local luaengines = allocate { ["lua"] = true, ["luajit"] = true, } @@ -66,6 +66,8 @@ elseif luaengines[file.removesuffix(arg[-1])] then -- arg[k-1] = arg[k] -- end -- remove(arg) -- last +-- +-- environment.used_as_library = true elseif validengines[file.removesuffix(arg[0])] then if arg[1] == "--luaonly" then arg[-1] = arg[0] diff --git a/lualibs-util-prs.lua b/lualibs-util-prs.lua index 29a57e0..e5b35a7 100644 --- a/lualibs-util-prs.lua +++ b/lualibs-util-prs.lua @@ -184,8 +184,10 @@ function parsers.settings_to_array(str,strict) else return { str } end - else + elseif find(str,",") then return lpegmatch(pattern,str) + else + return { str } end end @@ -483,8 +485,8 @@ function parsers.rfc4180splitter(specification) * Cs((dquotechar + (1 - quotechar))^0) * quotechar local non_escaped = C((1 - quotechar - newline - separator)^1) - local field = escaped + non_escaped - local record = Ct((field * separator^-1)^1) + local field = escaped + non_escaped + Cc("") + local record = Ct(field * (separator * field)^1) local headerline = record * Cp() local wholeblob = Ct((newline^-1 * record)^0) return function(data,getheader) diff --git a/lualibs-util-str.lua b/lualibs-util-str.lua index 24a3f6e..f04f0e5 100644 --- a/lualibs-util-str.lua +++ b/lualibs-util-str.lua @@ -20,8 +20,16 @@ local utfchar, utfbyte = utf.char, utf.byte ----- loadstripped = utilities.lua.loadstripped ----- setmetatableindex = table.setmetatableindex -local loadstripped = _LUAVERSION < 5.2 and load or function(str) - return load(dump(load(str),true)) -- it only makes sense in luajit and luatex where we have a stipped load +-- local loadstripped = _LUAVERSION < 5.2 and load or function(str) +-- return load(dump(load(str),true)) -- it only makes sense in luajit and luatex where we have a stipped load +-- end + +local loadstripped = function(str,shortcuts) + if shortcuts then + return load(dump(load(str),true),nil,nil,shortcuts) + else + return load(dump(load(str),true)) + end end -- todo: make a special namespace for the formatter @@ -291,33 +299,69 @@ function number.sparseexponent(f,n) return tostring(n) end -local preamble = [[ -local type = type -local tostring = tostring -local tonumber = tonumber -local format = string.format -local concat = table.concat -local signed = number.signed -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -local lpegmatch = lpeg.match -local nspaces = string.nspaces -local tracedchar = string.tracedchar -local autosingle = string.autosingle -local autodouble = string.autodouble -local sequenced = table.sequenced -local formattednumber = number.formatted -local sparseexponent = number.sparseexponent -]] - local template = [[ %s %s return function(%s) return %s end ]] +-- local environment = { +-- lpeg = lpeg, +-- type = type, +-- string = string, +-- number = number, +-- table = table, +-- utf = utf, +-- } +-- +-- local preamble = [[ +-- local type = type +-- local tostring = tostring +-- local tonumber = tonumber +-- local format = string.format +-- local concat = table.concat +-- local signed = number.signed +-- local points = number.points +-- local basepoints = number.basepoints +-- local utfchar = utf.char +-- local utfbyte = utf.byte +-- local lpegmatch = lpeg.match +-- local nspaces = string.nspaces +-- local tracedchar = string.tracedchar +-- local autosingle = string.autosingle +-- local autodouble = string.autodouble +-- local sequenced = table.sequenced +-- local formattednumber = number.formatted +-- local sparseexponent = number.sparseexponent +-- ]] + +local environment = { + global = global or _G, + lpeg = lpeg, + type = type, + tostring = tostring, + tonumber = tonumber, + format = string.format, + concat = table.concat, + signed = number.signed, + points = number.points, + basepoints = number.basepoints, + utfchar = utf.char, + utfbyte = utf.byte, + lpegmatch = lpeg.match, + nspaces = string.nspaces, + tracedchar = string.tracedchar, + autosingle = string.autosingle, + autodouble = string.autodouble, + sequenced = table.sequenced, + formattednumber = number.formatted, + sparseexponent = number.sparseexponent, +} + +local preamble = "" + +-- -- -- + local arguments = { "a1" } -- faster than previously used (select(n,...)) setmetatable(arguments, { __index = @@ -647,7 +691,7 @@ local format_extension = function(extensions,f,name) end end --- aA b cC d eE f gG hH iI jJ lL mM N o p qQ r sS tT uU wW xX +-- aA b cC d eE f gG hH iI jJ lL mM N o p qQ r sS tT uU wW xX z local builder = Cs { "start", start = ( @@ -740,28 +784,37 @@ local builder = Cs { "start", -- we can be clever and only alias what is needed +-- local direct = Cs ( +-- P("%")/"" +-- * Cc([[local format = string.format return function(str) return format("%]]) +-- * (S("+- .") + R("09"))^0 +-- * S("sqidfgGeExXo") +-- * Cc([[",str) end]]) +-- * P(-1) +-- ) + local direct = Cs ( - P("%")/"" - * Cc([[local format = string.format return function(str) return format("%]]) - * (S("+- .") + R("09"))^0 - * S("sqidfgGeExXo") - * Cc([[",str) end]]) - * P(-1) - ) + P("%") + * (S("+- .") + R("09"))^0 + * S("sqidfgGeExXo") + * P(-1) / [[local format = string.format return function(str) return format("%0",str) end]] +) local function make(t,str) local f local p local p = lpegmatch(direct,str) if p then + -- f = loadstripped(p)() + -- print("builder 1 >",p) f = loadstripped(p)() else n = 0 p = lpegmatch(builder,str,1,"..",t._extensions_) -- after this we know n if n > 0 then p = format(template,preamble,t._preamble_,arguments[n],p) --- print("builder>",p) - f = loadstripped(p)() + -- print("builder 2 >",p) + f = loadstripped(p,t._environment_)() -- t._environment is not populated (was experiment) else f = function() return str end end @@ -817,7 +870,11 @@ strings.formatters = { } -- clear that table when a threshold is reached function strings.formatters.new() - local t = { _extensions_ = { }, _preamble_ = "", _type_ = "formatter" } + local e = { } -- better make a copy as we can overload + for k, v in next, environment do + e[k] = v + end + local t = { _extensions_ = { }, _preamble_ = "", _environment_ = e, _type_ = "formatter" } setmetatable(t, { __index = make, __call = use }) return t end @@ -838,8 +895,12 @@ string.formatter = function(str,...) return formatters[str](...) end -- someti local function add(t,name,template,preamble) if type(t) == "table" and t._type_ == "formatter" then t._extensions_[name] = template or "%s" - if preamble then + if type(preamble) == "string" then t._preamble_ = preamble .. "\n" .. t._preamble_ -- so no overload ! + elseif type(preamble) == "table" then + for k, v in next, preamble do + t._environment_[k] = v + end end end end @@ -856,9 +917,13 @@ patterns.luaquoted = Cs(Cc('"') * ((1-S('"\n'))^1 + P('"')/'\\"' + P('\n')/'\\n" -- escaping by lpeg is faster for strings without quotes, slower on a string with quotes, but -- faster again when other q-escapables are found (the ones we don't need to escape) -add(formatters,"xml", [[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]]) -add(formatters,"tex", [[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]]) -add(formatters,"lua", [[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]]) +-- add(formatters,"xml", [[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]]) +-- add(formatters,"tex", [[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]]) +-- add(formatters,"lua", [[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]]) + +add(formatters,"xml", [[lpegmatch(xmlescape,%s)]],{ xmlescape = lpeg.patterns.xmlescape }) +add(formatters,"tex", [[lpegmatch(texescape,%s)]],{ texescape = lpeg.patterns.texescape }) +add(formatters,"lua", [[lpegmatch(luaescape,%s)]],{ luaescape = lpeg.patterns.luaescape }) -- -- yes or no: -- |