diff options
Diffstat (limited to 'tex')
23 files changed, 1840 insertions, 230 deletions
diff --git a/tex/context/base/cont-new.mkii b/tex/context/base/cont-new.mkii index 2e90a2dc9..d7edc921c 100644 --- a/tex/context/base/cont-new.mkii +++ b/tex/context/base/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2012.09.16 23:18} +\newcontextversion{2012.09.21 20:58} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv index cc6342b05..2f90130c7 100644 --- a/tex/context/base/cont-new.mkiv +++ b/tex/context/base/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2012.09.16 23:18} +\newcontextversion{2012.09.21 20:58} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf Binary files differindex 7db72b855..e232cfeb6 100644 --- a/tex/context/base/context-version.pdf +++ b/tex/context/base/context-version.pdf diff --git a/tex/context/base/context-version.png b/tex/context/base/context-version.png Binary files differindex 98253d101..048d04143 100644 --- a/tex/context/base/context-version.png +++ b/tex/context/base/context-version.png diff --git a/tex/context/base/context.mkii b/tex/context/base/context.mkii index 2582f1f9f..0aab24aad 100644 --- a/tex/context/base/context.mkii +++ b/tex/context/base/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2012.09.16 23:18} +\edef\contextversion{2012.09.21 20:58} %D For those who want to use this: diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv index 52c4d897e..d19d90a93 100644 --- a/tex/context/base/context.mkiv +++ b/tex/context/base/context.mkiv @@ -25,7 +25,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2012.09.16 23:18} +\edef\contextversion{2012.09.21 20:58} %D For those who want to use this: diff --git a/tex/context/base/core-dat.lua b/tex/context/base/core-dat.lua index 071a3fe0b..879ff6130 100644 --- a/tex/context/base/core-dat.lua +++ b/tex/context/base/core-dat.lua @@ -11,7 +11,7 @@ if not modules then modules = { } end modules ['core-dat'] = { replaces the twopass data mechanism.</p> --ldx]]-- -local tonumber = tonumber +local tonumber, type = tonumber, type local context, commands = context, commands @@ -69,7 +69,12 @@ local function setdata(settings) local tag = settings.tag local data = settings.data local list = tobesaved[name] - data = settings_to_hash(data) or { } + if settings.convert and type(data) == "string" then + data = settings_to_hash(data) + end + if type(data) ~= "table" then + data = { data = settings.data } + end if not tag then tag = #list + 1 else @@ -109,25 +114,30 @@ end function datasets.getdata(name,tag,key,default) local t = collected[name] - if t then + if t == nil then + if trace_datasets then + report_dataset("unknown: name %s",name) + end + elseif type(t) ~= "table" then + return t + else t = t[tag] or t[tonumber(tag)] - if t then - if key then - return t[key] or default - else - return t + if not t then + if trace_datasets then + report_dataset("unknown: name %s, tag %s",name,tag) end - elseif trace_datasets then - report_dataset("unknown: name %s, tag %s",name,tag) + elseif key then + return t[key] or default + else + return t end - elseif trace_datasets then - report_dataset("unknown: name %s",name) end return default end function commands.setdataset(settings) - local name, tag, data = setdata(settings) + settings.convert = true + local name, tag = setdata(settings) if settings.delay ~= v_yes then -- elseif type(tag) == "number" then @@ -139,11 +149,25 @@ end function commands.datasetvariable(name,tag,key) local t = collected[name] - t = t and (t[tag] or t[tonumber(tag)]) - if t then - local s = t[key] - if s then - context(s) + if t == nil then + if trace_datasets then + report_dataset("unknown: name %s (not passed to tex)",name) + end + elseif type(t) ~= "table" then + context(tostring(t)) + else + t = t and (t[tag] or t[tonumber(tag)]) + if not t then + if trace_datasets then + report_dataset("unknown: name %s with tag %s (not passed to tex)",name,tag) + end + elseif type(t) ~= "table" then + local s = t[key] + if type(s) ~= "table" then + context(tostring(s)) + elseif trace_datasets then + report_dataset("table: name %s, tag %s (not passed to tex)",name,tag) + end end end end diff --git a/tex/context/base/data-lua.lua b/tex/context/base/data-lua.lua index ab762f3d4..fec5856ea 100644 --- a/tex/context/base/data-lua.lua +++ b/tex/context/base/data-lua.lua @@ -6,9 +6,15 @@ if not modules then modules = { } end modules ['data-lua'] = { license = "see context related readme files" } --- some loading stuff ... we might move this one to slot 2 depending --- on the developments (the loaders must not trigger kpse); we could --- of course use a more extensive lib path spec +-- We overload the regular loader. We do so because we operate mostly in +-- tds and use our own loader code. Alternatively we could use a more +-- extensive definition of package.path and package.cpath but even then +-- we're not done. Also, we now have better tracing. +-- +-- -- local mylib = require("libtest") +-- -- local mysql = require("luasql.mysql") + +local concat = table.concat local trace_libraries = false @@ -18,160 +24,178 @@ trackers.register("resolvers.locating", function(v) trace_libraries = v end) local report_libraries = logs.reporter("resolvers","libraries") local gsub, insert = string.gsub, table.insert +local P, Cs, lpegmatch = lpeg.P, lpeg.Cs, lpeg.match local unpack = unpack or table.unpack +local is_readable = file.is_readable local resolvers, package = resolvers, package --- local libformats = { 'luatexlibs', 'tex', 'texmfscripts', 'othertextfiles' } -local libformats = { 'lua', 'tex' } -local clibformats = { 'lib' } +local libsuffixes = { 'tex', 'lua' } +local clibsuffixes = { 'lib' } +local libformats = { 'TEXINPUTS', 'LUAINPUTS' } +local clibformats = { 'CLUAINPUTS' } + +local libpaths = nil +local clibpaths = nil +local libhash = { } +local clibhash = { } +local libextras = { } +local clibextras = { } -local _path_, libpaths, _cpath_, clibpaths +local pattern = Cs(P("!")^0 / "" * (P("/") * P(-1) / "/" + P("/")^1 / "/" + 1)^0) + +local function cleanpath(path) --hm, don't we have a helper for this? + return resolvers.resolve(lpegmatch(pattern,path)) +end -function package.libpaths() - if not _path_ or package.path ~= _path_ then - _path_ = package.path - libpaths = file.splitpath(_path_,";") +local function getlibpaths() + if not libpaths then + libpaths = { } + for i=1,#libformats do + local paths = resolvers.expandedpathlistfromvariable(libformats[i]) + for i=1,#paths do + local path = cleanpath(paths[i]) + if not libhash[path] then + libpaths[#libpaths+1] = path + libhash[path] = true + end + end + end end return libpaths end -function package.clibpaths() - if not _cpath_ or package.cpath ~= _cpath_ then - _cpath_ = package.cpath - clibpaths = file.splitpath(_cpath_,";") +local function getclibpaths() + if not clibpaths then + clibpaths = { } + for i=1,#clibformats do + local paths = resolvers.expandedpathlistfromvariable(clibformats[i]) + for i=1,#paths do + local path = cleanpath(paths[i]) + if not clibhash[path] then + clibpaths[#clibpaths+1] = path + clibhash[path] = true + end + end + end end return clibpaths end -local function thepath(...) - local t = { ... } t[#t+1] = "?.lua" - local path = file.join(unpack(t)) - if trace_libraries then - report_libraries("! appending '%s' to 'package.path'",path) +package.libpaths = getlibpaths +package.clibpaths = getclibpaths + +function package.extralibpath(...) + local paths = { ... } + for i=1,#paths do + local path = cleanpath(paths[i]) + if not libhash[path] then + if trace_libraries then + report_libraries("! extra lua path '%s'",path) + end + libextras[#libextras+1] = path + libpaths[#libpaths +1] = path + end end - return path end -local p_libpaths, a_libpaths = { }, { } - -function package.appendtolibpath(...) - insert(a_libpath,thepath(...)) +function package.extraclibpath(...) + local paths = { ... } + for i=1,#paths do + local path = cleanpath(paths[i]) + if not clibhash[path] then + if trace_libraries then + report_libraries("! extra lib path '%s'",path) + end + clibextras[#clibextras+1] = path + clibpaths[#clibpaths +1] = path + end + end end -function package.prependtolibpath(...) - insert(p_libpaths,1,thepath(...)) +if not package.loaders[-2] then + -- use package-path and package-cpath + package.loaders[-2] = package.loaders[2] end --- beware, we need to return a loadfile result ! +local function loadedaslib(resolved,rawname) + return package.loadlib(resolved,"luaopen_" .. gsub(rawname,"%.","_")) +end -local function loaded(libpaths,name,simple) - for i=1,#libpaths do -- package.path, might become option - local libpath = libpaths[i] - local resolved = gsub(libpath,"%?",simple) - if trace_libraries then -- more detail - report_libraries("! checking for '%s' on 'package.path': '%s' => '%s'",simple,libpath,resolved) - end - if file.is_readable(resolved) then - if trace_libraries then - report_libraries("! lib '%s' located via 'package.path': '%s'",name,resolved) - end - return loadfile(resolved) - end +local function loadedbylua(name) + if trace_libraries then + report_libraries("! locating %q using normal loader",name) end + local resolved = package.loaders[-2](name) end -package.loaders[2] = function(name) -- was [#package.loaders+1] - if file.suffix(name) == "" then - name = file.addsuffix(name,"lua") -- maybe a list - if trace_libraries then -- mode detail - report_libraries("! locating '%s' with forced suffix",name) - end - else - if trace_libraries then -- mode detail - report_libraries("! locating '%s'",name) - end +local function loadedbyformat(name,rawname,suffixes,islib) + if trace_libraries then + report_libraries("! locating %q as %q using formats %q",rawname,name,concat(suffixes)) end - for i=1,#libformats do - local format = libformats[i] + for i=1,#suffixes do -- so we use findfile and not a lookup loop + local format = suffixes[i] local resolved = resolvers.findfile(name,format) or "" - if trace_libraries then -- mode detail - report_libraries("! checking for '%s' using 'libformat path': '%s'",name,format) + if trace_libraries then + report_libraries("! checking for %q' using format %q",name,format) end if resolved ~= "" then if trace_libraries then - report_libraries("! lib '%s' located via environment: '%s'",name,resolved) - end - return loadfile(resolved) - end - end - -- libpaths - local libpaths, clibpaths = package.libpaths(), package.clibpaths() - local simple = gsub(name,"%.lua$","") - local simple = gsub(simple,"%.","/") - local resolved = loaded(p_libpaths,name,simple) or loaded(libpaths,name,simple) or loaded(a_libpaths,name,simple) - if resolved then - return resolved - end - -- - local libname = file.addsuffix(simple,os.libsuffix) - for i=1,#clibformats do - -- better have a dedicated loop - local format = clibformats[i] - local paths = resolvers.expandedpathlistfromvariable(format) - for p=1,#paths do - local path = paths[p] - local resolved = file.join(path,libname) - if trace_libraries then -- mode detail - report_libraries("! checking for '%s' using 'clibformat path': '%s'",libname,path) + report_libraries("! lib %q located on %q",name,resolved) end - if file.is_readable(resolved) then - if trace_libraries then - report_libraries("! lib '%s' located via 'clibformat': '%s'",libname,resolved) - end - return package.loadlib(resolved,name) + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) end end end - for i=1,#clibpaths do -- package.path, might become option - local libpath = clibpaths[i] - local resolved = gsub(libpath,"?",simple) - if trace_libraries then -- more detail - report_libraries("! checking for '%s' on 'package.cpath': '%s'",simple,libpath) +end + +local function loadedbypath(name,rawname,paths,islib,what) + if trace_libraries then + report_libraries("! locating %q as %q on %q paths",rawname,name,what) + end + for p=1,#paths do + local path = paths[p] + local resolved = file.join(path,name) + if trace_libraries then -- mode detail + report_libraries("! checking for %q using %q path %q",name,what,path) end - if file.is_readable(resolved) then + if is_readable(resolved) then if trace_libraries then - report_libraries("! lib '%s' located via 'package.cpath': '%s'",name,resolved) + report_libraries("! lib %q located on %q",name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) end - return package.loadlib(resolved,name) - end - end - -- just in case the distribution is messed up - if trace_loading then -- more detail - report_libraries("! checking for '%s' using 'luatexlibs': '%s'",name) - end - local resolved = resolvers.findfile(file.basename(name),'luatexlibs') or "" - if resolved ~= "" then - if trace_libraries then - report_libraries("! lib '%s' located by basename via environment: '%s'",name,resolved) end - return loadfile(resolved) end +end + +local function notloaded(name) if trace_libraries then - report_libraries('? unable to locate lib: %s',name) + report_libraries("? unable to locate library %q",name) end --- return "unable to locate " .. name end -resolvers.loadlualib = require - --- -- -- -- - -package.obsolete = package.obsolete or { } - -package.append_libpath = appendtolibpath -- will become obsolete -package.prepend_libpath = prependtolibpath -- will become obsolete +package.loaders[2] = function(name) + local thename = gsub(name,"%.","/") + local luaname = file.addsuffix(thename,"lua") + local libname = file.addsuffix(thename,os.libsuffix) + return + loadedbyformat(luaname,name,libsuffixes, false) + or loadedbyformat(libname,name,clibsuffixes, true) + or loadedbypath (luaname,name,getlibpaths (),false,"lua") + or loadedbypath (luaname,name,getclibpaths(),false,"lua") + or loadedbypath (libname,name,getclibpaths(),true, "lib") + or loadedbylua (name) + or notloaded (name) +end -package.obsolete.append_libpath = appendtolibpath -- will become obsolete -package.obsolete.prepend_libpath = prependtolibpath -- will become obsolete +-- package.loaders[3] = nil +-- package.loaders[4] = nil +resolvers.loadlualib = require diff --git a/tex/context/base/data-pre.lua b/tex/context/base/data-pre.lua index 4e82b6186..214477d87 100644 --- a/tex/context/base/data-pre.lua +++ b/tex/context/base/data-pre.lua @@ -121,6 +121,8 @@ function resolvers.resetresolve(str) resolved, abstract = { }, { } end +-- todo: use an lpeg (see data-lua for !! / stripper) + local function resolve(str) -- use schemes, this one is then for the commandline only if type(str) == "table" then local t = { } diff --git a/tex/context/base/data-res.lua b/tex/context/base/data-res.lua index e51b980ff..9236cbe02 100644 --- a/tex/context/base/data-res.lua +++ b/tex/context/base/data-res.lua @@ -805,7 +805,7 @@ function resolvers.expandedpathlist(str) end end -function resolvers.expandedpathlistfromvariable(str) -- brrr +function resolvers.expandedpathlistfromvariable(str) -- brrr / could also have cleaner ^!! /$ // str = lpegmatch(dollarstripper,str) local tmp = resolvers.variableofformatorsuffix(str) return resolvers.expandedpathlist(tmp ~= "" and tmp or str) diff --git a/tex/context/base/l-os.lua b/tex/context/base/l-os.lua index 700affdac..799f44957 100644 --- a/tex/context/base/l-os.lua +++ b/tex/context/base/l-os.lua @@ -22,18 +22,28 @@ if not modules then modules = { } end modules ['l-os'] = { -- os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) -- os.platform : extended os.name with architecture +-- os.sleep() => socket.sleep() +-- math.randomseed(tonumber(string.sub(string.reverse(tostring(math.floor(socket.gettime()*10000))),1,6))) + -- maybe build io.flush in os.execute local os = os -local date = os.date +local date, time = os.date, os.time local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch local concat = table.concat local random, ceil, randomseed = math.random, math.ceil, math.randomseed -local rawget, rawset, type, getmetatable, setmetatable, tonumber = rawget, rawset, type, getmetatable, setmetatable, tonumber +local rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring = rawget, rawset, type, getmetatable, setmetatable, tonumber, tostring -- The following code permits traversing the environment table, at least -- in luatex. Internally all environment names are uppercase. +-- The randomseed in Lua is not that random, although this depends on the operating system as well +-- as the binary (Luatex is normally okay). But to be sure we set the seed anyway. + +math.initialseed = tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) + +randomseed(math.initialseed) + if not os.__getenv__ then os.__getenv__ = os.getenv @@ -375,10 +385,41 @@ end local timeformat = format("%%s%s",os.timezone(true)) local dateformat = "!%Y-%m-%d %H:%M:%S" -function os.fulltime(t) +function os.fulltime(t,default) + t = tonumber(t) or 0 + if t > 0 then + -- valid time + elseif default then + return default + else + t = nil + end return format(timeformat,date(dateformat,t)) end +local dateformat = "%Y-%m-%d %H:%M:%S" + +function os.localtime(t,default) + t = tonumber(t) or 0 + if t > 0 then + -- valid time + elseif default then + return default + else + t = nil + end + return date(dateformat,t) +end + +function os.converttime(t,default) + local t = tonumber(t) + if t and t > 0 then + return date(dateformat,t) + else + return default or "-" + end +end + local memory = { } local function which(filename) diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua index 413d31eb5..2b3319e45 100644 --- a/tex/context/base/l-table.lua +++ b/tex/context/base/l-table.lua @@ -913,23 +913,27 @@ function table.reversed(t) end end -function table.sequenced(t,sep,simple) -- hash only - local s, n = { }, 0 - for k, v in sortedhash(t) do - if simple then - if v == true then - n = n + 1 - s[n] = k - elseif v and v~= "" then +function table.sequenced(t,sep) -- hash only + if t then + local s, n = { }, 0 + for k, v in sortedhash(t) do + if simple then + if v == true then + n = n + 1 + s[n] = k + elseif v and v~= "" then + n = n + 1 + s[n] = k .. "=" .. tostring(v) + end + else n = n + 1 s[n] = k .. "=" .. tostring(v) end - else - n = n + 1 - s[n] = k .. "=" .. tostring(v) end + return concat(s, sep or " | ") + else + return "" end - return concat(s, sep or " | ") end function table.print(t,...) diff --git a/tex/context/base/luat-env.lua b/tex/context/base/luat-env.lua index 4d9a44d42..c8a391e76 100644 --- a/tex/context/base/luat-env.lua +++ b/tex/context/base/luat-env.lua @@ -261,7 +261,7 @@ function environment.texfile(filename) return resolvers.findfile(filename,'tex') end -function environment.luafile(filename) +function environment.luafile(filename) -- needs checking local resolved = resolvers.findfile(filename,'tex') or "" if resolved ~= "" then return resolved diff --git a/tex/context/base/m-pstricks.lua b/tex/context/base/m-pstricks.lua index 7f795feac..b151e313a 100644 --- a/tex/context/base/m-pstricks.lua +++ b/tex/context/base/m-pstricks.lua @@ -39,12 +39,12 @@ local template = [[ \stoptext ]] -local modules = { } +local loaded = { } local graphics = 0 function moduledata.pstricks.usemodule(names) for name in gmatch(names,"([^%s,]+)") do - modules[#modules+1] = format([[\readfile{%s}{}{}]],name) + loaded[#loaded+1] = format([[\readfile{%s}{}{}]],name) end end @@ -55,10 +55,10 @@ function moduledata.pstricks.process(n) local tmpfile = name .. ".tmp" local epsfile = name .. ".ps" local pdffile = name .. ".pdf" - local modules = concat(modules,"\n") + local loaded = concat(loaded,"\n") os.remove(epsfile) os.remove(pdffile) - io.savedata(tmpfile,format(template,modules,data)) + io.savedata(tmpfile,format(template,loaded,data)) os.execute(format("mtxrun --script texexec %s --once --dvips",tmpfile)) if lfs.isfile(epsfile) then os.execute(format("ps2pdf %s %s",epsfile,pdffile)) diff --git a/tex/context/base/mult-low.lua b/tex/context/base/mult-low.lua index 7d5078b60..bffdd288f 100644 --- a/tex/context/base/mult-low.lua +++ b/tex/context/base/mult-low.lua @@ -271,6 +271,8 @@ return { -- "startnointerference", "stopnointerference", -- + "twodigits","threedigits", + -- "strut", "setstrut", "strutbox", "strutht", "strutdp", "strutwd", "struthtdp", "begstrut", "endstrut", "lineheight", } } diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf Binary files differindex c31ba403b..e5263bffa 100644 --- a/tex/context/base/status-files.pdf +++ b/tex/context/base/status-files.pdf diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf Binary files differindex 8ad2ea57b..6a8332f63 100644 --- a/tex/context/base/status-lua.pdf +++ b/tex/context/base/status-lua.pdf diff --git a/tex/context/base/util-sql-loggers.lua b/tex/context/base/util-sql-loggers.lua new file mode 100644 index 000000000..b2dccf4b5 --- /dev/null +++ b/tex/context/base/util-sql-loggers.lua @@ -0,0 +1,277 @@ +if not modules then modules = { } end modules ['util-sql-loggers'] = { + version = 1.001, + comment = "companion to lmx-*", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This is experimental code and currently part of the base installation simply +-- because it's easier to dirtribute this way. Eventually it will be documented +-- and the related scripts will show up as well. + +local tonumber = tonumber +local format = string.format +local concat = table.concat +local ostime, uuid, osfulltime = os.time, os.uuid, os.fulltime +local random = math.random + +local sql = utilities.sql +local loggers = { } +sql.loggers = loggers + +local trace_sql = false trackers.register("sql.loggers.trace", function(v) trace_sql = v end) +local report = logs.reporter("sql","loggers") + +loggers.newtoken = sql.tokens.new +local makeconverter = sql.makeconverter + +local function checkeddb(presets,datatable) + return sql.usedatabase(presets,datatable or presets.datatable or "loggers") +end + +loggers.usedb = checkeddb + +local totype = { + ["error"] = 1, [1] = 1, ["1"] = 1, + ["warning"] = 2, [2] = 2, ["2"] = 2, + ["debug"] = 3, [3] = 3, ["3"] = 3, + ["info"] = 4, [4] = 4, ["4"] = 4, +} + +local fromtype = { + ["error"] = "error", [1] = "error", ["1"] = "error", + ["warning"] = "warning", [2] = "warning", ["2"] = "warning", + ["debug"] = "debug", [3] = "debug", ["3"] = "debug", + ["info"] = "info", [4] = "info", ["4"] = "info", +} + +table.setmetatableindex(totype, function() return 4 end) +table.setmetatableindex(fromtype,function() return "info" end) + +loggers.totype = totype +loggers.fromtype = fromtype + +local template =[[ + CREATE TABLE IF NOT EXISTS %basename% ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `time` int(11) NOT NULL, + `type` int(11) NOT NULL, + `action` varchar(15) NOT NULL, + `data` longtext, + PRIMARY KEY (`id`), + UNIQUE KEY `id_unique_key` (`id`) + ) + DEFAULT CHARSET = utf8 ; +]] + +function loggers.createdb(presets,datatable) + + local db = checkeddb(presets,datatable) + + db.execute { + template = template, + variables = { + basename = db.basename, + }, + } + + report("datatable %q created in %q",db.name,db.base) + + return db + +end + +local template =[[ + DROP TABLE IF EXISTS %basename% ; +]] + +function loggers.deletedb(presets,datatable) + + local db = checkeddb(presets,datatable) + + db.execute { + template = template, + variables = { + basename = db.basename, + }, + } + + report("datatable %q removed in %q",db.name,db.base) + +end + +local template =[[ + INSERT INTO %basename% ( + `time`, + `type`, + `action`, + `data` + ) VALUES ( + %time%, + %type%, + '%action%', + '%[data]%' + ) ; +]] + +function loggers.save(db,data) -- beware, we pass type and action in the data (saves a table) + + if data then + + local time = ostime() + local kind = totype[data.type] + local action = data.action or "unknown" + + data.type = nil + data.action = nil + + db.execute { + template = template, + variables = { + basename = db.basename, + time = ostime(), + type = kind, + action = action, + data = data and db.serialize(data,"return") or "", + }, + } + + end + +end + +-- local template =[[ +-- REMOVE FROM +-- %basename% +-- WHERE +-- `token` = '%token%' ; +-- ]] +-- +-- function loggers.remove(db,token) +-- +-- db.execute { +-- template = template, +-- variables = { +-- basename = db.basename, +-- token = token, +-- }, +-- } +-- +-- if trace_sql then +-- report("removed: %s",token) +-- end +-- +-- end + +local template_nop =[[ + SELECT + `time`, + `type`, + `action`, + `data` + FROM + %basename% + ORDER BY + `time`, `type`, `action` + DESC LIMIT + %limit% ; +]] + +local template_yes =[[ + SELECT + `time`, + `type`, + `action`, + `data` + FROM + %basename% + %WHERE% + ORDER BY + `time`, `type`, `action` + DESC LIMIT + %limit% ; +]] + +local converter = makeconverter { + -- { name = "time", type = os.localtime }, + { name = "time", type = "number" }, + { name = "type", type = fromtype }, + { name = "action", type = "string" }, + { name = "data", type = "deserialize" }, +} + +function loggers.collect(db,specification) + + specification = specification or { } + + local start = specification.start + local stop = specification.stop + local limit = specification.limit or 100 + local kind = specification.type + local action = specification.action + + local filtered = start or stop + + local where = { } + + if filtered then + local today = os.date("*t") + + if type(start) ~= "table" then + start = { } + end + start = os.time { + day = start.day or today.day, + month = start.month or today.month, + year = start.year or today.year, + hour = start.hour or 0, + minute = start.minute or 0, + second = start.second or 0, + isdst = true, + } + + if type(stop) ~= "table" then + stop = { } + end + stop = os.time { + day = stop.day or today.day, + month = stop.month or today.month, + year = stop.year or today.year, + hour = stop.hour or 24, + minute = stop.minute or 0, + second = stop.second or 0, + isdst = true, + } + + -- report("filter: %s => %s",start,stop) + + where[#where+1] = format("`time` BETWEEN %s AND %s",start,stop) + + end + + if kind then + where[#where+1] = format("`type` = %s",totype[kind]) + end + + if action then + where[#where+1] = format("`action` = '%s'",action) + end + + local records = db.execute { + template = filtered and template_yes or template_nop, + converter = converter, + variables = { + basename = db.basename, + limit = limit, + WHERE = #where > 0 and format("WHERE\n%s",concat(where," AND ")) or "", + }, + } + + if trace_sql then + report("collected: %s loggers",#records) + end + + return records, keys + +end diff --git a/tex/context/base/util-sql-sessions.lua b/tex/context/base/util-sql-sessions.lua new file mode 100644 index 000000000..40556dd5e --- /dev/null +++ b/tex/context/base/util-sql-sessions.lua @@ -0,0 +1,349 @@ +if not modules then modules = { } end modules ['util-sql-sessions'] = { + version = 1.001, + comment = "companion to lmx-*", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This is experimental code and currently part of the base installation simply +-- because it's easier to dirtribute this way. Eventually it will be documented +-- and the related scripts will show up as well. + +-- maybe store threshold in session (in seconds) + +local tonumber = tonumber +local format = string.format +local ostime, uuid, osfulltime = os.time, os.uuid, os.fulltime +local random = math.random + +-- In older frameworks we kept a session table in memory. This time we +-- follow a route where we store session data in a sql table. Each session +-- has a token (similar to what we do on q2p and pod services), a data +-- blob which is just a serialized lua table (we could consider a dump instead) +-- and two times: the creation and last accessed time. The first one is handy +-- for statistics and the second one for cleanup. Both are just numbers so that +-- we don't have to waste code on conversions. Anyhow, we provide variants so that +-- we can always choose what is best. + +local sql = utilities.sql +local sessions = { } +sql.sessions = sessions + +local trace_sql = false trackers.register("sql.sessions.trace", function(v) trace_sql = v end) +local report = logs.reporter("sql","sessions") + +sessions.newtoken = sql.tokens.new + +local function checkeddb(presets,datatable) + return sql.usedatabase(presets,datatable or presets.datatable or "sessions") +end + +sessions.usedb = checkeddb + +local template =[[ + CREATE TABLE IF NOT EXISTS %basename% ( + `token` varchar(50) NOT NULL, + `data` longtext NOT NULL, + `created` int(11) NOT NULL, + `accessed` int(11) NOT NULL, + UNIQUE KEY `token_unique_key` (`token`) + ) + DEFAULT CHARSET = utf8 ; +]] + +function sessions.createdb(presets,datatable) + + local db = checkeddb(presets,datatable) + + db.execute { + template = template, + variables = { + basename = db.basename, + }, + } + + report("datatable %q created in %q",db.name,db.base) + + return db + +end + +local template =[[ + DROP TABLE IF EXISTS %basename% ; +]] + +function sessions.deletedb(presets,datatable) + + local db = checkeddb(presets,datatable) + + db.execute { + template = template, + variables = { + basename = db.basename, + }, + } + + report("datatable %q removed in %q",db.name,db.base) + +end + +local template =[[ + INSERT INTO %basename% ( + `token`, + `created`, + `accessed`, + `data` + ) VALUES ( + '%token%', + %time%, + %time%, + '%[data]%' + ) ; +]] + +function sessions.create(db,data) + + local token = sessions.newtoken() + local time = ostime() + + db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + time = time, + data = db.serialize(data or { },"return") + }, + } + + if trace_sql then + report("created: %s at %s",token,osfulltime(time)) + end + + return { + token = token, + created = time, + accessed = time, + data = data, + } +end + +local template =[[ + UPDATE + %basename% + SET + `data` = '%[data]%', + `accessed` = %time% + WHERE + `token` = '%token%' ; +]] + +function sessions.save(db,session) + + local time = ostime() + local data = db.serialize(session.data or { },"return") + local token = session.token + + session.accessed = time + + db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + time = ostime(), + data = data, + }, + } + + if trace_sql then + report("saved: %s at %s",token,osfulltime(time)) + end + + return session +end + +local template = [[ + UPDATE + %basename% + SET + `accessed` = %time% + WHERE + `token` = '%token%' ; +]] + +function sessions.touch(db,token) + + db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + time = ostime(), + }, + } + +end + +local template = [[ + UPDATE + %basename% + SET + `accessed` = %time% + WHERE + `token` = '%token%' ; + SELECT + * + FROM + %basename% + WHERE + `token` = '%token%' ; +]] + +function sessions.restore(db,token) + + local records, keys = db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + time = ostime(), + }, + } + + local record = records and records[1] + + if record then + if trace_sql then + report("restored: %s",token) + end + record.data = db.deserialize(record.data or "") + return record, keys + elseif trace_sql then + report("unknown: %s",token) + end + +end + +local template =[[ + DELETE FROM + %basename% + WHERE + `token` = '%token%' ; +]] + +function sessions.remove(db,token) + + db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + }, + } + + if trace_sql then + report("removed: %s",token) + end + +end + +local template_collect_yes =[[ + SELECT + * + FROM + %basename% + ORDER BY + `created` ; +]] + +local template_collect_nop =[[ + SELECT + `accessed`, + `created`, + `accessed`, + `token` + FROM + %basename% + ORDER BY + `created` ; +]] + +function sessions.collect(db,nodata) + + local records, keys = db.execute { + template = nodata and template_collect_nop or template_collect_yes, + variables = { + basename = db.basename, + }, + } + + if not nodata then + db.unpackdata(records) + end + + if trace_sql then + report("collected: %s sessions",#records) + end + + return records, keys + +end + +local template_cleanup_yes =[[ + SELECT + * + FROM + %basename% + WHERE + `accessed` < %time% + ORDER BY + `created` ; + DELETE FROM + %basename% + WHERE + `accessed` < %time% ; +]] + +local template_cleanup_nop =[[ + SELECT + `accessed`, + `created`, + `accessed`, + `token` + FROM + %basename% + WHERE + `accessed` < %time% + ORDER BY + `created` ; + DELETE FROM + %basename% + WHERE + `accessed` < %time% ; +]] + +function sessions.cleanupdb(db,delta,nodata) + + local time = ostime() + + local records, keys = db.execute { + template = nodata and template_cleanup_nop or template_cleanup_yes, + variables = { + basename = db.basename, + time = time - delta + }, + } + + if not nodata then + db.unpackdata(records) + end + + if trace_sql then + report("cleaned: %s seconds before %s",delta,osfulltime(time)) + end + + return records, keys + +end diff --git a/tex/context/base/util-sql-tickets.lua b/tex/context/base/util-sql-tickets.lua new file mode 100644 index 000000000..ad885e34e --- /dev/null +++ b/tex/context/base/util-sql-tickets.lua @@ -0,0 +1,698 @@ +if not modules then modules = { } end modules ['util-sql-tickets'] = { + version = 1.001, + comment = "companion to lmx-*", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This is experimental code and currently part of the base installation simply +-- because it's easier to dirtribute this way. Eventually it will be documented +-- and the related scripts will show up as well. + +local tonumber = tonumber +local format = string.format +local ostime, uuid, osfulltime = os.time, os.uuid, os.fulltime +local random = math.random +local concat = table.concat + +local sql = utilities.sql +local tickets = { } +sql.tickets = tickets + +local trace_sql = false trackers.register("sql.tickets.trace", function(v) trace_sql = v end) +local report = logs.reporter("sql","tickets") + +local serialize = sql.serialize +local deserialize = sql.deserialize +local execute = sql.execute + +tickets.newtoken = sql.tokens.new + +local statustags = { [0] = -- beware index can be string or number, maybe status should be a string in the database + "unknown", + "pending", + "busy", + "finished", + "error", + "deleted", +} + +local status = table.swapped(statustags) + +tickets.status = status +tickets.statustags = statustags + +local function checkeddb(presets,datatable) + return sql.usedatabase(presets,datatable or presets.datatable or "tickets") +end + +tickets.usedb = checkeddb + +local template =[[ + CREATE TABLE IF NOT EXISTS %basename% ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `token` varchar(50) NOT NULL, + `subtoken` INT(11) NOT NULL, + `created` int(11) NOT NULL, + `accessed` int(11) NOT NULL, + `category` int(11) NOT NULL, + `status` int(11) NOT NULL, + `usertoken` varchar(50) NOT NULL, + `data` longtext NOT NULL, + `comment` longtext NOT NULL, + + PRIMARY KEY (`id`), + UNIQUE INDEX `id_unique_index` (`id` ASC), + KEY `token_unique_key` (`token`) + ) + DEFAULT CHARSET = utf8 ; +]] + +function tickets.createdb(presets,datatable) + + local db = checkeddb(presets,datatable) + + local data, keys = db.execute { + template = template, + variables = { + basename = db.basename, + }, + } + + report("datatable %q created in %q",db.name,db.base) + + return db + +end + +local template =[[ + DROP TABLE IF EXISTS %basename% ; +]] + +function tickets.deletedb(presets,datatable) + + local db = checkeddb(presets,datatable) + + local data, keys = db.execute { + template = template, + variables = { + basename = db.basename, + }, + } + + report("datatable %q removed in %q",db.name,db.base) + +end + +local template =[[ + LOCK TABLES + %basename% + WRITE ; + INSERT INTO %basename% ( + `token`, + `subtoken`, + `created`, + `accessed`, + `status`, + `category`, + `usertoken`, + `data`, + `comment` + ) VALUES ( + '%token%', + %subtoken%, + %time%, + %time%, + %status%, + %category%, + '%usertoken%', + '%[data]%', + '%[comment]%' + ) ; + SELECT + LAST_INSERT_ID() AS `id` ; + UNLOCK TABLES ; +]] + +function tickets.create(db,ticket) + + local token = ticket.token or tickets.newtoken() + local time = ostime() + local status = ticket.status or 0 + local category = ticket.category or 0 + local subtoken = ticket.subtoken or 0 + local usertoken = ticket.usertoken or "" + local comment = ticket.comment or "" + + local result, message = db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + subtoken = subtoken, + time = time, + status = status, + category = category, + usertoken = usertoken, + data = db.serialize(ticket.data or { },"return"), + comment = comment, + }, + } + + if trace_sql then + report("created: %s at %s",token,osfulltime(time)) + end + + local r = result and result[1] + + if r then + + return { + id = r.id, + token = token, + subtoken = subtoken, + created = time, + accessed = time, + status = status, + category = category, + usertoken = usertoken, + data = data, + comment = comment, + } + + end +end + +local template =[[ + LOCK TABLES + %basename% + WRITE ; + UPDATE %basename% SET + `data` = '%[data]%', + `status` = %status%, + `accessed` = %time% + WHERE + `id` = %id% ; + UNLOCK TABLES ; +]] + +function tickets.save(db,ticket) + + local time = ostime() + local data = db.serialize(ticket.data or { },"return") + local status = ticket.status or 0 + local id = ticket.id + + if not status then + status = 0 + ticket.status = 0 + end + + ticket.accessed = time + + db.execute { + template = template, + variables = { + basename = db.basename, + id = id, + time = ostime(), + status = status, + data = data, + }, + } + + if trace_sql then + report("saved: id %s, time %s",id,osfulltime(time)) + end + + return ticket +end + +local template =[[ + UPDATE + %basename% + SET + `accessed` = %time% + WHERE + `token` = '%token%' ; + + SELECT + * + FROM + %basename% + WHERE + `id` = %id% ; +]] + +function tickets.restore(db,id) + + local record, keys = db.execute { + template = template, + variables = { + basename = db.basename, + id = id, + time = ostime(), + }, + } + + local record = record and record[1] + + if record then + if trace_sql then + report("restored: id %s",id) + end + record.data = db.deserialize(record.data or "") + return record + elseif trace_sql then + report("unknown: id %s",id) + end + +end + +local template =[[ + DELETE FROM + %basename% + WHERE + `id` = %id% ; +]] + +function tickets.remove(db,id) + + db.execute { + template = template, + variables = { + basename = db.basename, + id = id, + }, + } + + if trace_sql then + report("removed: id %s",id) + end + +end + +local template_yes =[[ + SELECT + * + FROM + %basename% + ORDER BY + `created` ; +]] + +local template_nop =[[ + SELECT + `created`, + `usertoken`, + `accessed`, + `status` + FROM + %basename% + ORDER BY + `created` ; +]] + +function tickets.collect(db,nodata) + + local records, keys = db.execute { + template = nodata and template_nop or template_yes, + variables = { + basename = db.basename, + token = token, + }, + } + + if not nodata then + db.unpackdata(records) + end + + if trace_sql then + report("collected: %s tickets",#records) + end + + return records, keys + +end + +local template =[[ + DELETE FROM + %basename% + WHERE + `accessed` < %time% OR `status` = 5 ; +]] + +local template_cleanup_yes =[[ + SELECT + * + FROM + %basename% + WHERE + `accessed` < %time% + ORDER BY + `created` ; + DELETE FROM + %basename% + WHERE + `accessed` < %time% OR `status` = 5 ; +]] + +local template_cleanup_nop =[[ + SELECT + `accessed`, + `created`, + `accessed`, + `token` + `usertoken` + FROM + %basename% + WHERE + `accessed` < %time% + ORDER BY + `created` ; + DELETE FROM + %basename% + WHERE + `accessed` < %time% OR `status` = 5 ; +]] + +function tickets.cleanupdb(db,delta,nodata) -- maybe delta in db + + local time = delta and (ostime() - delta) or 0 + + local records, keys = db.execute { + template = nodata and template_cleanup_nop or template_cleanup_yes, + variables = { + basename = db.basename, + time = time, + }, + } + + if not nodata then + db.unpackdata(records) + end + + if trace_sql then + report("cleaned: %s seconds before %s",delta,osfulltime(time)) + end + + return records, keys + +end + +-- status related functions + +local template =[[ + SELECT + `status` + FROM + %basename% + WHERE + `token` = '%token%' ; +]] + +function tickets.getstatus(db,token) + + local record, keys = db.execute { + template = template, + variables = { + basename = db.basename, + token = token, + }, + } + + local record = record and record[1] + + return record and record.status or 0 + +end + +local template =[[ + SELECT + `status` + FROM + %basename% + WHERE + `status` = 5 OR `accessed` < %time% ; +]] + +function tickets.getobsolete(db,delta) + + local time = delta and (ostime() - delta) or 0 + + local records = db.execute { + template = template, + variables = { + basename = db.basename, + time = time, + }, + } + + db.unpackdata(records) + + return records + +end + +local template =[[ + SELECT + `id` + FROM + %basename% + WHERE + `status` = %status% + LIMIT + 1 ; +]] + +function tickets.hasstatus(db,status) + + local record = db.execute { + template = template, + variables = { + basename = db.basename, + status = status or 0, + }, + } + + return record and #record > 0 or false + +end + +local template =[[ + UPDATE + %basename% + SET + `status` = %status%, + `accessed` = %time% + WHERE + `id` = %id% ; +]] + +function tickets.setstatus(db,id,status) + + local record, keys = db.execute { + template = template, + variables = { + basename = db.basename, + id = id, + time = ostime(), + status = status or 0, + }, + } + +end + +local template =[[ + DELETE FROM + %basename% + WHERE + `status` IN (%status%) ; +]] + +function tickets.prunedb(db,status) + + if type(status) == "table" then + status = concat(status,",") + end + + local data, keys = db.execute { + template = template, + variables = { + basename = db.basename, + status = status or 0, + }, + } + + if trace_sql then + report("pruned: status %s removed",status) + end + +end + +local template_a = [[ + LOCK TABLES + %basename% + WRITE ; + SET + @first_token = "?" ; + SELECT + `token` + INTO + @first_token + FROM + %basename% + WHERE + `status` = %status% + ORDER BY + `id` + LIMIT 1 ; + UPDATE + %basename% + SET + `status` = %newstatus%, + `accessed` = %time% + WHERE + `token` = @first_token ; + SELECT + * + FROM + %basename% + WHERE + `token` = @first_token + ORDER BY + `id` ; + UNLOCK TABLES ; +]] + +local template_b = [[ + SET + @first_token = "?" ; + SELECT + `token` + INTO + @first_token + FROM + %basename% + WHERE + `status` = %status% + ORDER BY + `id` + LIMIT 1 ; + SELECT + * + FROM + %basename% + WHERE + `token` = @first_token + ORDER BY + `id` ; +]] + +function tickets.getfirstwithstatus(db,status,newstatus) + + local records + + if type(newstatus) == "number" then + + records = db.execute { + template = template_a, + variables = { + basename = db.basename, + status = status or 0, + newstatus = newstatus, + time = ostime(), + }, + } + + + else + + records = db.execute { + template = template_b, + variables = { + basename = db.basename, + status = status or 0, + }, + } + + end + + if type(records) == "table" and #records > 0 then + + for i=1,#records do + local record = records[i] + record.data = db.deserialize(record.data or "") + record.status = newstatus + end + + return records + + end +end + +local template =[[ + SELECT + * + FROM + %basename% + WHERE + `usertoken` = '%usertoken%' AND `status` != 5 + ORDER BY + `created` ; +]] + +function tickets.getusertickets(db,usertoken) + + -- todo: update accessed + -- todo: get less fields + -- maybe only data for status changed (hard to check) + + local records, keys = db.execute { + template = template, + variables = { + basename = db.basename, + usertoken = usertoken, + }, + } + + db.unpackdata(records) + + return records + +end + +local template =[[ + LOCK TABLES + %basename% + WRITE ; + UPDATE %basename% SET + `status` = 5 + WHERE + `usertoken` = '%usertoken%' ; + UNLOCK TABLES ; +]] + +function tickets.removeusertickets(db,usertoken) + + db.execute { + template = template, + variables = { + basename = db.basename, + usertoken = usertoken, + }, + } + + if trace_sql then + report("removed: usertoken %s",usertoken) + end + +end + +-- -- left-overs -- + +-- LOCK TABLES `m4alltickets` WRITE ; +-- CREATE TEMPORARY TABLE ticketset SELECT * FROM m4alltickets WHERE token = @first_token ; +-- DROP TABLE ticketset ; +-- UNLOCK TABLES ; diff --git a/tex/context/base/util-sql.lua b/tex/context/base/util-sql.lua index f86607377..fea60c96a 100644 --- a/tex/context/base/util-sql.lua +++ b/tex/context/base/util-sql.lua @@ -25,6 +25,9 @@ if not modules then modules = { } end modules ['util-sql'] = { -- efficiency issues (like creating a keys and types table for each row) but that could be -- optimized. Anyhow, fecthing results can be done as follows: +-- We use the template mechanism from util-tpl which inturn is just using the dos cq +-- windows convention of %whatever% variables that I've used for ages. + -- local function collect_1(r) -- local t = { } -- for i=1,r:numrows() do @@ -75,7 +78,7 @@ if not modules then modules = { } end modules ['util-sql'] = { local format, match = string.format, string.match local random = math.random -local rawset, setmetatable, loadstring, type = rawset, setmetatable, loadstring, type +local rawset, setmetatable, getmetatable, loadstring, type = rawset, setmetatable, getmetatable, loadstring, type local P, S, V, C, Cs, Ct, Cc, Cg, Cf, patterns, lpegmatch = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Ct, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.patterns, lpeg.match local concat = table.concat @@ -87,6 +90,9 @@ local trace_sql = false trackers.register("sql.trace", function(v) trace local trace_queries = false trackers.register("sql.queries",function(v) trace_queries = v end) local report_state = logs.reporter("sql") +-- trace_sql = true +-- trace_queries = true + utilities.sql = utilities.sql or { } local sql = utilities.sql @@ -96,8 +102,11 @@ local loadtemplate = utilities.templates.load local methods = { } sql.methods = methods -sql.serialize = table.fastserialize -sql.deserialize = table.deserialize +local serialize = table.fastserialize +local deserialize = table.deserialize + +sql.serialize = serialize +sql.deserialize = deserialize local defaults = { __index = { @@ -207,14 +216,19 @@ local function validspecification(specification) presets = dofile(presets) end if type(presets) == "table" then - setmetatable(presets,defaults) + local m = getmetatable(presets) + if m then + setmetatable(m,defaults) + else + setmetatable(presets,defaults) + end setmetatable(specification,{ __index = presets }) else setmetatable(specification,defaults) end - local templatefile = specification.templatefile - local queryfile = specification.queryfile or file.nameonly(templatefile) .. "-temp.sql" - local resultfile = specification.resultfile or file.nameonly(templatefile) .. "-temp.dat" + local templatefile = specification.templatefile or "query" + local queryfile = specification.queryfile or presets.queryfile or file.nameonly(templatefile) .. "-temp.sql" + local resultfile = specification.resultfile or presets.resultfile or file.nameonly(templatefile) .. "-temp.dat" specification.queryfile = queryfile specification.resultfile = resultfile if trace_sql then @@ -310,7 +324,7 @@ sql.splitdata = splitdata local function execute(specification) if trace_sql then - report_state("executing") + report_state("executing client") end if not validspecification(specification) then report_state("error in specification") @@ -342,6 +356,7 @@ methods.client = { execute = execute, serialize = serialize, deserialize = deserialize, + usesfiles = true, } local function dataloaded(specification) @@ -368,7 +383,7 @@ end local function execute(specification) if trace_sql then - report_state("executing") + report_state("executing lmxsql") end if not validspecification(specification) then report_state("error in specification") @@ -400,6 +415,7 @@ methods.lmxsql = { execute = execute, serialize = serialize, deserialize = deserialize, + usesfiles = true, } local mysql = nil @@ -447,7 +463,7 @@ local query = whitespace * whitespace local splitter = Ct(query * (separator * query)^0) -local function datafetched(specification,query) +local function datafetched(specification,query,converter) local id = specification.id local session, connection if id then @@ -466,18 +482,32 @@ local function datafetched(specification,query) connection = connect(session,specification) end if not connection then + report_state("error in connection: %s@%s to %s:%s", + specification.database or "no database", + specification.username or "no username", + specification.host or "no host", + specification.port or "no port" + ) return { }, { } end query = lpegmatch(splitter,query) - local result, message + local result, message, okay for i=1,#query do local q = query[i] - result, message = connection:execute(q) - if message then - report_state("error in query: %s",string.collapsespaces(q)) + local r, m = connection:execute(q) + if m then + report_state("error in query, stage 1: %s",string.collapsespaces(q)) + message = message and format("%s\n%s",message,m) or m + end + local t = type(r) + if t == "userdata" then + result = r + okay = true + elseif t == "number" then + okay = true end end - if not result and id then + if not okay and id then if session then session:close() end @@ -489,34 +519,46 @@ local function datafetched(specification,query) cache[id] = { session = session, connection = connection } for i=1,#query do local q = query[i] - result, message = connection:execute(q) - if message then - report_state("error in query: %s",string.collapsespaces(q)) + local r, m = connection:execute(q) + if m then + report_state("error in query, stage 2: %s",string.collapsespaces(q)) + message = message and format("%s\n%s",message,m) or m + end + local t = type(r) + if t == "userdata" then + result = r + okay = true + elseif t == "number" then + okay = true end end end local data, keys - if result and type(result) ~= "number" then - keys = result:getcolnames() - if keys then - local n = result:numrows() or 0 - if n == 0 then - data = { } - -- elseif n == 1 then - -- -- data = { result:fetch({},"a") } - else - data = { } - -- for i=1,n do - -- data[i] = result:fetch({},"a") - -- end - local k = #keys - for i=1,n do - local v = { result:fetch() } - local d = { } - for i=1,k do - d[keys[i]] = v[i] + if result then + if converter then + data = converter(result,deserialize) + else + keys = result:getcolnames() + if keys then + local n = result:numrows() or 0 + if n == 0 then + data = { } + -- elseif n == 1 then + -- -- data = { result:fetch({},"a") } + else + data = { } + -- for i=1,n do + -- data[i] = result:fetch({},"a") + -- end + local k = #keys + for i=1,n do + local v = { result:fetch() } + local d = { } + for i=1,k do + d[keys[i]] = v[i] + end + data[#data+1] = d end - data[#data+1] = d end end end @@ -547,7 +589,7 @@ local function execute(specification) end end if trace_sql then - report_state("executing") + report_state("executing library") end if not validspecification(specification) then report_state("error in specification") @@ -558,7 +600,7 @@ local function execute(specification) report_state("error in preparation") return end - local data, keys = datafetched(specification,query) + local data, keys = datafetched(specification,query,specification.converter) if not data then report_state("error in fetching") return @@ -571,6 +613,7 @@ methods.library = { execute = execute, serialize = serialize, deserialize = deserialize, + usesfiles = false, } -- -- -- @@ -590,6 +633,66 @@ end sql.setmethod("client") +-- helper: + +local execute = sql.execute + +function sql.usedatabase(presets,datatable) + local name = datatable or presets.datatable + if name then + local method = presets.method and sql.methods[presets.method] or sql.methods.client + local base = presets.database or "test" + local basename = format("`%s`.`%s`",base,name) + m_execute = execute + deserialize = deserialize + serialize = serialize + if method then + m_execute = method.execute or m_execute + deserialize = method.deserialize or deserialize + serialize = method.serialize or serialize + end + local execute + if method.usesfiles then + local queryfile = presets.queryfile or format("%s-temp.sql",name) + local resultfile = presets.resultfile or format("%s-temp.dat",name) + execute = function(specification) -- variables template + if not specification.presets then specification.presets = presets end + if not specification.queryfile then specification.queryfile = queryfile end + if not specification.resultfile then specification.resultfile = queryfile end + return m_execute(specification) + end + else + execute = function(specification) -- variables template + if not specification.presets then specification.presets = presets end + return m_execute(specification) + end + end + local function unpackdata(records,name) + if records then + name = name or "data" + for i=1,#records do + local record = records[i] + local data = record[name] + if data then + record[name] = deserialize(data) + end + end + end + end + return { + presets = preset, + base = base, + name = name, + basename = basename, + execute = execute, + serialize = serialize, + deserialize = deserialize, + unpackdata = unpackdata, + } + end +end + + -- local data = utilities.sql.prepare { -- templatefile = "test.sql", -- variables = { }, @@ -631,12 +734,92 @@ sql.setmethod("client") sql.tokens = { length = 42, -- but in practice we will reserve some 50 characters new = function() - return format("%s-%s",osuuid(),match(format("%x05",random(ostime())),".-(.....)$")) -- 36 + 1 + 5 = 42 - end, -- or random(0xFFFFF*osclock()) + return format("%s-%x05",osuuid(),random(0xFFFFF)) -- 36 + 1 + 5 = 42 + end, } -- -- -- +local converters = { } + +sql.converters = converters + +local template = [[ +local converters = utilities.sql.converters + +local tostring = tostring +local tonumber = tonumber +local toboolean = toboolean + +%s + +return function(result,deserialize) + if not result then + return { } + end + local nofrows = result:numrows() or 0 + if nofrows == 0 then + return { } + end + local data = { } + for i=1,nofrows do + local v = { result:fetch() } + data[#data+1] = { + %s + } + end + return data +end +]] + +function sql.makeconverter(entries,deserialize) + local shortcuts = { } + local assignments = { } + for i=1,#entries do + local entry = entries[i] + local nam = entry.name + local typ = entry.type + if typ == "boolean" then + assignments[i] = format("[%q] = toboolean(v[%s],true),",nam,i) + elseif typ == "number" then + assignments[i] = format("[%q] = tonumber(v[%s]),",nam,i) + elseif type(typ) == "function" then + local c = #converters + 1 + converters[c] = typ + shortcuts[#shortcuts+1] = format("local fun_%s = converters[%s]",c,c) + assignments[i] = format("[%q] = fun_%s(v[%s]),",nam,c,i) + elseif type(typ) == "table" then + local c = #converters + 1 + converters[c] = typ + shortcuts[#shortcuts+1] = format("local tab_%s = converters[%s]",c,c) + assignments[i] = format("[%q] = tab_%s[v[%s]],",nam,#converters,i) + elseif typ == "deserialize" then + assignments[i] = format("[%q] = deserialize(v[%s]),",nam,i) + else + assignments[i] = format("[%q] = v[%s],",nam,i) + end + end + local code = string.format(template,table.concat(shortcuts,"\n"),table.concat(assignments,"\n ")) + local func = loadstring(code) + if type(func) == "function" then + return func(), code + else + return false, code + end +end + +-- local func, code = sql.makeconverter { +-- { name = "a", type = "number" }, +-- { name = "b", type = "string" }, +-- { name = "c", type = "boolean" }, +-- { name = "d", type = { x = "1" } }, +-- { name = "e", type = os.fulltime }, +-- } +-- +-- print(code) + +-- -- -- + if tex and tex.systemmodes then local droptable = table.drop diff --git a/tex/context/base/util-tpl.lua b/tex/context/base/util-tpl.lua index f6648b224..4cde1863b 100644 --- a/tex/context/base/util-tpl.lua +++ b/tex/context/base/util-tpl.lua @@ -6,10 +6,9 @@ if not modules then modules = { } end modules ['util-tpl'] = { license = "see context related readme files" } --- experimental code - --- maybe make %% scanning optional --- maybe use $[ and ]$ or {{ }} +-- This is experimental code. Coming from dos and windows, I've always used %whatever% +-- as template variables so let's stick to it. After all, it's easy to parse and stands +-- out well. A double %% is turned into a regular %. utilities.templates = utilities.templates or { } local templates = utilities.templates @@ -24,7 +23,7 @@ local P, C, Cs, Carg, lpegmatch = lpeg.P, lpeg.C, lpeg.Cs, lpeg.Carg, lpeg.match local replacer -local function replacekey(k,t) +local function replacekey(k,t,recursive) local v = t[k] if not v then if trace_template then @@ -35,7 +34,11 @@ local function replacekey(k,t) if trace_template then report_template("setting key %q to value %q",k,v) end - return lpegmatch(replacer,v,1,t) -- recursive + if recursive then + return lpegmatch(replacer,v,1,t) + else + return v + end end end @@ -56,9 +59,9 @@ local escapers = { end, } -local function replacekeyunquoted(s,t,how) -- ".. \" " +local function replacekeyunquoted(s,t,how,recurse) -- ".. \" " local escaper = how and escapers[how] or escapers.lua - return escaper(replacekey(s,t)) + return escaper(replacekey(s,t,recurse)) end local single = P("%") -- test %test% test : resolves test @@ -72,15 +75,15 @@ local nodouble = double / '' local nolquoted = lquoted / '' local norquoted = rquoted / '' -local key = nosingle * (C((1-nosingle)^1 * Carg(1))/replacekey) * nosingle -local unquoted = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2))/replacekeyunquoted) * norquoted +local key = nosingle * (C((1-nosingle)^1 * Carg(1) * Carg(2) * Carg(3))/replacekey) * nosingle +local unquoted = nolquoted * ((C((1 - norquoted)^1) * Carg(1) * Carg(2) * Carg(3))/replacekeyunquoted) * norquoted local any = P(1) replacer = Cs((unquoted + escape + key + any)^0) -local function replace(str,mapping,how) +local function replace(str,mapping,how,recurse) if mapping then - return lpegmatch(replacer,str,1,mapping,how or "lua") or str + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str else return str end @@ -91,25 +94,24 @@ end templates.replace = replace -function templates.load(filename,mapping) +function templates.load(filename,mapping,how,recurse) local data = io.loaddata(filename) or "" if mapping and next(mapping) then - return replace(data,mapping) + return replace(data,mapping,how,recurse) else return data end end -function templates.resolve(t,mapping) +function templates.resolve(t,mapping,how,recurse) if not mapping then mapping = t end for k, v in next, t do - t[k] = replace(v,mapping) + t[k] = replace(v,mapping,how,recurse) end return t end -- inspect(utilities.templates.replace("test %one% test", { one = "%two%", two = "two" })) -- inspect(utilities.templates.resolve({ one = "%two%", two = "two", three = "%three%" })) - diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index b40cb3eae..323c70c3e 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 09/16/12 23:18:22 +-- merge date : 09/21/12 20:58:19 do -- begin closure to overcome local limits and interference @@ -1059,23 +1059,27 @@ function table.reversed(t) end end -function table.sequenced(t,sep,simple) -- hash only - local s, n = { }, 0 - for k, v in sortedhash(t) do - if simple then - if v == true then - n = n + 1 - s[n] = k - elseif v and v~= "" then +function table.sequenced(t,sep) -- hash only + if t then + local s, n = { }, 0 + for k, v in sortedhash(t) do + if simple then + if v == true then + n = n + 1 + s[n] = k + elseif v and v~= "" then + n = n + 1 + s[n] = k .. "=" .. tostring(v) + end + else n = n + 1 s[n] = k .. "=" .. tostring(v) end - else - n = n + 1 - s[n] = k .. "=" .. tostring(v) end + return concat(s, sep or " | ") + else + return "" end - return concat(s, sep or " | ") end function table.print(t,...) |
