diff options
Diffstat (limited to 'tex/context/base/mkiv/util-sql-imp-ffi.lua')
| -rw-r--r-- | tex/context/base/mkiv/util-sql-imp-ffi.lua | 590 |
1 files changed, 321 insertions, 269 deletions
diff --git a/tex/context/base/mkiv/util-sql-imp-ffi.lua b/tex/context/base/mkiv/util-sql-imp-ffi.lua index 2a2bc6569..3731933f1 100644 --- a/tex/context/base/mkiv/util-sql-imp-ffi.lua +++ b/tex/context/base/mkiv/util-sql-imp-ffi.lua @@ -8,7 +8,7 @@ if not modules then modules = { } end modules ['util-sql-imp-ffi'] = { -- I looked at luajit-mysql to see how the ffi mapping was done but it didn't work -- out that well (at least not on windows) but I got the picture. As I have somewhat --- different demands I simplified / redid the ffi bti and just took the swiglib +-- different demands I simplified / redid the ffi bit and just took the swiglib -- variant and adapted that. local tonumber = tonumber @@ -17,6 +17,7 @@ local format, byte = string.format, string.byte local lpegmatch = lpeg.match local setmetatable, type = setmetatable, type local sleep = os.sleep +local formatters = string.formatters local trace_sql = false trackers.register("sql.trace", function(v) trace_sql = v end) local trace_queries = false trackers.register("sql.queries",function(v) trace_queries = v end) @@ -33,6 +34,9 @@ ffi.cdef [[ a query. The rest is handled already in the Lua code elsewhere. */ + void free(void*ptr); + void * malloc(size_t size); + typedef void MYSQL_instance; typedef void MYSQL_result; typedef char **MYSQL_row; @@ -115,6 +119,14 @@ ffi.cdef [[ MYSQL_result *result ); + unsigned int mysql_affected_rows ( + MYSQL_instance *mysql + ); + + unsigned int mysql_field_count ( + MYSQL_instance *mysql + ); + unsigned int mysql_num_fields ( MYSQL_result *res ); @@ -163,12 +175,14 @@ local dataprepared = helpers.preparetemplate local serialize = sql.serialize local deserialize = sql.deserialize -local mysql_initialize = mysql.mysql_init +local mysql_open_session = mysql.mysql_init local mysql_open_connection = mysql.mysql_real_connect local mysql_execute_query = mysql.mysql_real_query local mysql_close_connection = mysql.mysql_close +local mysql_affected_rows = mysql.mysql_affected_rows +local mysql_field_count = mysql.mysql_field_count local mysql_field_seek = mysql.mysql_field_seek local mysql_num_fields = mysql.mysql_num_fields local mysql_fetch_fields = mysql.mysql_fetch_fields @@ -180,6 +194,7 @@ local mysql_init = mysql.mysql_init local mysql_store_result = mysql.mysql_store_result local mysql_free_result = mysql.mysql_free_result +local mysql_error_number = mysql.mysql_errno local mysql_error_message = mysql.mysql_error local NULL = ffi.cast("MYSQL_result *",0) @@ -187,325 +202,361 @@ local NULL = ffi.cast("MYSQL_result *",0) local ffi_tostring = ffi.string local ffi_gc = ffi.gc ------ mysqldata = ffi.cast("MYSQL_instance*",mysql.malloc(1024*1024)) -local instance = mysql.mysql_init(nil) -- (mysqldata) +local instance = mysql.mysql_init(nil) local mysql_constant_false = false local mysql_constant_true = true -local function finish(t) - local r = t._result_ - if r then - ffi_gc(r,mysql_free_result) +local wrapresult do + + local function collect(t) + local result = t._result_ + if result then + ffi_gc(result,mysql_free_result) + end end -end -local function getcolnames(t) - return t.names -end + local function finish(t) + local result = t._result_ + if result then + t._result_ = nil + ffi_gc(result,mysql_free_result) + end + end -local function getcoltypes(t) - return t.types -end + local function getcoldata(t) + local result = t._result_ + local nofrows = t.nofrows + local noffields = t.noffields + local names = { } + local types = { } + local fields = mysql_fetch_fields(result) + for i=1,noffields do + local field = fields[i-1] + names[i] = ffi_tostring(field.name) + types[i] = tonumber(field.type) -- todo + end + t.names = names + t.types = types + end -local function numrows(t) - return tonumber(t.nofrows) -end + local function getcolnames(t) + local names = t.names + if names then + return names + end + getcoldata(t) + return t.names + end -local function list(t) - local result = t._result_ - if result then - local row = mysql_fetch_row(result) - -- local len = mysql_fetch_lengths(result) - local result = { } - for i=1,t.noffields do - result[i] = ffi_tostring(row[i-1]) + local function getcoltypes(t) + local types = t.types + if types then + return types end - return result + getcoldata(t) + return t.types end -end -local function hash(t) - local result = t._result_ - local fields = t.names - if result then - local row = mysql_fetch_row(result) - -- local len = mysql_fetch_lengths(result) - local result = { } - for i=1,t.noffields do - result[fields[i]] = ffi_tostring(row[i-1]) + local function numrows(t) + return t.nofrows + end + + -- local function fetch(t) + -- local + -- local row = mysql_fetch_row(result) + -- local result = { } + -- for i=1,t.noffields do + -- result[i] = ffi_tostring(row[i-1]) + -- end + -- return unpack(result) + -- end + + local mt = { + __gc = collect, + __index = { + _result_ = nil, + close = finish, + numrows = numrows, + getcolnames = getcolnames, + getcoltypes = getcoltypes, + -- fetch = fetch, -- not efficient + } + } + + wrapresult = function(connection) + local result = mysql_store_result(connection) + if result ~= NULL then + mysql_field_seek(result,0) + local t = { + _result_ = result, + nofrows = tonumber(mysql_num_rows (result) or 0) or 0, + noffields = tonumber(mysql_num_fields(result) or 0) or 0, + } + return setmetatable(t,mt) + elseif tonumber(mysql_field_count(connection) or 0) or 0 > 0 then + return tonumber(mysql_affected_rows(connection)) end - return result end -end -local function wholelist(t) - return fetch_all_rows(t._result_) end -local mt = { __index = { - -- regular - finish = finish, - list = list, - hash = hash, - wholelist = wholelist, - -- compatibility - numrows = numrows, - getcolnames = getcolnames, - getcoltypes = getcoltypes, - -- fallback - _result_ = nil, - names = { }, - types = { }, - noffields = 0, - nofrows = 0, - } -} +local initializesession do -local nt = setmetatable({},mt) + -- timeouts = [ connect_timeout |wait_timeout | interactive_timeout ] --- session + local timeout -- = 3600 -- to be tested -local function close(t) - mysql_close_connection(t._connection_) -end + -- connection -local function execute(t,query) - if query and query ~= "" then - local connection = t._connection_ - local result = mysql_execute_query(connection,query,#query) - if result == 0 then - local result = mysql_store_result(connection) - if result ~= NULL then - mysql_field_seek(result,0) - local nofrows = tonumber(mysql_num_rows(result) or 0) - local noffields = tonumber(mysql_num_fields(result)) - local names = { } - local types = { } - local fields = mysql_fetch_fields(result) - for i=1,noffields do - local field = fields[i-1] - names[i] = ffi_tostring(field.name) - types[i] = tonumber(field.type) -- todo - end - local t = { - _result_ = result, - names = names, - types = types, - noffields = noffields, - nofrows = nofrows, - } - return setmetatable(t,mt) + local function close(t) + -- just a struct ? + end + + local function execute(t,query) + if query and query ~= "" then + local connection = t._connection_ + local result = mysql_execute_query(connection,query,#query) + if result == 0 then + return wrapresult(connection) else - return nt + -- mysql_error_number(connection) + return false, ffi_tostring(mysql_error_message(connection)) end end + return false end - return false -end -local mt = { __index = { - close = close, - execute = execute, + local mt = { + __index = { + close = close, + execute = execute, + } } -} -local function open(t,database,username,password,host,port) - local connection = mysql_open_connection( - t._session_, - host or "localhost", - username or "", - password or "", - database or "", - port or 0, - NULL, - 0 - ) - if connection ~= NULL then - local t = { - _connection_ = connection, - } - return setmetatable(t,mt) + -- session + + local function open(t,database,username,password,host,port) + local connection = mysql_open_connection( + t._session_, + host or "localhost", + username or "", + password or "", + database or "", + port or 0, + NULL, + 0 + ) + if connection ~= NULL then + if timeout then + execute(connection,formatters["SET SESSION connect_timeout=%s ;"](timeout)) + end + local t = { + _connection_ = connection, + } + return setmetatable(t,mt) + end end -end -local function message(t) - return mysql_error_message(t._session_) -end - -local function close(t) - -- dummy, as we have a global session -end + local function message(t) + return mysql_error_message(t._session_) + end -local mt = { - __index = { - connect = open, - close = close, - message = message, - } -} + local function close(t) + local connection = t._connection_ + if connection and connection ~= NULL then + ffi_gc(connection, mysql_close) + t.connection = nil + end + end -local function initialize() - local session = { - _session_ = mysql_initialize(instance) -- maybe share, single thread anyway + local mt = { + __index = { + connect = open, + close = close, + message = message, + }, } - return setmetatable(session,mt) -end --- -- -- -- + initializesession = function() + local session = { + _session_ = mysql_open_session(instance) -- maybe share, single thread anyway + } + return setmetatable(session,mt) + end -local function connect(session,specification) - return session:connect( - specification.database or "", - specification.username or "", - specification.password or "", - specification.host or "", - specification.port - ) end -local function error_in_connection(specification,action) - report_state("error in connection: [%s] user %s into %s at %s:%s", - action or "unknown", - specification.username or "no username", - specification.database or "no database", - specification.host or "no host", - specification.port or "no port" - ) -end +local executequery do -local function datafetched(specification,query,converter) - if not query or query == "" then - report_state("no valid query") - return { }, { } + local function connect(session,specification) + return session:connect( + specification.database or "", + specification.username or "", + specification.password or "", + specification.host or "", + specification.port + ) end - local id = specification.id - local session, connection - if id then - local c = cache[id] - if c then - session = c.session - connection = c.connection + + local function fetched(specification,query,converter) + if not query or query == "" then + report_state("no valid query") + return false end - if not connection then - session = initialize() - connection = connect(session,specification) + local id = specification.id + local session, connection + if id then + local c = cache[id] + if c then + session = c.session + connection = c.connection + end if not connection then - for i=1,nofretries do - sleep(retrydelay) - report_state("retrying to connect: [%s.%s] %s@%s to %s:%s", - id,i, - specification.database or "no database", - specification.username or "no username", - specification.host or "no host", - specification.port or "no port" - ) - connection = connect(session,specification) - if connection then - break - end + session = initializesession() + if not session then + return formatters["no session for %a"](id) end - end - if connection then - cache[id] = { session = session, connection = connection } - end - end - else - session = initialize() - connection = connect(session,specification) - if not connection then - for i=1,nofretries do - sleep(retrydelay) - report_state("retrying to connect: [%s] %s@%s to %s:%s", - i, - specification.database or "no database", - specification.username or "no username", - specification.host or "no host", - specification.port or "no port" - ) connection = connect(session,specification) - if connection then - break + if not connection then + return formatters["no connection for %a"](id) end + cache[id] = { session = session, connection = connection } + end + else + session = initializesession() + if not session then + return "no session" + end + connection = connect(session,specification) + if not connection then + return "no connection" end end - end - if not connection then - report_state("error in connection: %s@%s to %s:%s", + if not connection then + report_state("error in connection: %s@%s to %s:%s", specification.database or "no database", specification.username or "no username", specification.host or "no host", specification.port or "no port" ) - return { }, { } - end - query = lpegmatch(querysplitter,query) - local result, message, okay - for i=1,#query do - local q = query[i] - local r, m = connection:execute(q) - if m then - report_state("error in query, stage: %s",string.collapsespaces(q or "?")) - message = message and format("%s\n%s",message,m) or m + return "no connection" end - if type(r) == "table" then - result = r - okay = true - elseif not m then - okay = true + query = lpegmatch(querysplitter,query) + local result, okay + for i=1,#query do + local q = query[i] + local r, m = connection:execute(q) + if m then + report_state("error in query to host %a: %s",specification.host,string.collapsespaces(q or "?")) + if m then + report_state("message: %s",m) + end + end + local t = type(r) + if t == "table" then + result = r + okay = true + elseif t == "number" then + okay = true + end end - end - local data, keys - if result then - if converter then - data = converter.ffi(result) - else - keys = result.names - data = { } - for i=1,result.nofrows do - data[i] = result:hash() + if not okay then -- can go + -- why do we close a session + if connection then + connection:close() + end + if session then + session:close() end + if id then + cache[id] = nil + end + return "execution error" end - result:finish() -- result:close() - elseif message then - report_state("message %s",message) - end - if not keys then - keys = { } - end - if not data then - data = { } - end - if not id then - connection:close() - session:close() + local data, keys + if result then + if converter then + data = converter.ffi(result) + else + local _result_ = result._result_ + local noffields = result.noffields + local nofrows = result.nofrows + keys = result:getcolnames() + data = { } + if noffields > 0 and nofrows > 0 then + for i=1,nofrows do + local cells = { } + local row = mysql_fetch_row(_result_) + for j=1,noffields do + local s = row[j-1] + local k = keys[j] + if s == NULL then + cells[k] = "" + else + cells[k] = ffi_tostring(s) + end + end + data[i] = cells + end + end + end + result:close() + end + -- + if not id then + if connection then + connection:close() + end + if session then + session:close() + end + end + return false, data, keys end - return data, keys -end -local function execute(specification) - if trace_sql then - report_state("executing library") - end - if not validspecification(specification) then - report_state("error in specification") - return - end - local query = dataprepared(specification) - if not query then - report_state("error in preparation") - return - end - local data, keys = datafetched(specification,query,specification.converter) - if not data then - report_state("error in fetching") - return + local function datafetched(specification,query,converter) + local callokay, connectionerror, data, keys = pcall(fetched,specification,query,converter) + if not callokay then + report_state("call error, retrying") + callokay, connectionerror, data, keys = pcall(fetched,specification,query,converter) + elseif connectionerror then + report_state("error: %s, retrying",connectionerror) + callokay, connectionerror, data, keys = pcall(fetched,specification,query,converter) + end + if not callokay then + report_state("persistent call error") + elseif connectionerror then + report_state("persistent error: %s",connectionerror) + end + return data or { }, keys or { } end - local one = data[1] - if one then - setmetatable(data,{ __index = one } ) + + executequery = function(specification) + if trace_sql then + report_state("executing library") + end + if not validspecification(specification) then + report_state("error in specification") + return + end + local query = dataprepared(specification) + if not query then + report_state("error in preparation") + return + end + local data, keys = datafetched(specification,query,specification.converter) + if not data then + report_state("error in fetching") + return + end + local one = data[1] + if one then + setmetatable(data,{ __index = one } ) + end + return data, keys end - return data, keys + end local wraptemplate = [[ @@ -530,13 +581,14 @@ return function(result) if not result then return { } end - local nofrows = result.nofrows or 0 + local nofrows = result.nofrows if nofrows == 0 then return { } end - local noffields = result.noffields or 0 - local _result_ = result._result_ + local noffields = result.noffields local target = { } -- no %s needed here + local _result_ = result._result_ + -- we can share cells for i=1,nofrows do local cells = { } local row = mysql_fetch_row(_result_) @@ -552,7 +604,7 @@ return function(result) %s } end - result:finish() -- result:close() + result:close() return target end ]] @@ -560,9 +612,9 @@ end local celltemplate = "cells[%s]" methods.ffi = { - runner = function() end, -- never called - execute = execute, - initialize = initialize, -- returns session + runner = function() end, -- never called + execute = executequery, + initialize = initializesession, -- returns session usesfiles = false, wraptemplate = wraptemplate, celltemplate = celltemplate, |