#!/usr/bin/env texlua if not modules then modules = { } end modules ['mtxrun'] = { version = 1.001, comment = "runner, lua replacement for texmfstart.rb", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- one can make a stub: -- -- #!/bin/sh -- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly mtxrun.lua "$@" -- filename : mtxrun.lua -- comment : companion to context.tex -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL -- copyright: PRAGMA ADE / ConTeXt Development Team -- license : see context related readme files -- This script is based on texmfstart.rb but does not use kpsewhich to -- locate files. Although kpse is a library it never came to opening up -- its interface to other programs (esp scripting languages) and so we -- do it ourselves. The lua variant evolved out of an experimental ruby -- one. Interesting is that using a scripting language instead of c does -- not have a speed penalty. Actually the lua variant is more efficient, -- especially when multiple calls to kpsewhich are involved. The lua -- library also gives way more control. -- to be done / considered -- -- support for --exec or make it default -- support for jar files (or maybe not, never used, too messy) -- support for $RUBYINPUTS cum suis (if still needed) -- remember for subruns: _CTX_K_V_#{original}_ -- remember for subruns: _CTX_K_S_#{original}_ -- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] texlua = true -- begin library merge do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-string'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep if not string.split then -- this will be overloaded by a faster lpeg variant function string:split(pattern) if #self > 0 then local t = { } for s in gmatch(self..pattern,"(.-)"..pattern) do t[#t+1] = s end return t else return { } end end end local chr_to_esc = { ["%"] = "%%", ["."] = "%.", ["+"] = "%+", ["-"] = "%-", ["*"] = "%*", ["^"] = "%^", ["$"] = "%$", ["["] = "%[", ["]"] = "%]", ["("] = "%(", [")"] = "%)", ["{"] = "%{", ["}"] = "%}" } string.chr_to_esc = chr_to_esc function string:esc() -- variant 2 return (gsub(self,"(.)",chr_to_esc)) end function string:unquote() return (gsub(self,"^([\"\'])(.*)%1$","%2")) end --~ function string:unquote() --~ if find(self,"^[\'\"]") then --~ return self:sub(2,-2) --~ else --~ return self --~ end --~ end function string:quote() -- we could use format("%q") return '"' .. self:unquote() .. '"' end function string:count(pattern) -- variant 3 local n = 0 for _ in gmatch(self,pattern) do n = n + 1 end return n end function string:limit(n,sentinel) if #self > n then sentinel = sentinel or " ..." return sub(self,1,(n-#sentinel)) .. sentinel else return self end end function string:strip() return (gsub(self,"^%s*(.-)%s*$", "%1")) end function string:is_empty() return not find(self,"%S") end function string:enhance(pattern,action) local ok, n = true, 0 while ok do ok = false self = gsub(self,pattern, function(...) ok, n = true, n + 1 return action(...) end) end return self, n end local chr_to_hex, hex_to_chr = { }, { } for i=0,255 do local c, h = char(i), format("%02X",i) chr_to_hex[c], hex_to_chr[h] = h, c end function string:to_hex() return (gsub(self or "","(.)",chr_to_hex)) end function string:from_hex() return (gsub(self or "","(..)",hex_to_chr)) end if not string.characters then local function nextchar(str, index) index = index + 1 return (index <= #str) and index or nil, str:sub(index,index) end function string:characters() return nextchar, self, 0 end local function nextbyte(str, index) index = index + 1 return (index <= #str) and index or nil, byte(str:sub(index,index)) end function string:bytes() return nextbyte, self, 0 end end -- we can use format for this (neg n) function string:rpadd(n,chr) local m = n-#self if m > 0 then return self .. self.rep(chr or " ",m) else return self end end function string:lpadd(n,chr) local m = n-#self if m > 0 then return self.rep(chr or " ",m) .. self else return self end end string.padd = string.rpadd function is_number(str) -- tonumber return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 end --~ print(is_number("1")) --~ print(is_number("1.1")) --~ print(is_number(".1")) --~ print(is_number("-0.1")) --~ print(is_number("+0.1")) --~ print(is_number("-.1")) --~ print(is_number("+.1")) function string:split_settings() -- no {} handling, see l-aux for lpeg variant if find(self,"=") then local t = { } for k,v in gmatch(self,"(%a+)=([^%,]*)") do t[k] = v end return t else return nil end end local patterns_escapes = { ["-"] = "%-", ["."] = "%.", ["+"] = "%+", ["*"] = "%*", ["%"] = "%%", ["("] = "%)", [")"] = "%)", ["["] = "%[", ["]"] = "%]", } function string:pattesc() return (gsub(self,".",patterns_escapes)) end function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg t[s] = true end return t end local pattern = lpeg.Ct(lpeg.C(1)^0) function string:totable() return pattern:match(self) end --~ for _, str in ipairs { --~ "1234567123456712345671234567", --~ "a\tb\tc", --~ "aa\tbb\tcc", --~ "aaa\tbbb\tccc", --~ "aaaa\tbbbb\tcccc", --~ "aaaaa\tbbbbb\tccccc", --~ "aaaaaa\tbbbbbb\tcccccc", --~ } do print(string.tabtospace(str)) end function string.tabtospace(str,tab) -- we don't handle embedded newlines while true do local s = find(str,"\t") if s then if not tab then tab = 7 end -- only when found local d = tab-(s-1)%tab if d > 0 then str = gsub(str,"\t",rep(" ",d),1) else str = gsub(str,"\t","",1) end else break end end return str end function string:compactlong() -- strips newlines and leading spaces self = gsub(self,"[\n\r]+ *","") self = gsub(self,"^ *","") return self end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-lpeg'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc --~ l-lpeg.lua : --~ lpeg.digit = lpeg.R('09')^1 --~ lpeg.sign = lpeg.S('+-')^1 --~ lpeg.cardinal = lpeg.P(lpeg.sign^0 * lpeg.digit^1) --~ lpeg.integer = lpeg.P(lpeg.sign^0 * lpeg.digit^1) --~ lpeg.float = lpeg.P(lpeg.sign^0 * lpeg.digit^0 * lpeg.P('.') * lpeg.digit^1) --~ lpeg.number = lpeg.float + lpeg.integer --~ lpeg.oct = lpeg.P("0") * lpeg.R('07')^1 --~ lpeg.hex = lpeg.P("0x") * (lpeg.R('09') + lpeg.R('AF'))^1 --~ lpeg.uppercase = lpeg.P("AZ") --~ lpeg.lowercase = lpeg.P("az") --~ lpeg.eol = lpeg.S('\r\n\f')^1 -- includes formfeed --~ lpeg.space = lpeg.S(' ')^1 --~ lpeg.nonspace = lpeg.P(1-lpeg.space)^1 --~ lpeg.whitespace = lpeg.S(' \r\n\f\t')^1 --~ lpeg.nonwhitespace = lpeg.P(1-lpeg.whitespace)^1 local hash = { } function lpeg.anywhere(pattern) --slightly adapted from website return P { P(pattern) + 1 * lpeg.V(1) } end function lpeg.startswith(pattern) --slightly adapted return P(pattern) end function lpeg.splitter(pattern, action) return (((1-P(pattern))^1)/action+1)^0 end -- variant: --~ local parser = lpeg.Ct(lpeg.splitat(newline)) local crlf = P("\r\n") local cr = P("\r") local lf = P("\n") local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) local newline = crlf + cr + lf local spacing = space^0 * newline local empty = spacing * Cc("") local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 local capture = Ct(content^0) function string:splitlines() return capture:match(self) end lpeg.linebyline = content -- better make a sublibrary --~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more --~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more --~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps --~ local p = lpeg.splitat("->",true) print(p:match("oeps")) -- oeps local splitters_s, splitters_m = { }, { } local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] if not splitter then separator = P(separator) if single then local other, any = C((1 - separator)^0), P(1) splitter = other * (separator * C(any^0) + "") splitters_s[separator] = splitter else local other = C((1 - separator)^0) splitter = other * (separator * other)^0 splitters_m[separator] = splitter end end return splitter end lpeg.splitat = splitat local cache = { } function string:split(separator) local c = cache[separator] if not c then c = Ct(splitat(separator)) cache[separator] = c end return c:match(self) end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-table'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump local getmetatable, setmetatable = getmetatable, setmetatable local type, next, tostring, ipairs = type, next, tostring, ipairs function table.strip(tab) local lst = { } for i=1,#tab do local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else lst[#lst+1] = s end end return lst end local function compare(a,b) return (tostring(a) < tostring(b)) end local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed for key,_ in next, tab do srt[#srt+1] = key if kind == 3 then -- no further check else local tkey = type(key) if tkey == "string" then -- if kind == 2 then kind = 3 else kind = 1 end kind = (kind == 2 and 3) or 1 elseif tkey == "number" then -- if kind == 1 then kind = 3 else kind = 2 end kind = (kind == 1 and 3) or 2 else kind = 3 end end end if kind == 0 or kind == 3 then sort(srt,compare) else sort(srt) end return srt end local function sortedhashkeys(tab) -- fast one local srt = { } for key,_ in next, tab do srt[#srt+1] = key end sort(srt) return srt end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys function table.sortedpairs(t) local s = sortedhashkeys(t) -- maybe just sortedkeys local n = 0 local function kv(s) n = n + 1 local k = s[n] return k, t[k] end return kv, s end function table.append(t, list) for _,v in next, list do insert(t,v) end end function table.prepend(t, list) for k,v in next, list do insert(t,k,v) end end function table.merge(t, ...) -- first one is target t = t or {} local lst = {...} for i=1,#lst do for k, v in next, lst[i] do t[k] = v end end return t end function table.merged(...) local tmp, lst = { }, {...} for i=1,#lst do for k, v in next, lst[i] do tmp[k] = v end end return tmp end function table.imerge(t, ...) local lst = {...} for i=1,#lst do local nst = lst[i] for j=1,#nst do t[#t+1] = nst[j] end end return t end function table.imerged(...) local tmp, lst = { }, {...} for i=1,#lst do local nst = lst[i] for j=1,#nst do tmp[#tmp+1] = nst[j] end end return tmp end local function fastcopy(old) -- fast one if old then local new = { } for k,v in next, old do if type(v) == "table" then new[k] = fastcopy(v) -- was just table.copy else new[k] = v end end -- optional second arg local mt = getmetatable(old) if mt then setmetatable(new,mt) end return new else return { } end end local function copy(t, tables) -- taken from lua wiki, slightly adapted tables = tables or { } local tcopy = {} if not tables[t] then tables[t] = tcopy end for i,v in next, t do -- brrr, what happens with sparse indexed if type(i) == "table" then if tables[i] then i = tables[i] else i = copy(i, tables) end end if type(v) ~= "table" then tcopy[i] = v elseif tables[v] then tcopy[i] = tables[v] else tcopy[i] = copy(v, tables) end end local mt = getmetatable(t) if mt then setmetatable(tcopy,mt) end return tcopy end table.fastcopy = fastcopy table.copy = copy -- rougly: copy-loop : unpack : sub == 0.9 : 0.4 : 0.45 (so in critical apps, use unpack) function table.sub(t,i,j) return { unpack(t,i,j) } end function table.replace(a,b) for k,v in next, b do a[k] = v end end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) function table.is_empty(t) return not t or not next(t) end function table.one_entry(t) local n = next(t) return n and not next(t,n) end function table.starts_at(t) return ipairs(t,1)(t,0) end function table.tohash(t,value) local h = { } if t then if value == nil then value = true end for _, v in next, t do -- no ipairs here h[v] = value end end return h end function table.fromhash(t) local h = { } for k, v in next, t do -- no ipairs here if v then h[#h+1] = k end end return h end --~ print(table.serialize(t), "\n") --~ print(table.serialize(t,"name"), "\n") --~ print(table.serialize(t,false), "\n") --~ print(table.serialize(t,true), "\n") --~ print(table.serialize(t,"name",true), "\n") --~ print(table.serialize(t,"name",true,true), "\n") table.serialize_functions = true table.serialize_compact = true table.serialize_inline = true local noquotes, hexify, handle, reduce, compact, inline, functions local reserved = table.tohash { -- intercept a language flaw, no reserved words as key 'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function', 'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', } local function simple_table(t) if #t > 0 then local n = 0 for _,v in next, t do n = n + 1 end if n == #t then local tt = { } for i=1,#t do local v = t[i] local tv = type(v) if tv == "number" then if hexify then tt[#tt+1] = format("0x%04X",v) else tt[#tt+1] = tostring(v) -- tostring not needed end elseif tv == "boolean" then tt[#tt+1] = tostring(v) elseif tv == "string" then tt[#tt+1] = format("%q",v) else tt = nil break end end return tt end end return nil end -- Because this is a core function of mkiv I moved some function calls -- inline. -- -- twice as fast in a test: -- -- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " if indexed then handle(format("%s{",depth)) elseif name then --~ handle(format("%s%s={",depth,key(name))) if type(name) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s[0x%04X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then handle(format("%s%s={",depth,name)) else handle(format("%s[%q]={",depth,name)) end else handle(format("%s{",depth)) end end if root and next(root) then local first, last = nil, 0 -- #root cannot be trusted here if compact then -- NOT: for k=1,#root do (we need to quit at nil) for k,v in ipairs(root) do -- can we use next? if not first then first = k end last = last + 1 end end local sk = sortedkeys(root) for i=1,#sk do local k = sk[i] local v = root[k] --~ if v == root then -- circular --~ else local t = type(v) if compact and first and type(k) == "number" and k >= first and k <= last then if t == "number" then if hexify then handle(format("%s 0x%04X,",depth,v)) else handle(format("%s %s,",depth,v)) end elseif t == "string" then if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then handle(format("%s %s,",depth,v)) else handle(format("%s %q,",depth,v)) end elseif t == "table" then if not next(v) then handle(format("%s {},",depth)) elseif inline then -- and #t > 0 local st = simple_table(v) if st then handle(format("%s { %s },",depth,concat(st,", "))) else do_serialize(v,k,depth,level+1,true) end else do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then handle(format("%s %s,",depth,tostring(v))) elseif t == "function" then if functions then handle(format('%s loadstring(%q),',depth,dump(v))) else handle(format('%s "function",',depth)) end else handle(format("%s %q,",depth,tostring(v))) end elseif k == "__p__" then -- parent if false then handle(format("%s __p__=nil,",depth)) end elseif t == "number" then --~ if hexify then --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) --~ else --~ handle(format("%s %s=%s,",depth,key(k),v)) --~ end if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then if hexify then handle(format("%s %s=0x%04X,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then handle(format("%s [%q]=0x%04X,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end end elseif t == "string" then if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then --~ handle(format("%s %s=%s,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%s,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%s,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end else --~ handle(format("%s %s=%q,",depth,key(k),v)) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%q,",depth,k,v)) else handle(format("%s [%s]=%q,",depth,k,v)) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%q,",depth,k,v)) else handle(format("%s [%q]=%q,",depth,k,v)) end end elseif t == "table" then if not next(v) then --~ handle(format("%s %s={},",depth,key(k))) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then local st = simple_table(v) if st then --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) end else do_serialize(v,k,depth,level+1) end else do_serialize(v,k,depth,level+1) end elseif t == "boolean" then --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) else handle(format("%s [%s]=%s,",depth,k,tostring(v))) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%s,",depth,k,tostring(v))) else handle(format("%s [%q]=%s,",depth,k,tostring(v))) end elseif t == "function" then if functions then --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) else handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) else handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) end end else --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) if type(k) == "number" then -- or find(k,"^%d+$") then if hexify then handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then handle(format("%s %s=%q,",depth,k,tostring(v))) else handle(format("%s [%q]=%q,",depth,k,tostring(v))) end end --~ end end end if level > 0 then handle(format("%s},",depth)) end end -- replacing handle by a direct t[#t+1] = ... (plus test) is not much -- faster (0.03 on 1.00 for zapfino.tma) local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) noquotes = _noquotes hexify = _hexify handle = _handle or print reduce = _reduce or false compact = table.serialize_compact inline = compact and table.serialize_inline functions = table.serialize_functions local tname = type(name) if tname == "string" then if name == "return" then handle("return {") else handle(name .. "={") end elseif tname == "number" then if hexify then handle(format("[0x%04X]={",name)) else handle("[" .. name .. "]={") end elseif tname == "boolean" then if name then handle("return {") else handle("{") end else handle("t={") end if root and next(root) then do_serialize(root,name,"",0,indexed) end handle("}") end --~ name: --~ --~ true : return { } --~ false : { } --~ nil : t = { } --~ string : string = { } --~ 'return' : return { } --~ number : [number] = { } function table.serialize(root,name,reduce,noquotes,hexify) local t = { } local function flush(s) t[#t+1] = s end serialize(root,name,flush,reduce,noquotes,hexify) return concat(t,"\n") end function table.tohandle(handle,root,name,reduce,noquotes,hexify) serialize(root,name,handle,reduce,noquotes,hexify) end -- sometimes tables are real use (zapfino extra pro is some 85M) in which -- case a stepwise serialization is nice; actually, we could consider: -- -- for line in table.serializer(root,name,reduce,noquotes) do -- ...(line) -- end -- -- so this is on the todo list table.tofile_maxtab = 2*1024 function table.tofile(filename,root,name,reduce,noquotes,hexify) local f = io.open(filename,'w') if f then local maxtab = table.tofile_maxtab if maxtab > 1 then local t = { } local function flush(s) t[#t+1] = s if #t > maxtab then f:write(concat(t,"\n"),"\n") -- hm, write(sometable) should be nice t = { } end end serialize(root,name,flush,reduce,noquotes,hexify) f:write(concat(t,"\n"),"\n") else local function flush(s) f:write(s,"\n") end serialize(root,name,flush,reduce,noquotes,hexify) end f:close() end end local function flatten(t,f,complete) for i=1,#t do local v = t[i] if type(v) == "table" then if complete or type(v[1]) == "table" then flatten(v,f,complete) else f[#f+1] = v end else f[#f+1] = v end end end function table.flatten(t) local f = { } flatten(t,f,true) return f end function table.unnest(t) -- bad name local f = { } flatten(t,f,false) return f end table.flatten_one_level = table.unnest -- the next three may disappear function table.remove_value(t,value) -- todo: n if value then for i=1,#t do if t[i] == value then remove(t,i) -- remove all, so no: return end end end end function table.insert_before_value(t,value,str) if str then if value then for i=1,#t do if t[i] == value then insert(t,i,str) return end end end insert(t,1,str) elseif value then insert(t,1,value) end end function table.insert_after_value(t,value,str) if str then if value then for i=1,#t do if t[i] == value then insert(t,i+1,str) return end end end t[#t+1] = str elseif value then t[#t+1] = value end end local function are_equal(a,b,n,m) -- indexed if a and b and #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] if ai==bi then -- same elseif type(ai)=="table" and type(bi)=="table" then if not are_equal(ai,bi) then return false end else return false end end return true else return false end end local function identical(a,b) -- assumes same structure for ka, va in next, a do local vb = b[k] if va == vb then -- same elseif type(va) == "table" and type(vb) == "table" then if not identical(va,vb) then return false end else return false end end return true end table.are_equal = are_equal table.identical = identical -- maybe also make a combined one function table.compact(t) if t then for k,v in next, t do if not next(v) then t[k] = nil end end end end function table.contains(t, v) if t then for i=1, #t do if t[i] == v then return i end end end return false end function table.count(t) local n, e = 0, next(t) while e do n, e = n + 1, next(t,e) end return n end function table.swapped(t) local s = { } for k, v in next, t do s[v] = k end return s end --~ function table.are_equal(a,b) --~ return table.serialize(a) == table.serialize(b) --~ end function table.clone(t,p) -- t is optional or nil or table if not p then t, p = { }, t or { } elseif not t then t = { } end setmetatable(t, { __index = function(_,key) return p[key] end }) return t end function table.hexed(t,seperator) local tt = { } for i=1,#t do tt[i] = format("0x%04X",t[i]) end return concat(tt,seperator or " ") end function table.reverse_hash(h) local r = { } for k,v in next, h do r[v] = lower(gsub(k," ","")) end return r end function table.reverse(t) local tt = { } if #t > 0 then for i=#t,1,-1 do tt[#tt+1] = t[i] end end return tt end --~ function table.keys(t) --~ local k = { } --~ for k,_ in next, t do --~ k[#k+1] = k --~ end --~ return k --~ end --~ function table.keys_as_string(t) --~ local k = { } --~ for k,_ in next, t do --~ k[#k+1] = k --~ end --~ return concat(k,"") --~ end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-io'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local byte = string.byte if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" else io.fileseparator, io.pathseparator = "/" , ":" end function io.loaddata(filename,textmode) local f = io.open(filename,(textmode and 'r') or 'rb') if f then -- collectgarbage("step") -- sometimes makes a big difference in mem consumption local data = f:read('*all') -- garbagecollector.check(data) f:close() return data else return nil end end function io.savedata(filename,data,joiner) local f = io.open(filename,"wb") if f then if type(data) == "table" then f:write(table.join(data,joiner or "")) elseif type(data) == "function" then data(f) else f:write(data) end f:close() return true else return false end end function io.exists(filename) local f = io.open(filename) if f == nil then return false else assert(f:close()) return true end end function io.size(filename) local f = io.open(filename) if f == nil then return 0 else local s = f:seek("end") assert(f:close()) return s end end function io.noflines(f) local n = 0 for _ in f:lines() do n = n + 1 end f:seek('set',0) return n end local nextchar = { [ 4] = function(f) return f:read(1,1,1,1) end, [ 2] = function(f) return f:read(1,1) end, [ 1] = function(f) return f:read(1) end, [-2] = function(f) local a, b = f:read(1,1) return b, a end, [-4] = function(f) local a, b, c, d = f:read(1,1,1,1) return d, c, b, a end } function io.characters(f,n) if f then return nextchar[n or 1], f else return nil, nil end end local nextbyte = { [4] = function(f) local a, b, c, d = f:read(1,1,1,1) if d then return byte(a), byte(b), byte(c), byte(d) else return nil, nil, nil, nil end end, [2] = function(f) local a, b = f:read(1,1) if b then return byte(a), byte(b) else return nil, nil end end, [1] = function (f) local a = f:read(1) if a then return byte(a) else return nil end end, [-2] = function (f) local a, b = f:read(1,1) if b then return byte(b), byte(a) else return nil, nil end end, [-4] = function(f) local a, b, c, d = f:read(1,1,1,1) if d then return byte(d), byte(c), byte(b), byte(a) else return nil, nil, nil, nil end end } function io.bytes(f,n) if f then return nextbyte[n or 1], f else return nil, nil end end function io.ask(question,default,options) while true do io.write(question) if options then io.write(string.format(" [%s]",table.concat(options,"|"))) end if default then io.write(string.format(" [%s]",default)) end io.write(string.format(" ")) local answer = io.read() answer = answer:gsub("^%s*(.*)%s*$","%1") if answer == "" and default then return default elseif not options then return answer else for _,v in pairs(options) do if v == answer then return answer end end local pattern = "^" .. answer for _,v in pairs(options) do if v:find(pattern) then return v end end end end end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-number'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local format = string.format number = number or { } -- a,b,c,d,e,f = number.toset(100101) function number.toset(n) return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end function number.toevenhex(n) local s = format("%X",n) if #s % 2 == 0 then return s else return "0" .. s end end -- the lpeg way is slower on 8 digits, but faster on 4 digits, some 7.5% -- on -- -- for i=1,1000000 do -- local a,b,c,d,e,f,g,h = number.toset(12345678) -- local a,b,c,d = number.toset(1234) -- local a,b,c = number.toset(123) -- end -- -- of course dedicated "(.)(.)(.)(.)" matches are even faster local one = lpeg.C(1-lpeg.S(''))^1 function number.toset(n) return one:match(tostring(n)) end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-set'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } set = set or { } local nums = { } local tabs = { } local concat = table.concat local next, type = next, type set.create = table.tohash function set.tonumber(t) if next(t) then local s = "" -- we could save mem by sorting, but it slows down for k, v in next, t do if v then -- why bother about the leading space s = s .. " " .. k end end local n = nums[s] if not n then n = #tabs + 1 tabs[n] = t nums[s] = n end return n else return 0 end end function set.totable(n) if n == 0 then return { } else return tabs[n] or { } end end function set.tolist(n) if n == 0 or not tabs[n] then return "" else local t = { } for k, v in next, tabs[n] do if v then t[#t+1] = k end end return concat(t," ") end end function set.contains(n,s) if type(n) == "table" then return n[s] elseif n == 0 then return false else local t = tabs[n] return t and t[s] end end --~ local c = set.create{'aap','noot','mies'} --~ local s = set.tonumber(c) --~ local t = set.totable(s) --~ print(t['aap']) --~ local c = set.create{'zus','wim','jet'} --~ local s = set.tonumber(c) --~ local t = set.totable(s) --~ print(t['aap']) --~ print(t['jet']) --~ print(set.contains(t,'jet')) --~ print(set.contains(t,'aap')) end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-os'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local find = string.find function os.resultof(command) return io.popen(command,"r"):read("*all") end if not os.exec then os.exec = os.execute end if not os.spawn then os.spawn = os.execute end --~ os.type : windows | unix (new, we already guessed os.platform) --~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) if not io.fileseparator then if find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" else io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" end end os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" function os.launch(str) if os.platform == "windows" then os.execute("start " .. str) -- os.spawn ? else os.execute(str .. " &") -- os.spawn ? end end if not os.setenv then function os.setenv() return false end end if not os.times then -- utime = user time -- stime = system time -- cutime = children user time -- cstime = children system time function os.times() return { utime = os.gettimeofday(), -- user stime = 0, -- system cutime = 0, -- children user cstime = 0, -- children system } end end os.gettimeofday = os.gettimeofday or os.clock local startuptime = os.gettimeofday() function os.runtime() return os.gettimeofday() - startuptime end --~ print(os.gettimeofday()-os.time()) --~ os.sleep(1.234) --~ print (">>",os.runtime()) --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) os.arch = os.arch or function() local a = os.resultof("uname -m") or "linux" os.arch = function() return a end return a end local platform function os.currentplatform(name,default) if not platform then local name = os.name or os.platform or name -- os.name is built in, os.platform is mine if not name then platform = default or "linux" elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then platform = "mswin-64" else platform = "mswin" end else local architecture = os.arch() if name == "linux" then if find(architecture,"x86_64") then platform = "linux-64" elseif find(architecture,"ppc") then platform = "linux-ppc" else platform = "linux" end elseif name == "macosx" then if find(architecture,"i386") then platform = "osx-intel" else platform = "osx-ppc" end elseif name == "sunos" then if find(architecture,"sparc") then platform = "solaris-sparc" else -- if architecture == 'i86pc' platform = "solaris-intel" end elseif name == "freebsd" then if find(architecture,"amd64") then platform = "freebsd-amd64" else platform = "freebsd" end else platform = default or name end end function os.currentplatform() return platform end end return platform end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-file'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- needs a cleanup file = file or { } local concat = table.concat local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub function file.removesuffix(filename) return (gsub(filename,"%.[%a%d]+$","")) end function file.addsuffix(filename, suffix) if not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename end end function file.replacesuffix(filename, suffix) return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end function file.dirname(name,default) return match(name,"^(.+)[/\\].-$") or (default or "") end function file.basename(name) return match(name,"^.+[/\\](.-)$") or name end function file.nameonly(name) return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end function file.extname(name) return match(name,"^.+%.([^/\\]-)$") or "" end file.suffix = file.extname --~ print(file.join("x/","/y")) --~ print(file.join("http://","/y")) --~ print(file.join("http://a","/y")) --~ print(file.join("http:///a","/y")) --~ print(file.join("//nas-1","/y")) function file.join(...) local pth = concat({...},"/") pth = gsub(pth,"\\","/") local a, b = match(pth,"^(.*://)(.*)$") if a and b then return a .. gsub(b,"//+","/") end a, b = match(pth,"^(//)(.*)$") if a and b then return a .. gsub(b,"//+","/") end return (gsub(pth,"//+","/")) end function file.iswritable(name) local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) return a and a.permissions:sub(2,2) == "w" end function file.isreadable(name) local a = lfs.attributes(name) return a and a.permissions:sub(1,1) == "r" end file.is_readable = file.isreadable file.is_writable = file.iswritable -- todo: lpeg function file.split_path(str) local t = { } str = gsub(str,"\\", "/") str = gsub(str,"(%a):([;/])", "%1\001%2") for name in gmatch(str,"([^;:]+)") do if name ~= "" then t[#t+1] = gsub(name,"\001",":") end end return t end function file.join_path(tab) return concat(tab,io.pathseparator) -- can have trailing // end function file.collapse_path(str) str = gsub(str,"/%./","/") local n, m = 1, 1 while n > 0 or m > 0 do str, n = gsub(str,"[^/%.]+/%.%.$","") str, m = gsub(str,"[^/%.]+/%.%./","") end str = gsub(str,"([^/])/$","%1") str = gsub(str,"^%./","") str = gsub(str,"/%.$","") if str == "" then str = "." end return str end --~ print(file.collapse_path("a/./b/..")) --~ print(file.collapse_path("a/aa/../b/bb")) --~ print(file.collapse_path("a/../..")) --~ print(file.collapse_path("a/.././././b/..")) --~ print(file.collapse_path("a/./././b/..")) --~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) return (gsub(str,"[^%a%d%/%-%.\\]+","-")) end file.readdata = io.loaddata file.savedata = io.savedata function file.copy(oldname,newname) file.savedata(newname,io.loaddata(oldname)) end -- lpeg variants, slightly faster, not always --~ local period = lpeg.P(".") --~ local slashes = lpeg.S("\\/") --~ local noperiod = 1-period --~ local noslashes = 1-slashes --~ local name = noperiod^1 --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.C(noperiod^1) * -1 --~ function file.extname(name) --~ return pattern:match(name) or "" --~ end --~ local pattern = lpeg.Cs(((period * noperiod^1 * -1)/"" + 1)^1) --~ function file.removesuffix(name) --~ return pattern:match(name) --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) --~ return pattern:match(name) or name --~ end --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.Cp() * noslashes^1 * -1 --~ function file.dirname(name) --~ local p = pattern:match(name) --~ if p then --~ return name:sub(1,p-2) --~ else --~ return "" --~ end --~ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.addsuffix(name, suffix) --~ local p = pattern:match(name) --~ if p then --~ return name --~ else --~ return name .. "." .. suffix --~ end --~ end --~ local pattern = (noslashes^0 * slashes)^0 * (noperiod^1 * period)^1 * lpeg.Cp() * noperiod^1 * -1 --~ function file.replacesuffix(name,suffix) --~ local p = pattern:match(name) --~ if p then --~ return name:sub(1,p-2) .. "." .. suffix --~ else --~ return name .. "." .. suffix --~ end --~ end --~ local pattern = (noslashes^0 * slashes)^0 * lpeg.Cp() * ((noperiod^1 * period)^1 * lpeg.Cp() + lpeg.P(true)) * noperiod^1 * -1 --~ function file.nameonly(name) --~ local a, b = pattern:match(name) --~ if b then --~ return name:sub(a,b-2) --~ elseif a then --~ return name:sub(a) --~ else --~ return name --~ end --~ end --~ local test = file.extname --~ local test = file.basename --~ local test = file.dirname --~ local test = file.addsuffix --~ local test = file.replacesuffix --~ local test = file.nameonly --~ print(1,test("./a/b/c/abd.def.xxx","!!!")) --~ print(2,test("./../b/c/abd.def.xxx","!!!")) --~ print(3,test("a/b/c/abd.def.xxx","!!!")) --~ print(4,test("a/b/c/def.xxx","!!!")) --~ print(5,test("a/b/c/def","!!!")) --~ print(6,test("def","!!!")) --~ print(7,test("def.xxx","!!!")) --~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) -- also rewrite previous local letter = lpeg.R("az","AZ") + lpeg.S("_-+") local separator = lpeg.P("://") local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) return qualified:match(filename) end function file.is_rootbased_path(filename) return rootbased:match(filename) end local slash = lpeg.S("\\/") local period = lpeg.P(".") local drive = lpeg.C(lpeg.R("az","AZ")) * lpeg.P(":") local path = lpeg.C(((1-slash)^0 * slash)^0) local suffix = period * lpeg.C(lpeg.P(1-period)^0 * lpeg.P(-1)) local base = lpeg.C((1-suffix)^0) local pattern = (drive + lpeg.Cc("")) * (path + lpeg.Cc("")) * (base + lpeg.Cc("")) * (suffix + lpeg.Cc("")) function file.splitname(str) -- returns drive, path, base, suffix return pattern:match(str) end -- function test(t) for k, v in pairs(t) do print(v, "=>", file.splitname(v)) end end -- -- test { "c:", "c:/aa", "c:/aa/bb", "c:/aa/bb/cc", "c:/aa/bb/cc.dd", "c:/aa/bb/cc.dd.ee" } -- test { "c:", "c:aa", "c:aa/bb", "c:aa/bb/cc", "c:aa/bb/cc.dd", "c:aa/bb/cc.dd.ee" } -- test { "/aa", "/aa/bb", "/aa/bb/cc", "/aa/bb/cc.dd", "/aa/bb/cc.dd.ee" } -- test { "aa", "aa/bb", "aa/bb/cc", "aa/bb/cc.dd", "aa/bb/cc.dd.ee" } end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-md5'] = { version = 1.001, author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- This also provides file checksums and checkers. local gsub, format, byte = string.gsub, string.format, string.byte local function convert(str,fmt) return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) end if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end --~ if not md5.HEX then --~ local function remap(chr) return format("%02X",byte(chr)) end --~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end --~ end --~ if not md5.hex then --~ local function remap(chr) return format("%02x",byte(chr)) end --~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end --~ end --~ if not md5.dec then --~ local function remap(chr) return format("%03i",byte(chr)) end --~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end --~ end file.needs_updating_threshold = 1 function file.needs_updating(oldname,newname) -- size modification access change local oldtime = lfs.attributes(oldname, modification) local newtime = lfs.attributes(newname, modification) if newtime >= oldtime then return false elseif oldtime - newtime < file.needs_updating_threshold then return false else return true end end function file.checksum(name) if md5 then local data = io.loaddata(name) if data then return md5.HEX(data) end end return nil end function file.loadchecksum(name) if md5 then local data = io.loaddata(name .. ".md5") return data and data:gsub("%s","") end return nil end function file.savechecksum(name, checksum) if not checksum then checksum = file.checksum(name) end if checksum then io.savedata(name .. ".md5",checksum) return checksum end return nil end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-dir'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local type = type local find, gmatch = string.find, string.gmatch dir = dir or { } -- optimizing for no string.find (*) does not save time local attributes = lfs.attributes local walkdir = lfs.dir local function glob_pattern(path,patt,recurse,action) local ok, scanner if path == "/" then ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe else ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe end if ok and type(scanner) == "function" then if not find(path,"/$") then path = path .. '/' end for name in scanner do local full = path .. name local mode = attributes(full,'mode') if mode == 'file' then if find(full,patt) then action(full) end elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then glob_pattern(full,patt,recurse,action) end end end end dir.glob_pattern = glob_pattern local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V local pattern = Ct { [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), [2] = C(((1-S("*?/"))^0 * P("/"))^0), [3] = C(P(1)^0) } local filter = Cs ( ( P("**") / ".*" + P("*") / "[^/]*" + P("?") / "[^/]" + P(".") / "%%." + P("+") / "%%+" + P("-") / "%%-" + P(1) )^0 ) local function glob(str,t) if type(str) == "table" then local t = t or { } for s=1,#str do glob(str[s],t) end return t elseif lfs.isfile(str) then local t = t or { } t[#t+1] = str return t else local split = pattern:match(str) if split then local t = t or { } local action = action or function(name) t[#t+1] = name end local root, path, base = split[1], split[2], split[3] local recurse = find(base,"%*%*") local start = root .. path local result = filter:match(start .. base) glob_pattern(start,result,recurse,action) return t else return { } end end end dir.glob = glob --~ list = dir.glob("**/*.tif") --~ list = dir.glob("/**/*.tif") --~ list = dir.glob("./**/*.tif") --~ list = dir.glob("oeps/**/*.tif") --~ list = dir.glob("/oeps/**/*.tif") local function globfiles(path,recurse,func,files) -- func == pattern or function if type(func) == "string" then local s = func -- alas, we need this indirect way func = function(name) return find(name,s) end end files = files or { } for name in walkdir(path) do if find(name,"^%.") then --- skip else local mode = attributes(name,'mode') if mode == "directory" then if recurse then globfiles(path .. "/" .. name,recurse,func,files) end elseif mode == "file" then if func then if func(name) then files[#files+1] = path .. "/" .. name end else files[#files+1] = path .. "/" .. name end end end end return files end dir.globfiles = globfiles -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") -- t = dir.glob("f:/minimal/tex/**/*") -- print(dir.ls("f:/minimal/tex/**/*")) -- print(dir.ls("*.tex")) function dir.ls(pattern) return table.concat(glob(pattern),"\n") end --~ mkdirs("temp") --~ mkdirs("a/b/c") --~ mkdirs(".","/a/b/c") --~ mkdirs("a","b","c") local make_indeed = true -- false if string.find(os.getenv("PATH"),";") then function dir.mkdirs(...) local str, pth = "", "" for _, s in ipairs({...}) do if s ~= "" then if str ~= "" then str = str .. "/" .. s else str = s end end end local first, middle, last local drive = false first, middle, last = str:match("^(//)(//*)(.*)$") if first then -- empty network path == local path else first, last = str:match("^(//)/*(.-)$") if first then middle, last = str:match("([^/]+)/+(.-)$") if middle then pth = "//" .. middle else pth = "//" .. last last = "" end else first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") if first then pth, drive = first .. middle, true else middle, last = str:match("^(/*)(.-)$") if not middle then last = str end end end end for s in gmatch(last,"[^/]+") do if pth == "" then pth = s elseif drive then pth, drive = pth .. s, false else pth = pth .. "/" .. s end if make_indeed and not lfs.isdir(pth) then lfs.mkdir(pth) end end return pth, (lfs.isdir(pth) == true) end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) --~ print(dir.mkdirs("a:")) --~ print(dir.mkdirs("a:/b/c")) --~ print(dir.mkdirs("a:b/c")) --~ print(dir.mkdirs("a:/bbb/c")) --~ print(dir.mkdirs("/a/b/c")) --~ print(dir.mkdirs("/aaa/b/c")) --~ print(dir.mkdirs("//a/b/c")) --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) function dir.expand_name(str) local first, nothing, last = str:match("^(//)(//*)(.*)$") if first then first = lfs.currentdir() .. "/" first = first:gsub("\\","/") end if not first then first, last = str:match("^(//)/*(.*)$") end if not first then first, last = str:match("^([a-zA-Z]:)(.*)$") if first and not find(last,"^/") then local d = lfs.currentdir() if lfs.chdir(first) then first = lfs.currentdir() first = first:gsub("\\","/") end lfs.chdir(d) end end if not first then first, last = lfs.currentdir(), str first = first:gsub("\\","/") end last = last:gsub("//","/") last = last:gsub("/%./","/") last = last:gsub("^/*","") first = first:gsub("/*$","") if last == "" then return first else return first .. "/" .. last end end else function dir.mkdirs(...) local str, pth = "", "" for _, s in ipairs({...}) do if s ~= "" then if str ~= "" then str = str .. "/" .. s else str = s end end end str = str:gsub("/+","/") if find(str,"^/") then pth = "/" for s in gmatch(str,"[^/]+") do local first = (pth == "/") if first then pth = pth .. s else pth = pth .. "/" .. s end if make_indeed and not first and not lfs.isdir(pth) then lfs.mkdir(pth) end end else pth = "." for s in gmatch(str,"[^/]+") do pth = pth .. "/" .. s if make_indeed and not lfs.isdir(pth) then lfs.mkdir(pth) end end end return pth, (lfs.isdir(pth) == true) end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) --~ print(dir.mkdirs("/a/b/c")) --~ print(dir.mkdirs("/aaa/b/c")) --~ print(dir.mkdirs("//a/b/c")) --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) function dir.expand_name(str) if not find(str,"^/") then str = lfs.currentdir() .. "/" .. str end str = str:gsub("//","/") str = str:gsub("/%./","/") return str end end dir.makedirs = dir.mkdirs end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-boolean'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } boolean = boolean or { } local type, tonumber = type, tonumber function boolean.tonumber(b) if b then return 1 else return 0 end end function toboolean(str,tolerant) if tolerant then local tstr = type(str) if tstr == "string" then return str == "true" or str == "yes" or str == "on" or str == "1" or str == "t" elseif tstr == "number" then return tonumber(str) ~= 0 elseif tstr == "nil" then return false else return str end elseif str == "true" then return true elseif str == "false" then return false else return str end end function string.is_boolean(str) if type(str) == "string" then if str == "true" or str == "yes" or str == "on" or str == "t" then return true elseif str == "false" or str == "no" or str == "off" or str == "f" then return false end end return nil end function boolean.alwaystrue() return true end function boolean.falsetrue() return false end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-math'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan if not math.round then function math.round(x) return floor(x + 0.5) end end if not math.div then function math.div(n,m) return floor(n/m) end end if not math.mod then function math.mod(n,m) return n % m end end local pipi = 2*math.pi/360 function math.sind(d) return sin(d*pipi) end function math.cosd(d) return cos(d*pipi) end function math.tand(d) return tan(d*pipi) end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- hm, quite unreadable if not utils then utils = { } end if not utils.merger then utils.merger = { } end if not utils.lua then utils.lua = { } end utils.merger.m_begin = "begin library merge" utils.merger.m_end = "end library merge" utils.merger.pattern = "%c+" .. "%-%-%s+" .. utils.merger.m_begin .. "%c+(.-)%c+" .. "%-%-%s+" .. utils.merger.m_end .. "%c+" function utils.merger._self_fake_() return "-- " .. "created merged file" .. "\n\n" .. "-- " .. utils.merger.m_begin .. "\n\n" .. "-- " .. utils.merger.m_end .. "\n\n" end function utils.report(...) print(...) end utils.merger.strip_comment = true function utils.merger._self_load_(name) local f, data = io.open(name), "" if f then utils.report("reading merge from %s",name) data = f:read("*all") f:close() else utils.report("unknown file to merge %s",name) end if data and utils.merger.strip_comment then -- saves some 20K data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") end return data or "" end function utils.merger._self_save_(name, data) if data ~= "" then local f = io.open(name,'w') if f then utils.report("saving merge from %s",name) f:write(data) f:close() end end end function utils.merger._self_swap_(data,code) if data ~= "" then return (data:gsub(utils.merger.pattern, function(s) return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" end, 1)) else return "" end end --~ stripper: --~ --~ data = string.gsub(data,"%-%-~[^\n]*\n","") --~ data = string.gsub(data,"\n\n+","\n") function utils.merger._self_libs_(libs,list) local result, f, frozen = { }, nil, false result[#result+1] = "\n" if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end local foundpath = nil for _, lib in ipairs(libs) do for _, pth in ipairs(list) do pth = string.gsub(pth,"\\","/") -- file.clean_path utils.report("checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then foundpath = pth end end if foundpath then break end end if foundpath then utils.report("using library path %s",foundpath) local right, wrong = { }, { } for _, lib in ipairs(libs) do local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then -- right[#right+1] = lib utils.report("merging library %s",fullname) result[#result+1] = "do -- create closure to overcome 200 locals limit" result[#result+1] = io.loaddata(fullname,true) result[#result+1] = "end -- of closure" else -- wrong[#wrong+1] = lib utils.report("no library %s",fullname) end end if #right > 0 then utils.report("merged libraries: %s",table.concat(right," ")) end if #wrong > 0 then utils.report("skipped libraries: %s",table.concat(wrong," ")) end else utils.report("no valid library path found") end return table.concat(result, "\n\n") end function utils.merger.selfcreate(libs,list,target) if target then utils.merger._self_save_( target, utils.merger._self_swap_( utils.merger._self_fake_(), utils.merger._self_libs_(libs,list) ) ) end end function utils.merger.selfmerge(name,libs,list,target) utils.merger._self_save_( target or name, utils.merger._self_swap_( utils.merger._self_load_(name), utils.merger._self_libs_(libs,list) ) ) end function utils.merger.selfclean(name) utils.merger._self_save_( name, utils.merger._self_swap_( utils.merger._self_load_(name), "" ) ) end function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true -- utils.report("compiling",luafile,"into",lucfile) os.remove(lucfile) local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) if strip ~= false then command = "-s " .. command end local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then -- utils.report("removing",luafile) os.remove(luafile) end return done end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['l-aux'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } aux = aux or { } local concat, format, gmatch = table.concat, string.format, string.gmatch local tostring, type = tostring, type local space = lpeg.P(' ') local equal = lpeg.P("=") local comma = lpeg.P(",") local lbrace = lpeg.P("{") local rbrace = lpeg.P("}") local nobrace = 1 - (lbrace+rbrace) local nested = lpeg.P{ lbrace * (nobrace + lpeg.V(1))^0 * rbrace } local spaces = space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) local key = lpeg.C((1-equal-comma)^1) local pattern_a = (space+comma)^0 * (key * equal * value + key * lpeg.C("")) local pattern_c = (space+comma)^0 * (key * equal * value) local key = lpeg.C((1-space-equal-comma)^1) local pattern_b = spaces * comma^0 * spaces * (key * ((spaces * equal * spaces * value) + lpeg.C(""))) -- "a=1, b=2, c=3, d={a{b,c}d}, e=12345, f=xx{a{b,c}d}xx, g={}" : outer {} removes, leading spaces ignored local hash = { } local function set(key,value) -- using Carg is slower here hash[key] = value end local pattern_a_s = (pattern_a/set)^1 local pattern_b_s = (pattern_b/set)^1 local pattern_c_s = (pattern_c/set)^1 aux.settings_to_hash_pattern_a = pattern_a_s aux.settings_to_hash_pattern_b = pattern_b_s aux.settings_to_hash_pattern_c = pattern_c_s function aux.make_settings_to_hash_pattern(set,how) if how == "strict" then return (pattern_c/set)^1 elseif how == "tolerant" then return (pattern_b/set)^1 else return (pattern_a/set)^1 end end function aux.settings_to_hash(str) if str and str ~= "" then hash = { } if moretolerant then pattern_b_s:match(str) else pattern_a_s:match(str) end return hash else return { } end end function aux.settings_to_hash_tolerant(str) if str and str ~= "" then hash = { } pattern_b_s:match(str) return hash else return { } end end function aux.settings_to_hash_strict(str) if str and str ~= "" then hash = { } pattern_c_s:match(str) return next(hash) and hash else return nil end end local seperator = comma * space^0 local value = lpeg.P(lbrace * lpeg.C((nobrace + nested)^0) * rbrace) + lpeg.C((nested + (1-comma))^0) local pattern = lpeg.Ct(value*(seperator*value)^0) -- "aap, {noot}, mies" : outer {} removes, leading spaces ignored aux.settings_to_array_pattern = pattern function aux.settings_to_array(str) if not str or str == "" then return { } else return pattern:match(str) end end local function set(t,v) t[#t+1] = v end local value = lpeg.P(lpeg.Carg(1)*value) / set local pattern = value*(seperator*value)^0 * lpeg.Carg(1) function aux.add_settings_to_array(t,str) return pattern:match(str, nil, t) end function aux.hash_to_string(h,separator,yes,no,strict,omit) if h then local t, s = { }, table.sortedkeys(h) omit = omit and table.tohash(omit) for i=1,#s do local key = s[i] if not omit or not omit[key] then local value = h[key] if type(value) == "boolean" then if yes and no then if value then t[#t+1] = key .. '=' .. yes elseif not strict then t[#t+1] = key .. '=' .. no end elseif value or not strict then t[#t+1] = key .. '=' .. tostring(value) end else t[#t+1] = key .. '=' .. value end end end return concat(t,separator or ",") else return "" end end function aux.array_to_string(a,separator) if a then return concat(a,separator or ",") else return "" end end function aux.settings_to_set(str,t) t = t or { } for s in gmatch(str,"%s*([^,]+)") do t[s] = true end return t end -- temporary here function aux.getparameters(self,class,parentclass,settings) local sc = self[class] if not sc then sc = table.clone(self[parent]) self[class] = sc end aux.add_settings_to_array(sc, settings) end -- temporary here local digit = lpeg.R("09") local period = lpeg.P(".") local zero = lpeg.P("0") --~ local finish = lpeg.P(-1) --~ local nodigit = (1-digit) + finish --~ local case_1 = (period * zero^1 * #nodigit)/"" -- .000 --~ local case_2 = (period * (1-(zero^0/"") * #nodigit)^1 * (zero^0/"") * nodigit) -- .010 .10 .100100 local trailingzeros = zero^0 * -digit -- suggested by Roberto R local case_1 = period * trailingzeros / "" local case_2 = period * (digit - trailingzeros)^1 * (trailingzeros / "") local number = digit^1 * (case_1 + case_2) local stripper = lpeg.Cs((number + 1)^0) --~ local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" --~ collectgarbage("collect") --~ str = string.rep(sample,10000) --~ local ts = os.clock() --~ stripper:match(str) --~ print(#str, os.clock()-ts, stripper:match(sample)) function aux.strip_zeros(str) return stripper:match(str) end function aux.definetable(target) -- defines undefined tables local composed, t = nil, { } for name in gmatch(target,"([^%.]+)") do if composed then composed = composed .. "." .. name else composed = name end t[#t+1] = format("%s = %s or { }",composed,composed) end return concat(t,"\n") end function aux.accesstable(target) local t = _G for name in gmatch(target,"([^%.]+)") do t = t[name] end return t end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['lxml-tab'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --[[ldx--
The parser used here is inspired by the variant discussed in the lua book, but
handles comment and processing instructions, has a different structure, provides
parent access; a first version used different trickery but was less optimized to we
went this route. First we had a find based parser, now we have an
Expecially the lpath code is experimental, we will support some of xpath, but
only things that make sense for us; as compensation it is possible to hook in your
own functions. Apart from preprocessing content for
Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.
--ldx]]-- xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert local type, next, setmetatable = type, next, setmetatable local format, lower, find = string.format, string.lower, string.find --[[ldx--This module can be used stand alone but also inside
First a hack to enable namespace resolving. A namespace is characterized by
a
The next function associates a namespace prefix with an
The next function also registers a namespace, but this time we map a
given namespace prefix onto a registered one, using the given
Next we provide a way to turn an
A namespace in an element can be remapped onto the registered
one efficiently by using the
This version uses
Next comes the parser. The rather messy doctype definition comes in many
disguises so it is no surprice that later on have to dedicate quite some
The code may look a bit complex but this is mostly due to the fact that we resolve namespaces and attach metatables. There is only one public function:
An optional second boolean argument tells this function not to create a root element.
Valid entities are:
We cannot load an
When we inject new elements, we need to convert strings to valid trees, which is what the next function does.
--ldx]]-- function xml.toxml(data) if type(data) == "string" then local root = { xml.convert(data,true) } return (#root > 1 and root) or root[1] else return data end end --[[ldx--For copying a tree we use a dedicated function instead of the generic table copier. Since we know what we're dealing with we can speed up things a bit. The second argument is not to be used!
--ldx]]-- local function copy(old,tables) if old then tables = tables or { } local new = { } if not tables[old] then tables[old] = new end for k,v in pairs(old) do new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v end local mt = getmetatable(old) if mt then setmetatable(new,mt) end return new else return { } end end xml.copy = copy --[[ldx--In
At the cost of some 25% runtime overhead you can first convert the tree to a string and then handle the lot.
--ldx]]-- function xml.tostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root elseif next(root) then -- next is faster than type (and >0 test) local result = { } serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) return concat(result,"") end end return "" end --[[ldx--The next function operated on the content only and needs a handle function that accepts a string.
--ldx]]-- function xml.string(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do xml.string(edt[i],handle) end end else handle(e) end end --[[ldx--How you deal with saving data depends on your preferences. For a 40 MB database file the timing on a 2.3 Core Duo are as follows (time in seconds):
The save function is given below.
--ldx]]-- function xml.save(root,name) local f = io.open(name,"w") if f then xml.serialize(root,function(s) f:write(s) end) f:close() end end --[[ldx--A few helpers:
--ldx]]-- function xml.body(root) return (root.ri and root.dt[root.ri]) or root end function xml.text(root) return (root and xml.tostring(root)) or "" end function xml.content(root) -- bugged return (root and root.dt and xml.tostring(root.dt)) or "" end function xml.isempty(root, pattern) if pattern == "" or pattern == "*" then pattern = nil end if pattern then -- todo return false else return not root or not root.dt or #root.dt == 0 or root.dt == "" end end --[[ldx--The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:
The next helper assigns a tree (or string). Usage:
This module can be used stand alone but also inside
We've now arrived at an intersting part: accessing the tree using a subset
of
An
Access to the root and data table makes it possible to construct insert and delete functions.
--ldx]]-- local functions = xml.functions local expressions = xml.expressions expressions.contains = string.find expressions.find = string.find expressions.upper = string.upper expressions.lower = string.lower expressions.number = tonumber expressions.boolean = toboolean expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end expressions.error = function(str) xml.error_handler("unknown function in lpath expression",str or "?") return false end functions.text = function(root,k,n) -- unchecked, maybe one deeper local t = type(t) if t == "string" then return t else -- todo n local rdt = root.dt return (rdt and rdt[k]) or root[k] or "" end end functions.name = function(d,k,n) -- ns + tg local found = false n = n or 0 if not k then -- not found elseif n == 0 then local dk = d[k] found = dk and (type(dk) == "table") and dk elseif n < 0 then for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then if n == -1 then found = di break else n = n + 1 end end end else for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then if n == 1 then found = di break else n = n - 1 end end end end if found then local ns, tg = found.rn or found.ns or "", found.tg if ns ~= "" then return ns .. ":" .. tg else return tg end else return "" end end functions.tag = function(d,k,n) -- only tg local found = false n = n or 0 if not k then -- not found elseif n == 0 then local dk = d[k] found = dk and (type(dk) == "table") and dk elseif n < 0 then for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then if n == -1 then found = di break else n = n + 1 end end end else for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then if n == 1 then found = di break else n = n - 1 end end end end return (found and found.tg) or "" end expressions.text = functions.text expressions.name = functions.name expressions.tag = functions.tag local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces if not root then -- error return false elseif pattern == false then -- root handle(root,root.dt,root.ri) return false elseif pattern == true then -- wildcard local rootdt = root.dt if rootdt then local start, stop, step = 1, #rootdt, 1 if reverse then start, stop, step = stop, start, -1 end for k=start,stop,step do if handle(root,rootdt,root.ri or k) then return false end if not traverse(rootdt[k],true,handle,reverse) then return false end end end return false elseif root.dt then index = index or 1 local action = pattern[index] local command = action[1] if command == 29 then -- fast case /oeps local rootdt = root.dt for k=1,#rootdt do local e = rootdt[k] local tg = e.tg if e.tg then local ns = e.rn or e.ns local ns_a, tg_a = action[3], action[4] local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) if not action[2] then matched = not matched end if matched then if handle(root,rootdt,k) then return false end end end end elseif command == 11 then -- parent local ep = root.__p__ or parent if index < #pattern then if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end elseif handle(root,rootdt,k) then return false end else if (command == 16 or command == 12) and index == 1 then -- initial -- wildcard = true wildcard = command == 16 -- ok? index = index + 1 action = pattern[index] command = action and action[1] or 0 -- something is wrong end if command == 11 then -- parent local ep = root.__p__ or parent if index < #pattern then if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end elseif handle(root,rootdt,k) then return false end else local rootdt = root.dt local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 if command == 30 then if action[5] < 0 then start, stop, step = stop, start, -1 dn = -1 end elseif reverse and index == #pattern then start, stop, step = stop, start, -1 end local idx = 0 local hsh = { } -- this will slooow down the lot for k=start,stop,step do -- we used to have functions for all but a case is faster local e = rootdt[k] local ns, tg = e.rn or e.ns, e.tg if tg then -- we can optimize this for simple searches, but it probably does not pay off hsh[tg] = (hsh[tg] or 0) + 1 idx = idx + 1 if command == 30 then local ns_a, tg_a = action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end if not action[2] then matched = not matched end if matched then n = n + dn if n == action[5] then if index == #pattern then if handle(root,rootdt,root.ri or k) then return false end else if not traverse(e,pattern,handle,reverse,index+1,root) then return false end end break end elseif wildcard then if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end else local matched, multiple = false, false if command == 20 then -- match local ns_a, tg_a = action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end if not action[2] then matched = not matched end elseif command == 21 then -- match one of multiple = true for i=3,#action,2 do local ns_a, tg_a = action[i], action[i+1] if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then matched = true break end end if not action[2] then matched = not matched end elseif command == 22 then -- eq local ns_a, tg_a = action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end matched = matched and e.at[action[6]] == action[7] elseif command == 23 then -- ne local ns_a, tg_a = action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end if not action[2] then matched = not matched end matched = mached and e.at[action[6]] ~= action[7] elseif command == 24 then -- one of eq multiple = true for i=3,#action-2,2 do local ns_a, tg_a = action[i], action[i+1] if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then matched = true break end end if not action[2] then matched = not matched end matched = matched and e.at[action[#action-1]] == action[#action] elseif command == 25 then -- one of ne multiple = true for i=3,#action-2,2 do local ns_a, tg_a = action[i], action[i+1] if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then matched = true break end end if not action[2] then matched = not matched end matched = matched and e.at[action[#action-1]] ~= action[#action] elseif command == 27 then -- has attribute local ns_a, tg_a = action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end if not action[2] then matched = not matched end matched = matched and e.at[action[5]] elseif command == 28 then -- has value local edt, ns_a, tg_a = e.dt, action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end if not action[2] then matched = not matched end matched = matched and edt and edt[1] == action[5] elseif command == 31 then local edt, ns_a, tg_a = e.dt, action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a elseif tg_a == '*' then matched, multiple = ns_a == "*" or ns == ns_a, true else matched = false end if not action[2] then matched = not matched end if matched then matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) end end if matched then -- combine tg test and at test if index == #pattern then if handle(root,rootdt,root.ri or k) then return false end if wildcard then if multiple then if not traverse(e,pattern,handle,reverse,index,root,true) then return false end else -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml if not traverse(e,pattern,handle,reverse,index,root) then return false end end end else if not traverse(e,pattern,handle,reverse,index+1,root) then return false end end elseif command == 14 then -- any if index == #pattern then if handle(root,rootdt,root.ri or k) then return false end else if not traverse(e,pattern,handle,reverse,index+1,root) then return false end end elseif command == 15 then -- many if index == #pattern then if handle(root,rootdt,root.ri or k) then return false end else if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end end -- not here : 11 elseif command == 11 then -- parent local ep = e.__p__ or parent if index < #pattern then if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end elseif handle(root,rootdt,k) then return false end elseif command == 40 and e.special and tg == "@pi@" then -- pi local pi = action[2] if pi ~= "" then local pt = e.dt[1] if pt and pt:find(pi) then if handle(root,rootdt,k) then return false end end elseif handle(root,rootdt,k) then return false end elseif wildcard then if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end end else -- not here : 11 if command == 11 then -- parent local ep = e.__p__ or parent if index < #pattern then if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end elseif handle(root,rootdt,k) then return false end break -- else loop end end end end end end return true end xml.traverse = traverse --[[ldx--Next come all kind of locators and manipulators. The most generic function here
is
For splitting the filter function from the path specification, we can use string matching or lpeg matching. Here the difference in speed is neglectable but the lpeg variant is more robust.
--ldx]]-- -- not faster but hipper ... although ... i can't get rid of the trailing / in the path local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc local slash = P('/') local name = (R("az","AZ","--","__"))^1 local path = C(((1-slash)^0 * slash)^1) local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } local action = Cc(1) * path * C(name) * argument local attribute = Cc(2) * path * P('@') * C(name) local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument local parser = direct + action + attribute local filters = xml.filters local attribute_filter = xml.filters.attributes local default_filter = xml.filters.default -- todo: also hash, could be gc'd function xml.filter(root,pattern) local kind, a, b, c = parser:match(pattern) if kind == 1 or kind == 3 then return (filters[b] or default_filter)(root,a,c) elseif kind == 2 then return attribute_filter(root,a,b) else return default_filter(root,pattern) end end --~ slightly faster, but first we need a proper test file --~ --~ local hash = { } --~ --~ function xml.filter(root,pattern) --~ local h = hash[pattern] --~ if not h then --~ local kind, a, b, c = parser:match(pattern) --~ if kind == 1 then --~ h = { kind, filters[b] or default_filter, a, b, c } --~ elseif kind == 2 then --~ h = { kind, attribute_filter, a, b, c } --~ else --~ h = { kind, default_filter, a, b, c } --~ end --~ hash[pattern] = h --~ end --~ local kind = h[1] --~ if kind == 1 then --~ return h[2](root,h[2],h[4]) --~ elseif kind == 2 then --~ return h[2](root,h[2],h[3]) --~ else --~ return h[2](root,pattern) --~ end --~ end --[[ldx--The following functions collect elements and texts.
--ldx]]-- -- still somewhat bugged function xml.collect_elements(root, pattern, ignorespaces) local rr, dd = { }, { } traverse(root, lpath(pattern), function(r,d,k) local dk = d and d[k] if dk then if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then -- ignore else local n = #rr+1 rr[n], dd[n] = r, dk end end end) return dd, rr end function xml.collect_texts(root, pattern, flatten) local t = { } -- no r collector traverse(root, lpath(pattern), function(r,d,k) if d then local ek = d[k] local tx = ek and ek.dt if flatten then if tx then t[#t+1] = xml.tostring(tx) or "" else t[#t+1] = "" end else t[#t+1] = tx or "" end else t[#t+1] = "" end end) return t end function xml.collect_tags(root, pattern, nonamespace) local t = { } xml.traverse(root, xml.lpath(pattern), function(r,d,k) local dk = d and d[k] if dk and type(dk) == "table" then local ns, tg = e.ns, e.tg if nonamespace then t[#t+1] = tg -- if needed we can return an extra table elseif ns == "" then t[#t+1] = tg else t[#t+1] = ns .. ":" .. tg end end end) return #t > 0 and {} end --[[ldx--Often using an iterators looks nicer in the code than passing handler
functions. The
Which will print all the titles in the document. The iterator variant takes 1.5 times the runtime of the function variant which is due to the overhead in creating the wrapper. So, instead of:
We use the function variants in the filters.
--ldx]]-- local wrap, yield = coroutine.wrap, coroutine.yield function xml.elements(root,pattern,reverse) return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) end function xml.elements_only(root,pattern,reverse) return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) end function xml.each_element(root, pattern, handle, reverse) local ok traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) return ok end function xml.process_elements(root, pattern, handle) traverse(root, lpath(pattern), function(r,d,k) local dkdt = d[k].dt if dkdt then for i=1,#dkdt do local v = dkdt[i] if v.tg then handle(v) end end end end) end function xml.process_attributes(root, pattern, handle) traverse(root, lpath(pattern), function(r,d,k) local ek = d[k] local a = ek.at or { } handle(a) if next(a) then -- next is faster than type (and >0 test) ek.at = a else ek.at = nil end end) end --[[ldx--We've now arrives at the functions that manipulate the tree.
--ldx]]-- function xml.inject_element(root, pattern, element, prepend) if root and element then local matches, collect = { }, nil if type(element) == "string" then element = convert(element,true) end if element then collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end traverse(root, lpath(pattern), collect) for i=1,#matches do local m = matches[i] local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil if element.ri then element = element.dt[element.ri].dt else element = element.dt end if r.ri then edt = r.dt[r.ri].dt else edt = d and d[k] and d[k].dt end if edt then local be, af if prepend then be, af = xml.copy(element), edt else be, af = edt, xml.copy(element) end for i=1,#af do be[#be+1] = af[i] end if r.ri then r.dt[r.ri].dt = be else d[k].dt = be end else -- r.dt = element.dt -- todo end end end end end -- todo: copy ! function xml.insert_element(root, pattern, element, before) -- todo: element als functie if root and element then if pattern == "/" then xml.inject_element(root, pattern, element, before) else local matches, collect = { }, nil if type(element) == "string" then element = convert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end traverse(root, lpath(pattern), collect) for i=#matches,1,-1 do local m = matches[i] local r, d, k, element = m[1], m[2], m[3], m[4] if not before then k = k + 1 end if element.tg then insert(d,k,element) -- untested --~ elseif element.dt then --~ for _,v in ipairs(element.dt) do -- i added --~ insert(d,k,v) --~ k = k + 1 --~ end --~ end else local edt = element.dt if edt then for i=1,#edt do insert(d,k,edt[i]) k = k + 1 end end end end end end end end xml.insert_element_after = xml.insert_element xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end xml.inject_element_after = xml.inject_element xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end function xml.delete_element(root, pattern) local matches, deleted = { }, { } local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end traverse(root, lpath(pattern), collect) for i=#matches,1,-1 do local m = matches[i] deleted[#deleted+1] = remove(m[2],m[3]) end return deleted end function xml.replace_element(root, pattern, element) if type(element) == "string" then element = convert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then traverse(root, lpath(pattern), function(rm, d, k) d[k] = element.dt -- maybe not clever enough end) end end local function load_data(name) -- == io.loaddata local f, data = io.open(name), "" if f then data = f:read("*all",'b') -- 'b' ? f:close() end return data end function xml.include(xmldata,pattern,attribute,recursive,loaddata) -- parse="text" (default: xml), encoding="" (todo) -- attribute = attribute or 'href' pattern = pattern or 'include' loaddata = loaddata or load_data local function include(r,d,k) local ek, name = d[k], nil if not attribute or attribute == "" then local ekdt = ek.dt name = (type(ekdt) == "table" and ekdt[1]) or ekdt end if not name then if ek.at then for a in gmatch(attribute or "href","([^|]+)") do name = ek.at[a] if name then break end end end end local data = (name and name ~= "" and loaddata(name)) or "" if data == "" then xml.empty(d,k) elseif ek.at["parse"] == "text" then -- for the moment hard coded d[k] = xml.escaped(data) else local xi = xml.convert(data) if not xi then xml.empty(d,k) else if recursive then xml.include(xi,pattern,attribute,recursive,loaddata) end xml.assign(d,k,xi) end end end xml.each_element(xmldata, pattern, include) end function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! traverse(root, lpath(pattern), function(r,d,k) local dkdt = d[k].dt if dkdt then -- can be optimized local t = { } for i=1,#dkdt do local str = dkdt[i] if type(str) == "string" then if str == "" then -- stripped else if nolines then str = gsub(str,"[ \n\r\t]+"," ") end if str == "" then -- stripped else t[#t+1] = str end end else t[#t+1] = str end end d[k].dt = t end end) end local function rename_space(root, oldspace, newspace) -- fast variant local ndt = #root.dt for i=1,ndt or 0 do local e = root[i] if type(e) == "table" then if e.ns == oldspace then e.ns = newspace if e.rn then e.rn = newspace end end local edt = e.dt if edt then rename_space(edt, oldspace, newspace) end end end end xml.rename_space = rename_space function xml.remap_tag(root, pattern, newtg) traverse(root, lpath(pattern), function(r,d,k) d[k].tg = newtg end) end function xml.remap_namespace(root, pattern, newns) traverse(root, lpath(pattern), function(r,d,k) d[k].ns = newns end) end function xml.check_namespace(root, pattern, newns) traverse(root, lpath(pattern), function(r,d,k) local dk = d[k] if (not dk.rn or dk.rn == "") and dk.ns == "" then dk.rn = newns end end) end function xml.remap_name(root, pattern, newtg, newns, newrn) traverse(root, lpath(pattern), function(r,d,k) local dk = d[k] dk.tg = newtg dk.ns = newns dk.rn = newrn end) end function xml.filters.found(root,pattern,check_content) local found = false traverse(root, lpath(pattern), function(r,d,k) if check_content then local dk = d and d[k] found = dk and dk.dt and next(dk.dt) and true else found = true end return true end) return found end --[[ldx--Here are a few synonyms.
--ldx]]-- xml.filters.position = xml.filters.index xml.count = xml.filters.count xml.index = xml.filters.index xml.position = xml.filters.index xml.first = xml.filters.first xml.last = xml.filters.last xml.found = xml.filters.found xml.each = xml.each_element xml.process = xml.process_element xml.strip = xml.strip_whitespace xml.collect = xml.collect_elements xml.all = xml.collect_elements xml.insert = xml.insert_element_after xml.inject = xml.inject_element_after xml.after = xml.insert_element_after xml.before = xml.insert_element_before xml.delete = xml.delete_element xml.replace = xml.replace_element --[[ldx--The following helper functions best belong to the
We provide (at least here) two entity handlers. The more extensive
resolver consults a hash first, tries to convert to
The following helper functions best belong to the
This is a prelude to a more extensive logging module. For the sake
of parsing log files, in addition to the standard logging we will
provide an
This looks pretty ugly but we need to speed things up a bit.
--ldx]]-- logs.moreinfo = [[ more information about ConTeXt and the tools that come with it can be found at: maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context webpage : http://www.pragma-ade.nl / http://tex.aanhet.net wiki : http://contextgarden.net ]] logs.levels = { ['error'] = 1, ['warning'] = 2, ['info'] = 3, ['debug'] = 4, } logs.functions = { 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', 'start_run', 'stop_run', 'start_page_number', 'stop_page_number', 'report_output_pages', 'report_output_log', 'report_tex_stat', 'report_job_stat', 'show_open', 'show_close', 'show_load', } logs.tracers = { } logs.level = 0 logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) function logs.set_level(level) logs.level = logs.levels[level] or level end function logs.set_method(method) for _, v in next, logs.functions do logs[v] = logs[method][v] or function() end end end -- tex logging function logs.tex.report(category,fmt,...) -- new if fmt then write_nl(category .. " | " .. format(fmt,...)) else write_nl(category .. " |") end end function logs.tex.line(fmt,...) -- new if fmt then write_nl(format(fmt,...)) else write_nl("") end end function logs.tex.start_page_number() local real, user, sub = texcount.realpageno, texcount.userpageno, texcount.subpageno if real > 0 then if user > 0 then if sub > 0 then write(format("[%s.%s.%s",real,user,sub)) else write(format("[%s.%s",real,user)) end else write(format("[%s",real)) end else write("[-") end end function logs.tex.stop_page_number() write("]") end logs.tex.report_job_stat = statistics.show_job_stat -- xml logging function logs.xml.report(category,fmt,...) -- new if fmt then write_nl(format("This module deals with caching data. It sets up the paths and implements loaders and savers for tables. Best is to set the following variable. When not set, the usual paths will be checked. Personally I prefer the (users) temporary path.
TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.Currently we do no locking when we write files. This is no real problem because most caching involves fonts and the chance of them being written at the same time is small. We also need to extend luatools with a recache feature.
--ldx]]-- local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) caches = caches or { } caches.path = caches.path or nil caches.base = caches.base or "luatex-cache" caches.more = caches.more or "context" caches.direct = false -- true is faster but may need huge amounts of memory caches.tree = false caches.paths = caches.paths or nil caches.force = false caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } function caches.temp() local cachepath = nil local function check(list,isenv) if not cachepath then for k=1,#list do local v = list[k] cachepath = (isenv and (os.env[v] or "")) or v or "" if cachepath == "" then -- next else cachepath = resolvers.clean_path(cachepath) if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" break elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then dir.mkdirs(cachepath) if lfs.isdir(cachepath) and file.iswritable(cachepath) then break end end end cachepath = nil end end end check(resolvers.clean_path_list("TEXMFCACHE") or { }) check(caches.defaults,true) if not cachepath then print("\nfatal error: there is no valid (writable) cache path defined\n") os.exit() elseif not lfs.isdir(cachepath) then -- lfs.attributes(cachepath,"mode") ~= "directory" print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end cachepath = file.collapse_path(cachepath) function caches.temp() return cachepath end return cachepath end function caches.configpath() return table.concat(resolvers.instance.cnffiles,";") end function caches.hashed(tree) return md5.hex(gsub(lower(tree),"[\\\/]+","/")) end function caches.treehash() local tree = caches.configpath() if not tree or tree == "" then return false else return caches.hashed(tree) end end function caches.setpath(...) if not caches.path then if not caches.path then caches.path = caches.temp() end caches.path = resolvers.clean_path(caches.path) -- to be sure caches.tree = caches.tree or caches.treehash() if caches.tree then caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) else caches.path = dir.mkdirs(caches.path,caches.base,caches.more) end end if not caches.path then caches.path = '.' end caches.path = resolvers.clean_path(caches.path) if not table.is_empty({...}) then local pth = dir.mkdirs(caches.path,...) return pth end caches.path = dir.expand_name(caches.path) return caches.path end function caches.definepath(category,subcategory) return function() return caches.setpath(category,subcategory) end end function caches.setluanames(path,name) return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" end function caches.loaddata(path,name) local tmaname, tmcname = caches.setluanames(path,name) local loader = loadfile(tmcname) or loadfile(tmaname) if loader then loader = loader() collectgarbage("step") return loader else return false end end --~ function caches.loaddata(path,name) --~ local tmaname, tmcname = caches.setluanames(path,name) --~ return dofile(tmcname) or dofile(tmaname) --~ end function caches.iswritable(filepath,filename) local tmaname, tmcname = caches.setluanames(filepath,filename) return file.iswritable(tmaname) end function caches.savedata(filepath,filename,data,raw) local tmaname, tmcname = caches.setluanames(filepath,filename) local reduce, simplify = true, true if raw then reduce, simplify = false, false end if caches.direct then file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true end local cleanup = resolvers.boolean_variable("PURGECACHE", false) local strip = resolvers.boolean_variable("LUACSTRIP", true) utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-res'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) local upper, lower, gsub = string.upper, string.lower, string.gsub local prefixes = { } prefixes.environment = function(str) return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "") end prefixes.relative = function(str,n) if io.exists(str) then -- nothing elseif io.exists("./" .. str) then str = "./" .. str else local p = "../" for i=1,n or 2 do if io.exists(p .. str) then str = p .. str break else p = p .. "../" end end end return resolvers.clean_path(str) end prefixes.locate = function(str) local fullname = resolvers.find_given_file(str) or "" return resolvers.clean_path((fullname ~= "" and fullname) or str) end prefixes.filename = function(str) local fullname = resolvers.find_given_file(str) or "" return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str)) end prefixes.pathname = function(str) local fullname = resolvers.find_given_file(str) or "" return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str)) end prefixes.env = prefixes.environment prefixes.rel = prefixes.relative prefixes.loc = prefixes.locate prefixes.kpse = prefixes.locate prefixes.full = prefixes.locate prefixes.file = prefixes.filename prefixes.path = prefixes.pathname local function _resolve_(method,target) if prefixes[method] then return prefixes[method](target) else return method .. ":" .. target end end local function resolve(str) if type(str) == "table" then for k, v in pairs(str) do -- ipairs str[k] = resolve(v) or v end elseif str and str ~= "" then str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_) end return str end resolvers.resolve = resolve if os.uname then for k, v in pairs(os.uname()) do if not prefixes[k] then prefixes[k] = function() return v end end end end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } resolvers.finders = resolvers.finders or { } resolvers.openers = resolvers.openers or { } resolvers.loaders = resolvers.loaders or { } resolvers.finders.notfound = { nil } resolvers.openers.notfound = { nil } resolvers.loaders.notfound = { false, nil, 0 } end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-out'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } outputs = outputs or { } end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-con'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local format, lower, gsub = string.format, string.lower, string.gsub local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) --[[ldx--Once we found ourselves defining similar cache constructs several times, containers were introduced. Containers are used to collect tables in memory and reuse them when possible based on (unique) hashes (to be provided by the calling function).
Caching to disk is disabled by default. Version numbers are stored in the saved table which makes it possible to change the table structures without bothering about the disk cache.
Examples of usage can be found in the font related code.
--ldx]]-- containers = containers or { } containers.usecache = true local function report(container,tag,name) if trace_cache or trace_containers then logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end end local allocated = { } -- tracing function containers.define(category, subcategory, version, enabled) return function() if category and subcategory then local c = allocated[category] if not c then c = { } allocated[category] = c end local s = c[subcategory] if not s then s = { category = category, subcategory = subcategory, storage = { }, enabled = enabled, version = version or 1.000, trace = false, path = caches and caches.setpath and caches.setpath(category,subcategory), } c[subcategory] = s end return s else return nil end end end function containers.is_usable(container, name) return container.enabled and caches and caches.iswritable(container.path, name) end function containers.is_valid(container, name) if name and name ~= "" then local storage = container.storage[name] return storage and not table.is_empty(storage) and storage.cache_version == container.version else return false end end function containers.read(container,name) if container.enabled and caches and not container.storage[name] and containers.usecache then container.storage[name] = caches.loaddata(container.path,name) if containers.is_valid(container,name) then report(container,"loaded",name) else container.storage[name] = nil end end if container.storage[name] then report(container,"reusing",name) end return container.storage[name] end function containers.write(container, name, data) if data then data.cache_version = container.version if container.enabled and caches then local unique, shared = data.unique, data.shared data.unique, data.shared = nil, nil caches.savedata(container.path, name, data) report(container,"saved",name) data.unique, data.shared = unique, shared end report(container,"stored",name) container.storage[name] = data end return data end function containers.content(container,name) return container.storage[name] end function containers.cleanname(name) return (gsub(lower(name),"[^%w%d]+","-")) end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-use'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local format, lower, gsub = string.format, string.lower, string.gsub local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. local save_data = resolvers.save_data local load_data = resolvers.load_data resolvers.cachepath = nil -- public, for tracing resolvers.usecache = true -- public, for tracing function resolvers.save_data(dataname) save_data(dataname, function(cachename,dataname) resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) if resolvers.usecache then resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") return file.join(resolvers.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) end end) end function resolvers.load_data(pathname,dataname,filename) load_data(pathname,dataname,filename,function(dataname,filename) resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) if resolvers.usecache then resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") return file.join(resolvers.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then filename = dataname end return file.join(pathname,filename) end end) end -- we will make a better format, maybe something xml or just text or lua resolvers.automounted = resolvers.automounted or { } function resolvers.automount(usecache) local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) if table.is_empty(mountpaths) and usecache then mountpaths = { caches.setpath("mount") } end if not table.is_empty(mountpaths) then statistics.starttiming(resolvers.instance) for k, root in pairs(mountpaths) do local f = io.open(root.."/url.tmi") if f then for line in f:lines() do if line then if line:find("^[%%#%-]") then -- or %W -- skip elseif line:find("^zip://") then if trace_locating then logs.report("fileio","mounting %s",line) end table.insert(resolvers.automounted,line) resolvers.usezipfile(line) end end end f:close() end end statistics.stoptiming(resolvers.instance) end end -- status info statistics.register("used config path", function() return caches.configpath() end) statistics.register("used cache path", function() return caches.temp() or "?" end) -- experiment (code will move) function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname local enginebanner = status.list().banner if formatbanner and enginebanner and sourcefile then local luvname = file.replacesuffix(texname,"luv") local luvdata = { enginebanner = enginebanner, formatbanner = formatbanner, sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), sourcefile = sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) end end function statistics.check_fmt_status(texname) local enginebanner = status.list().banner if enginebanner and texname then local luvname = file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then local luv = dofile(luvname) if luv and luv.sourcefile then local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") if luv.enginebanner and luv.enginebanner ~= enginebanner then return "engine mismatch" end if luv.sourcehash and luv.sourcehash ~= sourcehash then return "source mismatch" end else return "invalid status file" end else return "missing status file" end end return true end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-zip'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local format, find = string.format, string.find local trace_locating, trace_verbose = false, false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end) zip = zip or { } zip.archives = zip.archives or { } zip.registeredfiles = zip.registeredfiles or { } local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators local archives = zip.archives -- zip:///oeps.zip?name=bla/bla.tex -- zip:///oeps.zip?tree=tex/texmf-local local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then return "zip:///" .. str else return str end end function zip.openarchive(name) if not name or name == "" then return nil else local arch = archives[name] if not arch then local full = resolvers.find_file(name) or "" arch = (full ~= "" and zip.open(full)) or false archives[name] = arch end return arch end end function zip.closearchive(name) if not name or (name == "" and archives[name]) then zip.close(archives[name]) archives[name] = nil end end -- zip:///texmf.zip?tree=/tex/texmf -- zip:///texmf.zip?tree=/tex/texmf-local -- zip:///texmf-mine.zip?tree=/tex/texmf-projects function locators.zip(specification) -- where is this used? startup zips (untested) specification = resolvers.splitmethod(specification) local zipfile = specification.path local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree if trace_locating then if zfile then logs.report("fileio",'! zip locator, found: %s',specification.original) else logs.report("fileio",'? zip locator, not found: %s',specification.original) end end end function hashers.zip(tag,name) if trace_verbose then logs.report("fileio","loading zip file %s as %s",name,tag) end resolvers.usezipfile(format("%s?tree=%s",tag,name)) end function concatinators.zip(tag,path,name) if not path or path == "" then return format('%s?name=%s',tag,name) else return format('%s?name=%s/%s',tag,path,name) end end function resolvers.isreadable.zip(name) return true end function finders.zip(specification,filetype) specification = resolvers.splitmethod(specification) if specification.path then local q = url.query(specification.query) if q.name then local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then logs.report("fileio",'! zip finder, path: %s',specification.path) end local dfile = zfile:open(q.name) if dfile then dfile = zfile:close() if trace_locating then logs.report("fileio",'+ zip finder, name: %s',q.name) end return specification.original end elseif trace_locating then logs.report("fileio",'? zip finder, path %s',specification.path) end end end if trace_locating then logs.report("fileio",'- zip finder, name: %s',filename) end return unpack(finders.notfound) end function openers.zip(specification) local zipspecification = resolvers.splitmethod(specification) if zipspecification.path then local q = url.query(zipspecification.query) if q.name then local zfile = zip.openarchive(zipspecification.path) if zfile then if trace_locating then logs.report("fileio",'+ zip starter, path: %s',zipspecification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_open(specification) return openers.text_opener(specification,dfile,'zip') end elseif trace_locating then logs.report("fileio",'- zip starter, path %s',zipspecification.path) end end end if trace_locating then logs.report("fileio",'- zip opener, name: %s',filename) end return unpack(openers.notfound) end function loaders.zip(specification) specification = resolvers.splitmethod(specification) if specification.path then local q = url.query(specification.query) if q.name then local zfile = zip.openarchive(specification.path) if zfile then if trace_locating then logs.report("fileio",'+ zip starter, path: %s',specification.path) end local dfile = zfile:open(q.name) if dfile then logs.show_load(filename) if trace_locating then logs.report("fileio",'+ zip loader, name: %s',filename) end local s = dfile:read("*all") dfile:close() return true, s, #s end elseif trace_locating then logs.report("fileio",'- zip starter, path: %s',specification.path) end end end if trace_locating then logs.report("fileio",'- zip loader, name: %s',filename) end return unpack(openers.notfound) end -- zip:///somefile.zip -- zip:///somefile.zip?tree=texmf-local -> mount function resolvers.usezipfile(zipname) zipname = validzip(zipname) if trace_locating then logs.report("fileio",'! zip use, file: %s',zipname) end local specification = resolvers.splitmethod(zipname) local zipfile = specification.path if zipfile and not zip.registeredfiles[zipname] then local tree = url.query(specification.query).tree or "" if trace_locating then logs.report("fileio",'! zip register, file: %s',zipname) end local z = zip.openarchive(zipfile) if z then local instance = resolvers.instance if trace_locating then logs.report("fileio","= zipfile, registering: %s",zipname) end statistics.starttiming(instance) resolvers.prepend_hash('zip',zipname,zipfile) resolvers.extend_texmf_var(zipname) -- resets hashes too zip.registeredfiles[zipname] = z instance.files[zipname] = resolvers.register_zip_file(z,tree or "") statistics.stoptiming(instance) elseif trace_locating then logs.report("fileio","? zipfile, unknown: %s",zipname) end elseif trace_locating then logs.report("fileio",'! zip register, no file: %s',zipname) end end function resolvers.register_zip_file(z,tree) local files, filter = { }, "" if tree == "" then filter = "^(.+)/(.-)$" else filter = format("^%s/(.+)/(.-)$",tree) end if trace_locating then logs.report("fileio",'= zip filter: %s',filter) end local register, n = resolvers.register_file, 0 for i in z:files() do local path, name = i.filename:match(filter) if path then if name and name ~= '' then register(files, name, path) n = n + 1 else -- directory end else register(files, i.filename, '') n = n + 1 end end logs.report("fileio",'= zip entries: %s',n) return files end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-crl'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } curl = curl or { } curl.cached = { } curl.cachepath = caches.definepath("curl") local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders function curl.fetch(protocol, name) local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-") -- cachename = cachename:gsub("[\\/]", io.fileseparator) cachename = cachename:gsub("[\\]", "/") -- cleanup if not curl.cached[name] then if not io.exists(cachename) then curl.cached[name] = cachename local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" os.spawn(command) end if io.exists(cachename) then curl.cached[name] = cachename else curl.cached[name] = "" end end return curl.cached[name] end function finders.curl(protocol,filename) local foundname = curl.fetch(protocol, filename) return finders.generic(protocol,foundname,filetype) end function openers.curl(protocol,filename) return openers.generic(protocol,filename) end function loaders.curl(protocol,filename) return loaders.generic(protocol,filename) end -- todo: metamethod function curl.install(protocol) finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end openers[protocol] = function (filename) return openers.curl(protocol,filename) end loaders[protocol] = function (filename) return loaders.curl(protocol,filename) end end curl.install('http') curl.install('https') curl.install('ftp') end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-kps'] = { version = 1.001, comment = "companion to luatools.lua", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --[[ldx--This file is used when we want the input handlers to behave like
How about just forgetting about them?
--ldx]]-- local suffixes = resolvers.suffixes local formats = resolvers.formats suffixes['gf'] = { 'If you wondered abou tsome of the previous mappings, how about the next bunch:
--ldx]]-- formats['bib'] = '' formats['bst'] = '' formats['mft'] = '' formats['ist'] = '' formats['web'] = '' formats['cweb'] = '' formats['MetaPost support'] = '' formats['TeX system documentation'] = '' formats['TeX system sources'] = '' formats['Troff fonts'] = '' formats['dvips config'] = '' formats['graphic/figure'] = '' formats['ls-R'] = '' formats['other text files'] = '' formats['other binary files'] = '' formats['gf'] = '' formats['pk'] = '' formats['base'] = 'MFBASES' formats['cnf'] = '' formats['mem'] = 'MPMEMS' formats['mf'] = 'MFINPUTS' formats['mfpool'] = 'MFPOOL' formats['mppool'] = 'MPPOOL' formats['texpool'] = 'TEXPOOL' formats['PostScript header'] = 'TEXPSHEADERS' formats['cmap files'] = 'CMAPFONTS' formats['type42 fonts'] = 'T42FONTS' formats['web2c files'] = 'WEB2C' formats['pdftex config'] = 'PDFTEXCONFIG' formats['texmfscripts'] = 'TEXMFSCRIPTS' formats['bitmap font'] = '' formats['lig files'] = 'LIGFONTS' end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-aux'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local find = string.find local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix local scriptpath = "scripts/context/lua" newname = file.addsuffix(newname,"lua") local oldscript = resolvers.clean_path(oldname) if trace_verbose then logs.report("fileio","to be replaced old script %s", oldscript) end local newscripts = resolvers.find_files(newname) or { } if #newscripts == 0 then if trace_verbose then logs.report("fileio","unable to locate new script") end else for i=1,#newscripts do local newscript = resolvers.clean_path(newscripts[i]) if trace_verbose then logs.report("fileio","checking new script %s", newscript) end if oldscript == newscript then if trace_verbose then logs.report("fileio","old and new script are the same") end elseif not find(newscript,scriptpath) then if trace_verbose then logs.report("fileio","new script should come from %s",scriptpath) end elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then if trace_verbose then logs.report("fileio","invalid new script name") end else local newdata = io.loaddata(newscript) if newdata then if trace_verbose then logs.report("fileio","old script content replaced by new content") end io.savedata(oldscript,newdata) break elseif trace_verbose then logs.report("fileio","unable to load new script") end end end end end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['data-tmf'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- loads *.tmf files in minimal tree roots (to be optimized and documented) function resolvers.check_environment(tree) logs.simpleline() os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) logs.simpleline() logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH')) logs.simple("preset : TEXOS => %s", os.getenv('TEXOS')) logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) logs.simple("preset : TMP => %s", os.getenv('TMP')) logs.simple('') end function resolvers.load_environment(name) -- todo: key=value as well as lua local f = io.open(name) if f then for line in f:lines() do if line:find("^[%%%#]") then -- skip comment else local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") if how then value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) if how == "=" or how == "<<" then os.setenv(key,value) elseif how == "?" or how == "??" then os.setenv(key,os.getenv(key) or value) elseif how == "<" or how == "+=" then if os.getenv(key) then os.setenv(key,os.getenv(key) .. io.fileseparator .. value) else os.setenv(key,value) end elseif how == ">" or how == "=+" then if os.getenv(key) then os.setenv(key,value .. io.pathseparator .. os.getenv(key)) else os.setenv(key,value) end end end end end f:close() end end function resolvers.load_tree(tree) if tree and tree ~= "" then local setuptex = 'setuptex.tmf' if lfs.attributes(tree, "mode") == "directory" then -- check if not nil setuptex = tree .. "/" .. setuptex else setuptex = tree end if io.exists(setuptex) then resolvers.check_environment(tree) resolvers.load_environment(setuptex) end end end end -- of closure do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-sta'] = { version = 1.001, author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } -- this code is used in the updater states = states or { } states.data = states.data or { } states.hash = states.hash or { } states.tag = states.tag or "" states.filename = states.filename or "" function states.save(filename,tag) tag = tag or states.tag filename = file.addsuffix(filename or states.filename,'lus') io.savedata(filename, "-- generator : luat-sta.lua\n" .. "-- state tag : " .. tag .. "\n\n" .. table.serialize(states.data[tag or states.tag] or {},true) ) end function states.load(filename,tag) states.filename = filename states.tag = tag or "whatever" states.filename = file.addsuffix(states.filename,'lus') states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } end function states.set_by_tag(tag,key,value,default,persistent) local d, h = states.data[tag], states.hash[tag] if d then if type(d) == "table" then local dkey, hkey = key, key local pre, post = key:match("(.+)%.([^%.]+)$") if pre and post then for k in pre:gmatch("[^%.]+") do local dk = d[k] if not dk then dk = { } d[k] = dk end d = dk end dkey, hkey = post, key end if type(value) == nil then value = value or default elseif persistent then value = value or d[dkey] or default else value = value or default end d[dkey], h[hkey] = value, value elseif type(d) == "string" then -- weird states.data[tag], states.hash[tag] = value, value end end end function states.get_by_tag(tag,key,default) local h = states.hash[tag] if h and h[key] then return h[key] else local d = states.data[tag] if d then for k in key:gmatch("[^%.]+") do local dk = d[k] if dk then d = dk else return default end end return d or default end end end function states.set(key,value,default,persistent) states.set_by_tag(states.tag,key,value,default,persistent) end function states.get(key,default) return states.get_by_tag(states.tag,key,default) end --~ states.data.update = { --~ ["version"] = { --~ ["major"] = 0, --~ ["minor"] = 1, --~ }, --~ ["rsync"] = { --~ ["server"] = "contextgarden.net", --~ ["module"] = "minimals", --~ ["repository"] = "current", --~ ["flags"] = "-rpztlv --stats", --~ }, --~ ["tasks"] = { --~ ["update"] = true, --~ ["make"] = true, --~ ["delete"] = false, --~ }, --~ ["platform"] = { --~ ["host"] = true, --~ ["other"] = { --~ ["mswin"] = false, --~ ["linux"] = false, --~ ["linux-64"] = false, --~ ["osx-intel"] = false, --~ ["osx-ppc"] = false, --~ ["sun"] = false, --~ }, --~ }, --~ ["context"] = { --~ ["available"] = {"current", "beta", "alpha", "experimental"}, --~ ["selected"] = "current", --~ }, --~ ["formats"] = { --~ ["cont-en"] = true, --~ ["cont-nl"] = true, --~ ["cont-de"] = false, --~ ["cont-cz"] = false, --~ ["cont-fr"] = false, --~ ["cont-ro"] = false, --~ }, --~ ["engine"] = { --~ ["pdftex"] = { --~ ["install"] = true, --~ ["formats"] = { --~ ["pdftex"] = true, --~ }, --~ }, --~ ["luatex"] = { --~ ["install"] = true, --~ ["formats"] = { --~ }, --~ }, --~ ["xetex"] = { --~ ["install"] = true, --~ ["formats"] = { --~ ["xetex"] = false, --~ }, --~ }, --~ ["metapost"] = { --~ ["install"] = true, --~ ["formats"] = { --~ ["mpost"] = true, --~ ["metafun"] = true, --~ }, --~ }, --~ }, --~ ["fonts"] = { --~ }, --~ ["doc"] = { --~ }, --~ ["modules"] = { --~ ["f-urwgaramond"] = false, --~ ["f-urwgothic"] = false, --~ ["t-bnf"] = false, --~ ["t-chromato"] = false, --~ ["t-cmscbf"] = false, --~ ["t-cmttbf"] = false, --~ ["t-construction-plan"] = false, --~ ["t-degrade"] = false, --~ ["t-french"] = false, --~ ["t-lettrine"] = false, --~ ["t-lilypond"] = false, --~ ["t-mathsets"] = false, --~ ["t-tikz"] = false, --~ ["t-typearea"] = false, --~ ["t-vim"] = false, --~ }, --~ } --~ states.save("teststate", "update") --~ states.load("teststate", "update") --~ print(states.get_by_tag("update","rsync.server","unknown")) --~ states.set_by_tag("update","rsync.server","oeps") --~ print(states.get_by_tag("update","rsync.server","unknown")) --~ states.save("teststate", "update") --~ states.load("teststate", "update") --~ print(states.get_by_tag("update","rsync.server","unknown")) end -- of closure -- end library merge own = { } -- not local own.libs = { -- todo: check which ones are really needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-io.lua', 'l-number.lua', 'l-set.lua', 'l-os.lua', 'l-file.lua', 'l-md5.lua', 'l-dir.lua', 'l-boolean.lua', 'l-math.lua', -- 'l-unicode.lua', -- 'l-tex.lua', 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', 'lxml-tab.lua', 'lxml-pth.lua', 'lxml-ent.lua', 'lxml-mis.lua', 'trac-tra.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', 'data-res.lua', 'data-tmp.lua', 'data-pre.lua', 'data-inp.lua', 'data-out.lua', 'data-con.lua', 'data-use.lua', -- 'data-tex.lua', -- 'data-bin.lua', 'data-zip.lua', 'data-crl.lua', -- 'data-lua.lua', 'data-kps.lua', -- so that we can replace kpsewhich 'data-aux.lua', -- updater 'data-tmf.lua', -- tree files -- needed ? 'luat-sta.lua', -- states } -- We need this hack till luatex is fixed. -- -- for k,v in pairs(arg) do print(k,v) end if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end -- End of hack. own.name = (environment and environment.ownname) or arg[0] or 'luatools.lua' own.path = string.match(own.name,"^(.+)[\\/].-$") or "." own.list = { '.' } if own.path ~= '.' then table.insert(own.list,own.path) end table.insert(own.list,own.path.."/../../../tex/context/base") table.insert(own.list,own.path.."/mtx") table.insert(own.list,own.path.."/../sources") local function locate_libs() for _, lib in pairs(own.libs) do for _, pth in pairs(own.list) do local filename = string.gsub(pth .. "/" .. lib,"\\","/") local codeblob = loadfile(filename) if codeblob then codeblob() own.list = { pth } -- speed up te search break end end end end if not resolvers then locate_libs() end if not resolvers then print("") print("Mtxrun is unable to start up due to lack of libraries. You may") print("try to run 'lua mtxrun.lua --selfmerge' in the path where this") print("script is located (normally under ..../scripts/context/lua) which") print("will make this script library independent.") os.exit() end logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) local instance = resolvers.reset() runners = runners or { } -- global messages = messages or { } messages.help = [[ --script run an mtx script (--noquotes) --execute run a script or program (--noquotes) --resolve resolve prefixed arguments --ctxlua run internally (using preloaded libs) --locate locate given filename --autotree use texmf tree cf. env 'texmfstart_tree' or 'texmfstarttree' --tree=pathtotree use given texmf tree (default file: 'setuptex.tmf') --environment=name use given (tmf) environment file --path=runpath go to given path before execution --ifchanged=filename only execute when given file has changed (md checksum) --iftouched=old,new only execute when given file has changed (time stamp) --make create stubs for (context related) scripts --remove remove stubs (context related) scripts --stubpath=binpath paths where stubs wil be written --windows create windows (mswin) stubs --unix create unix (linux) stubs --verbose give a bit more info --engine=str target engine --progname=str format or backend --edit launch editor with found file --launch (--all) launch files like manuals, assumes os support --intern run script using built in libraries --usekpse use kpse as fallback (when no mkiv and cache installed, often slower) --forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) ]] runners.applications = { ["lua"] = "luatex --luaonly", ["luc"] = "luatex --luaonly", ["pl"] = "perl", ["py"] = "python", ["rb"] = "ruby", } runners.suffixes = { 'rb', 'lua', 'py', 'pl' } runners.registered = { texexec = { 'texexec.rb', true }, -- context mkii runner (only tool not to be luafied) texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied -- texwork = { \texwork.pl', false }, -- perltk based editing environment, only used at pragma makempy = { 'makempy.pl', true }, mptopdf = { 'mptopdf.pl', true }, pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) -- examplex = { 'examplex.rb', false }, concheck = { 'concheck.rb', false }, runtools = { 'runtools.rb', true }, textools = { 'textools.rb', true }, tmftools = { 'tmftools.rb', true }, ctxtools = { 'ctxtools.rb', true }, rlxtools = { 'rlxtools.rb', true }, pdftools = { 'pdftools.rb', true }, mpstools = { 'mpstools.rb', true }, -- exatools = { 'exatools.rb', true }, xmltools = { 'xmltools.rb', true }, -- luatools = { 'luatools.lua', true }, mtxtools = { 'mtxtools.rb', true }, pdftrimwhite = { 'pdftrimwhite.pl', false } } runners.launchers = { windows = { }, unix = { } } function runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then logs.simple("file '%s' is unchanged",checkname) return "skip" else logs.simple("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end local oldname, newname = string.split(environment.argument("iftouched") or "", ",") if oldname and newname and oldname ~= "" and newname ~= "" then if not file.needs_updating(oldname,newname) then logs.simple("file '%s' and '%s' have same age",oldname,newname) return "skip" else logs.simple("file '%s' is older than '%s'",oldname,newname) end end local tree = environment.argument('tree') or "" if environment.argument('autotree') then tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree end if tree and tree ~= "" then resolvers.load_tree(tree) end local env = environment.argument('environment') or "" if env and env ~= "" then for _,e in pairs(string.split(env)) do -- maybe force suffix when not given resolvers.load_tree(e) end end local runpath = environment.argument("path") if runpath and not lfs.chdir(runpath) then logs.simple("unable to change to path '%s'",runpath) return "error" end return "run" end function runners.execute_script(fullname,internal) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then return true elseif state == "run" then instance.progname = environment.argument("progname") or instance.progname instance.format = environment.argument("format") or instance.format local path, name, suffix, result = file.dirname(fullname), file.basename(fullname), file.extname(fullname), "" if path ~= "" then result = fullname elseif name then name = name:gsub("^int[%a]*:",function() internal = true return "" end ) name = name:gsub("^script:","") if suffix == "" and runners.registered[name] and runners.registered[name][1] then name = runners.registered[name][1] suffix = file.extname(name) end if suffix == "" then -- loop over known suffixes for _,s in pairs(runners.suffixes) do result = resolvers.find_file(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end elseif runners.applications[suffix] then result = resolvers.find_file(name, 'texmfscripts') else -- maybe look on path result = resolvers.find_file(name, 'other text files') end end if result and result ~= "" then local before, after = environment.split_arguments(fullname) -- already done environment.arguments_before, environment.arguments_after = before, after if internal then arg = { } for _,v in pairs(after) do arg[#arg+1] = v end dofile(result) else local binary = runners.applications[file.extname(result)] if binary and binary ~= "" then result = binary .. " " .. result end local command = result .. " " .. environment.reconstruct_commandline(after,noquote) if logs.verbose then logs.simpleline() logs.simple("executing: %s",command) logs.simpleline() logs.simpleline() io.flush() end local code = os.exec(command) -- maybe spawn return code == 0 end end end end return false end function runners.execute_program(fullname) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then return true elseif state == "run" then local before, after = environment.split_arguments(fullname) environment.initialize_arguments(after) fullname = fullname:gsub("^bin:","") local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") logs.simpleline() logs.simple("executing: %s",command) logs.simpleline() logs.simpleline() io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn return code == 0 end end return false end -- the --usekpse flag will fallback on kpse local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' function runners.handle_stubs(create) local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported local windows = environment.argument('windows') or environment.argument('mswin') or false local unix = environment.argument('unix') or environment.argument('linux') or false if not windows and not unix then if os.platform == "unix" then unix = true else windows = true end end for _,v in pairs(runners.registered) do local name, doit = v[1], v[2] if doit then local base = string.gsub(file.basename(name), "%.(.-)$", "") if create then if windows then io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name)) logs.simple("windows stub for '%s' created",base) end if unix then io.savedata(file.join(stubpath,base),string.format(unix_stub,name)) logs.simple("unix stub for '%s' created",base) end else if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then logs.simple("windows stub for '%s' removed", base) end if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then logs.simple("unix stub for '%s' removed",base) end end end end end function runners.resolve_string(filename) if filename and filename ~= "" then runners.report_location(resolvers.resolve(filename)) end end function runners.locate_file(filename) -- differs from texmfstart where locate appends .com .exe .bat ... todo if filename and filename ~= "" then runners.report_location(resolvers.find_given_file(filename)) end end function runners.locate_platform() runners.report_location(os.currentplatform()) end function runners.report_location(result) if logs.verbose then logs.simpleline() if result and result ~= "" then logs.simple(result) else logs.simple("not found") end else io.write(result) end end function runners.edit_script(filename) -- we assume that vim is present on most systems local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' local rest = resolvers.resolve(filename) if rest ~= "" then local command = editor .. " " .. rest if logs.verbose then logs.simpleline() logs.simple("starting editor: %s",command) logs.simple_line() logs.simple_line() end os.launch(command) end end function runners.save_script_session(filename, list) local t = { } for _, key in ipairs(list) do t[key] = environment.arguments[key] end io.savedata(filename,table.serialize(t,true)) end function runners.load_script_session(filename) if lfs.isfile(filename) then local t = io.loaddata(filename) if t then t = loadstring(t) if t then t = t() end for key, value in pairs(t) do environment.arguments[key] = value end end end end function resolvers.launch(str) -- maybe we also need to test on mtxrun.launcher.suffix environment -- variable or on windows consult the assoc and ftype vars and such local launchers = runners.launchers[os.platform] if launchers then local suffix = file.extname(str) if suffix then local runner = launchers[suffix] if runner then str = runner .. " " .. str end end end os.launch(str) end function runners.launch_file(filename) instance.allresults = true logs.setverbose(true) local pattern = environment.arguments["pattern"] if not pattern or pattern == "" then pattern = filename end if not pattern or pattern == "" then logs.simple("provide name or --pattern=") else local t = resolvers.find_files(pattern) if not t or #t == 0 then t = resolvers.find_files("*/" .. pattern) end if not t or #t == 0 then t = resolvers.find_files("*/" .. pattern .. "*") end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do logs.simple("launching %s", v) resolvers.launch(v) end else logs.simple("launching %s", t[1]) resolvers.launch(t[1]) end else logs.simple("no match for %s", pattern) end end end function runners.find_mtx_script(filename) local function found(name) local path = file.dirname(name) if path and path ~= "" then return false else local fullname = own and own.path and file.join(own.path,name) return io.exists(fullname) and fullname end end filename = file.addsuffix(filename,"lua") local basename = file.removesuffix(file.basename(filename)) local suffix = file.extname(filename) -- qualified path, raw name local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename if fullname and fullname ~= "" then return fullname end -- current path, raw name fullname = "./" .. filename fullname = io.exists(fullname) and fullname if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-