diff options
Diffstat (limited to 'scripts/context/lua')
25 files changed, 13390 insertions, 9406 deletions
diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua index 1e38edeab..aacdbd16d 100644 --- a/scripts/context/lua/luatools.lua +++ b/scripts/context/lua/luatools.lua @@ -1,25 +1,26 @@ #!/usr/bin/env texlua +if not modules then modules = { } end modules ['luatools'] = { + version = 1.001, + comment = "companion to context.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format = string.format + -- one can make a stub: -- -- #!/bin/sh -- env LUATEXDIR=/....../texmf/scripts/context/lua texlua luatools.lua "$@" --- filename : luatools.lua --- comment : companion to context.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -- Although this script is part of the ConTeXt distribution it is -- relatively indepent of ConTeXt. The same is true for some of -- the luat files. We may may make them even less dependent in -- the future. As long as Luatex is under development the -- interfaces and names of functions may change. -banner = "version 1.2.2 - 2006+ - PRAGMA ADE / CONTEXT" -texlua = true - -- For the sake of independence we optionally can merge the library -- code here. It's too much code, but that does not harm. Much of the -- library code is used elsewhere. We don't want dependencies on @@ -28,141 +29,42 @@ texlua = true -- needed when texmfstart is used, or when the proper stub is used or -- when (windows) suffix binding is active. +texlua = true + -- begin library merge --- filename : l-string.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-string'] = 1.001 - ---~ function string.split(str, pat) -- taken from the lua wiki ---~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ table.insert(t,cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end<=string.len(str) then ---~ table.insert(t,(string.sub(str,last_end))) ---~ end ---~ return t ---~ end ---~ function string:split(pat) -- taken from the lua wiki but adapted ---~ local t = { } -- self and colon usage (faster) ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = self:find(fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ t[#t+1] = cap ---~ end ---~ last_end = e+1 ---~ s, e, cap = self:find(fpat, last_end) ---~ end ---~ if last_end <= #self then ---~ t[#t+1] = self:sub(last_end) ---~ end ---~ return t ---~ end ---~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed ---~ ---~ function string:splitter(pat) ---~ local st, g = 1, self:gmatch("()"..pat.."()") ---~ local function splitter(self) ---~ if st then ---~ local s, f = g() ---~ local rv = self:sub(st, (s or 0)-1) ---~ st = f ---~ return rv ---~ end ---~ end ---~ return splitter, self ---~ end -function string:splitter(pat) - -- by Rici Lake (posted on lua list) -- only names changed - -- p 79 ref man: () returns position of match - local st, g = 1, self:gmatch("()("..pat..")") - local function strgetter(self, segs, seps, sep, cap1, ...) - st = sep and seps + #sep - return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... - end - local function strsplitter(self) - if st then return strgetter(self, st, g()) end - end - return strsplitter, self -end +do -- create closure to overcome 200 locals limit -function string:split(separator) - local t = {} - for k in self:splitter(separator) do t[#t+1] = k end - return t -end +if not modules then modules = { } end modules ['l-string'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --- faster than a string:split: +local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep -function string:splitchr(chr) - if #self > 0 then - local t = { } - for s in (self..chr):gmatch("(.-)"..chr) do - t[#t+1] = s +if not string.split then + + -- this will be overloaded by a faster lpeg variant + + function string:split(pattern) + if #self > 0 then + local t = { } + for s in gmatch(self..pattern,"(.-)"..pattern) do + t[#t+1] = s + end + return t + else + return { } end - return t - else - return { } end -end -function string.piecewise(str, pat, fnc) -- variant of split - for k in string.splitter(str,pat) do fnc(k) end end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ for k in str:splitter(pat) do fnc(k) end ---~ end - ---~ do if lpeg then - ---~ -- this alternative is 30% faster esp when we cache them ---~ -- problem: no expressions - ---~ splitters = { } - ---~ function string:split(separator) ---~ if #self > 0 then ---~ local split = splitters[separator] ---~ if not split then ---~ -- based on code by Roberto ---~ local p = lpeg.P(separator) ---~ local c = lpeg.C((1-p)^0) ---~ split = lpeg.Ct(c*(p*c)^0) ---~ splitters[separator] = split ---~ end ---~ return split:match(self) ---~ else ---~ return { } ---~ end ---~ end - ---~ string.splitchr = string.split - ---~ function string:piecewise(separator,fnc) ---~ for _,v in pairs(self:split(separator)) do ---~ fnc(v) ---~ end ---~ end - ---~ end end - local chr_to_esc = { ["%"] = "%%", ["."] = "%.", @@ -176,20 +78,20 @@ local chr_to_esc = { string.chr_to_esc = chr_to_esc function string:esc() -- variant 2 - return (self:gsub("(.)",chr_to_esc)) + return (gsub(self,"(.)",chr_to_esc)) end function string:unquote() - return (self:gsub("^([\"\'])(.*)%1$","%2")) + return (gsub(self,"^([\"\'])(.*)%1$","%2")) end -function string:quote() +function string:quote() -- we could use format("%q") return '"' .. self:unquote() .. '"' end function string:count(pattern) -- variant 3 local n = 0 - for _ in self:gmatch(pattern) do + for _ in gmatch(self,pattern) do n = n + 1 end return n @@ -198,29 +100,25 @@ end function string:limit(n,sentinel) if #self > n then sentinel = sentinel or " ..." - return self:sub(1,(n-#sentinel)) .. sentinel + return sub(self,1,(n-#sentinel)) .. sentinel else return self end end function string:strip() - return (self:gsub("^%s*(.-)%s*$", "%1")) + return (gsub(self,"^%s*(.-)%s*$", "%1")) end ---~ function string.strip(str) -- slightly different ---~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) ---~ end - function string:is_empty() - return not self:find("%S") + return not find(find,"%S") end function string:enhance(pattern,action) local ok, n = true, 0 while ok do ok = false - self = self:gsub(pattern, function(...) + self = gsub(self,pattern, function(...) ok, n = true, n + 1 return action(...) end) @@ -228,59 +126,19 @@ function string:enhance(pattern,action) return self, n end ---~ function string:enhance(pattern,action) ---~ local ok, n = 0, 0 ---~ repeat ---~ self, ok = self:gsub(pattern, function(...) ---~ n = n + 1 ---~ return action(...) ---~ end) ---~ until ok == 0 ---~ return self, n ---~ end - ---~ function string:to_hex() ---~ if self then ---~ return (self:gsub("(.)",function(c) ---~ return string.format("%02X",c:byte()) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - ---~ function string:from_hex() ---~ if self then ---~ return (self:gsub("(..)",function(c) ---~ return string.char(tonumber(c,16)) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - -string.chr_to_hex = { } -string.hex_to_chr = { } +local chr_to_hex, hex_to_chr = { }, { } for i=0,255 do - local c, h = string.char(i), string.format("%02X",i) - string.chr_to_hex[c], string.hex_to_chr[h] = h, c + local c, h = char(i), format("%02X",i) + chr_to_hex[c], hex_to_chr[h] = h, c end ---~ function string:to_hex() ---~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end ---~ end - ---~ function string:from_hex() ---~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end ---~ end - function string:to_hex() - return ((self or ""):gsub("(.)",string.chr_to_hex)) + return (gsub(self or "","(.)",chr_to_hex)) end function string:from_hex() - return ((self or ""):gsub("(..)",string.hex_to_chr)) + return (gsub(self or "","(..)",hex_to_chr)) end if not string.characters then @@ -294,7 +152,7 @@ if not string.characters then end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(str:sub(index,index)) end function string:bytes() return nextbyte, self, 0 @@ -302,9 +160,7 @@ if not string.characters then end ---~ function string:padd(n,chr) ---~ return self .. self.rep(chr or " ",n-#self) ---~ end +-- we can use format for this (neg n) function string:rpadd(n,chr) local m = n-#self @@ -326,8 +182,8 @@ end string.padd = string.rpadd -function is_number(str) - return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +function is_number(str) -- tonumber + return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 end --~ print(is_number("1")) @@ -339,9 +195,9 @@ end --~ print(is_number("+.1")) function string:split_settings() -- no {} handling, see l-aux for lpeg variant - if self:find("=") then + if find(self,"=") then local t = { } - for k,v in self:gmatch("(%a+)=([^%,]*)") do + for k,v in gmatch(self,"(%a+)=([^%,]*)") do t[k] = v end return t @@ -363,24 +219,67 @@ local patterns_escapes = { } function string:pattesc() - return (self:gsub(".",patterns_escapes)) + return (gsub(self,".",patterns_escapes)) end function string:tohash() local t = { } - for s in self:gmatch("([^, ]+)") do -- lpeg + for s in gmatch(self,"([^, ]+)") do -- lpeg t[s] = true end return t end +local pattern = lpeg.Ct(lpeg.C(1)^0) --- filename : l-lpeg.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function string:totable() + return pattern:match(self) +end -if not versions then versions = { } end versions['l-lpeg'] = 1.001 +--~ for _, str in ipairs { +--~ "1234567123456712345671234567", +--~ "a\tb\tc", +--~ "aa\tbb\tcc", +--~ "aaa\tbbb\tccc", +--~ "aaaa\tbbbb\tcccc", +--~ "aaaaa\tbbbbb\tccccc", +--~ "aaaaaa\tbbbbbb\tcccccc", +--~ } do print(string.tabtospace(str)) end + +function string.tabtospace(str,tab) + -- we don't handle embedded newlines + while true do + local s = find(str,"\t") + if s then + if not tab then tab = 7 end -- only when found + local d = tab-(s-1)%tab + if d > 0 then + str = gsub(str,"\t",rep(" ",d),1) + else + str = gsub(str,"\t","",1) + end + else + break + end + end + return str +end + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-lpeg'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc --~ l-lpeg.lua : @@ -404,36 +303,40 @@ if not versions then versions = { } end versions['l-lpeg'] = 1.001 local hash = { } function lpeg.anywhere(pattern) --slightly adapted from website - return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } + return P { P(pattern) + 1 * lpeg.V(1) } end function lpeg.startswith(pattern) --slightly adapted - return lpeg.P(pattern) + return P(pattern) end ---~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 - function lpeg.splitter(pattern, action) - return (((1-lpeg.P(pattern))^1)/action+1)^0 + return (((1-P(pattern))^1)/action+1)^0 end -local crlf = lpeg.P("\r\n") -local cr = lpeg.P("\r") -local lf = lpeg.P("\n") -local space = lpeg.S(" \t\f\v") +-- variant: + +--~ local parser = lpeg.Ct(lpeg.splitat(newline)) + +local crlf = P("\r\n") +local cr = P("\r") +local lf = P("\n") +local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) local newline = crlf + cr + lf local spacing = space^0 * newline -local empty = spacing * lpeg.Cc("") -local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local empty = spacing * Cc("") +local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 -local capture = lpeg.Ct(content^0) +local capture = Ct(content^0) function string:splitlines() return capture:match(self) end +lpeg.linebyline = content -- better make a sublibrary + --~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more --~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more --~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps @@ -441,16 +344,16 @@ end local splitters_s, splitters_m = { }, { } -function lpeg.splitat(separator,single) +local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] if not splitter then - separator = lpeg.P(separator) + separator = P(separator) if single then - local other, any = lpeg.C((1 - separator)^0), lpeg.P(1) - splitter = other * (separator * lpeg.C(any^0) + "") + local other, any = C((1 - separator)^0), P(1) + splitter = other * (separator * C(any^0) + "") splitters_s[separator] = splitter else - local other = lpeg.C((1 - separator)^0) + local other = C((1 - separator)^0) splitter = other * (separator * other)^0 splitters_m[separator] = splitter end @@ -458,26 +361,43 @@ function lpeg.splitat(separator,single) return splitter end +lpeg.splitat = splitat + +local cache = { } + +function string:split(separator) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return c:match(self) +end + + +end -- of closure --- filename : l-table.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +do -- create closure to overcome 200 locals limit -if not versions then versions = { } end versions['l-table'] = 1.001 +if not modules then modules = { } end modules ['l-table'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format = string.format +local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump local getmetatable, setmetatable = getmetatable, setmetatable -local pairs, ipairs, type, next, tostring = pairs, ipairs, type, next, tostring +local type, next, tostring, ipairs = type, next, tostring, ipairs function table.strip(tab) local lst = { } for i=1,#tab do - local s = tab[i]:gsub("^%s*(.-)%s*$","%1") + local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -489,7 +409,7 @@ end local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key if kind == 3 then -- no further check @@ -516,7 +436,7 @@ end local function sortedhashkeys(tab) -- fast one local srt = { } - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key end sort(srt) @@ -526,14 +446,25 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys +function table.sortedpairs(t) + local s = sortedhashkeys(t) -- maybe just sortedkeys + local n = 0 + local function kv(s) + n = n + 1 + local k = s[n] + return k, t[k] + end + return kv, s +end + function table.append(t, list) - for _,v in pairs(list) do + for _,v in next, list do insert(t,v) end end function table.prepend(t, list) - for k,v in pairs(list) do + for k,v in next, list do insert(t,k,v) end end @@ -542,7 +473,7 @@ function table.merge(t, ...) -- first one is target t = t or {} local lst = {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do t[k] = v end end @@ -552,7 +483,7 @@ end function table.merged(...) local tmp, lst = { }, {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do tmp[k] = v end end @@ -584,13 +515,14 @@ end local function fastcopy(old) -- fast one if old then local new = { } - for k,v in pairs(old) do + for k,v in next, old do if type(v) == "table" then new[k] = fastcopy(v) -- was just table.copy else new[k] = v end end + -- optional second arg local mt = getmetatable(old) if mt then setmetatable(new,mt) @@ -607,7 +539,7 @@ local function copy(t, tables) -- taken from lua wiki, slightly adapted if not tables[t] then tables[t] = tcopy end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed + for i,v in next, t do -- brrr, what happens with sparse indexed if type(i) == "table" then if tables[i] then i = tables[i] @@ -640,7 +572,7 @@ function table.sub(t,i,j) end function table.replace(a,b) - for k,v in pairs(b) do + for k,v in next, b do a[k] = v end end @@ -662,16 +594,18 @@ end function table.tohash(t,value) local h = { } - if value == nil then value = true end - for _, v in pairs(t) do -- no ipairs here - h[v] = value + if t then + if value == nil then value = true end + for _, v in next, t do -- no ipairs here + h[v] = value + end end return h end function table.fromhash(t) local h = { } - for k, v in pairs(t) do -- no ipairs here + for k, v in next, t do -- no ipairs here if v then h[#h+1] = k end end return h @@ -695,24 +629,10 @@ local reserved = table.tohash { -- intercept a language flaw, no reserved words 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', } -local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - if hexify then - return ("[0x%04X]"):format(k) - else - return "["..k.."]" - end - elseif noquotes and not reserved[k] and k:find("^%a[%a%d%_]*$") then - return k - else - return '["'..k..'"]' - end -end - local function simple_table(t) if #t > 0 then local n = 0 - for _,v in pairs(t) do + for _,v in next, t do n = n + 1 end if n == #t then @@ -722,14 +642,14 @@ local function simple_table(t) local tv = type(v) if tv == "number" then if hexify then - tt[#tt+1] = ("0x%04X"):format(v) + tt[#tt+1] = format("0x%04X",v) else - tt[#tt+1] = tostring(v) + tt[#tt+1] = tostring(v) -- tostring not needed end elseif tv == "boolean" then tt[#tt+1] = tostring(v) elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) + tt[#tt+1] = format("%q",v) else tt = nil break @@ -741,51 +661,72 @@ local function simple_table(t) return nil end +-- Because this is a core function of mkiv I moved some function calls +-- inline. +-- +-- twice as fast in a test: +-- +-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " if indexed then - handle(("%s{"):format(depth)) + handle(format("%s{",depth)) elseif name then - handle(("%s%s={"):format(depth,key(name))) + --~ handle(format("%s%s={",depth,key(name))) + if type(name) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end else - handle(("%s{"):format(depth)) + handle(format("%s{",depth)) end end if root and next(root) then local first, last = nil, 0 -- #root cannot be trusted here if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (we need to quit at nil) + -- NOT: for k=1,#root do (we need to quit at nil) + for k,v in ipairs(root) do -- can we use next? if not first then first = k end last = last + 1 end end - --~ for _,k in pairs(sortedkeys(root)) do -- 1% faster: local sk = sortedkeys(root) for i=1,#sk do local k = sk[i] local v = root[k] + --~ if v == root then + -- circular + --~ else local t = type(v) if compact and first and type(k) == "number" and k >= first and k <= last then if t == "number" then if hexify then - handle(("%s 0x%04X,"):format(depth,v)) + handle(format("%s 0x%04X,",depth,v)) else - handle(("%s %s,"):format(depth,v)) + handle(format("%s %s,",depth,v)) end elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) + if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(format("%s %s,",depth,v)) else - handle(("%s %q,"):format(depth,v)) + handle(format("%s %q,",depth,v)) end elseif t == "table" then if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then + handle(format("%s {},",depth)) + elseif inline then -- and #t > 0 local st = simple_table(v) if st then - handle(("%s { %s },"):format(depth,concat(st,", "))) + handle(format("%s { %s },",depth,concat(st,", "))) else do_serialize(v,k,depth,level+1,true) end @@ -793,39 +734,102 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) + handle(format("%s %s,",depth,tostring(v))) elseif t == "function" then if functions then - handle(('%s loadstring(%q),'):format(depth,v:dump())) + handle(format('%s loadstring(%q),',depth,dump(v))) else - handle(('%s "function",'):format(depth)) + handle(format('%s "function",',depth)) end else - handle(("%s %q,"):format(depth,tostring(v))) + handle(format("%s %q,",depth,tostring(v))) end elseif k == "__p__" then -- parent if false then - handle(("%s __p__=nil,"):format(depth)) + handle(format("%s __p__=nil,",depth)) end elseif t == "number" then - if hexify then - handle(("%s %s=0x%04X,"):format(depth,key(k),v)) + --~ if hexify then + --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) + --~ else + --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ end + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end else - handle(("%s %s=%s,"):format(depth,key(k),v)) + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end end elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + --~ handle(format("%s %s=%s,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end else - handle(("%s %s=%q,"):format(depth,key(k),v)) + --~ handle(format("%s %s=%q,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) + else + handle(format("%s [%s]=%q,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end end elseif t == "table" then if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) + --~ handle(format("%s %s={},",depth,key(k))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end elseif inline then local st = simple_table(v) if st then - handle(("%s %s={ %s },"):format(depth,key(k),concat(st,", "))) + --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) + end else do_serialize(v,k,depth,level+1) end @@ -833,24 +837,58 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1) end elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%s,",depth,k,tostring(v))) + end elseif t == "function" then if functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),v:dump())) - else - handle(('%s %s="function",'):format(depth,key(k))) + --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) + else + handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) + else + handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) + end end else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%q,",depth,k,tostring(v))) + end end + --~ end end end if level > 0 then - handle(("%s},"):format(depth)) + handle(format("%s},",depth)) end end +-- replacing handle by a direct t[#t+1] = ... (plus test) is not much +-- faster (0.03 on 1.00 for zapfino.tma) + local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) noquotes = _noquotes hexify = _hexify @@ -868,7 +906,7 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) end elseif tname == "number" then if hexify then - handle(("[0x%04X]={"):format(name)) + handle(format("[0x%04X]={",name)) else handle("[" .. name .. "]={") end @@ -1019,14 +1057,18 @@ function table.insert_after_value(t,value,str) end end -function table.are_equal(a,b,n,m) +local function are_equal(a,b,n,m) -- indexed if #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] - if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then - -- continue + if ai==bi then + -- same + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end else return false end @@ -1037,9 +1079,30 @@ function table.are_equal(a,b,n,m) end end +local function identical(a,b) -- assumes same structure + for ka, va in next, a do + local vb = b[k] + if va == vb then + -- same + elseif type(va) == "table" and type(vb) == "table" then + if not identical(va,vb) then + return false + end + else + return false + end + end + return true +end + +table.are_equal = are_equal +table.identical = identical + +-- maybe also make a combined one + function table.compact(t) if t then - for k,v in pairs(t) do + for k,v in next, t do if not next(v) then t[k] = nil end @@ -1068,7 +1131,7 @@ end function table.swapped(t) local s = { } - for k, v in pairs(t) do + for k, v in next, t do s[v] = k end return s @@ -1090,14 +1153,14 @@ end function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = ("0x%04X"):format(t[i]) end + for i=1,#t do tt[i] = format("0x%04X",t[i]) end return concat(tt,seperator or " ") end function table.reverse_hash(h) local r = { } - for k,v in pairs(h) do - r[v] = (k:gsub(" ","")):lower() + for k,v in next, h do + r[v] = lower(gsub(k," ","")) end return r end @@ -1112,14 +1175,36 @@ function table.reverse(t) return tt end +--~ function table.keys(t) +--~ local k = { } +--~ for k,_ in next, t do +--~ k[#k+1] = k +--~ end +--~ return k +--~ end + +--~ function table.keys_as_string(t) +--~ local k = { } +--~ for k,_ in next, t do +--~ k[#k+1] = k +--~ end +--~ return concat(k,"") +--~ end --- filename : l-io.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-io'] = 1.001 +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-io'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local byte = string.byte if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1127,8 +1212,8 @@ else io.fileseparator, io.pathseparator = "/" , ":" end -function io.loaddata(filename) - local f = io.open(filename,'rb') +function io.loaddata(filename,textmode) + local f = io.open(filename,(textmode and 'r') or 'rb') if f then local data = f:read('*all') -- garbagecollector.check(data) @@ -1186,146 +1271,83 @@ function io.noflines(f) return n end -do - - local sb = string.byte - - local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end - } - - function io.characters(f,n) - if f then - return nextchar[n or 1], f - else - return nil, nil - end +local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a end +} +function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end end -do - - local sb = string.byte - ---~ local nextbyte = { ---~ [4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(a), sb(b), sb(c), sb(d) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end, ---~ [2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(a), sb(b) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [1] = function (f) ---~ local a = f:read(1) ---~ if a then ---~ return sb(a) ---~ else ---~ return nil ---~ end ---~ end, ---~ [-2] = function (f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(b), sb(a) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(d), sb(c), sb(b), sb(a) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end ---~ } - - local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(a), sb(b), sb(c), sb(d) - else - return nil, nil, nil, nil - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return sb(a), sb(b) - else - return nil, nil - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return sb(a) - else - return nil - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return sb(b), sb(a) - else - return nil, nil - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(d), sb(c), sb(b), sb(a) - else - return nil, nil, nil, nil - end +local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(a), byte(b), byte(c), byte(d) + else + return nil, nil, nil, nil end - } - - function io.bytes(f,n) - if f then - return nextbyte[n or 1], f + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return byte(a), byte(b) else return nil, nil end + end, + [1] = function (f) + local a = f:read(1) + if a then + return byte(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return byte(b), byte(a) + else + return nil, nil + end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(d), byte(c), byte(b), byte(a) + else + return nil, nil, nil, nil + end end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end end function io.ask(question,default,options) @@ -1361,15 +1383,21 @@ function io.ask(question,default,options) end --- filename : l-number.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-number'] = 1.001 +do -- create closure to overcome 200 locals limit -if not number then number = { } end +if not modules then modules = { } end modules ['l-number'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format = string.format + +number = number or { } -- a,b,c,d,e,f = number.toset(100101) @@ -1377,8 +1405,6 @@ function number.toset(n) return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end -local format = string.format - function number.toevenhex(n) local s = format("%X",n) if #s % 2 == 0 then @@ -1399,72 +1425,72 @@ end -- -- of course dedicated "(.)(.)(.)(.)" matches are even faster -do - local one = lpeg.C(1-lpeg.S(''))^1 +local one = lpeg.C(1-lpeg.S(''))^1 - function number.toset(n) - return one:match(tostring(n)) - end +function number.toset(n) + return one:match(tostring(n)) end --- filename : l-set.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-set'] = 1.001 +end -- of closure -if not set then set = { } end +do -- create closure to overcome 200 locals limit -do +if not modules then modules = { } end modules ['l-set'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - local nums = { } - local tabs = { } - local concat = table.concat +set = set or { } - set.create = table.tohash +local nums = { } +local tabs = { } +local concat = table.concat - function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in pairs(t) do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - if not nums[s] then - tabs[#tabs+1] = t - nums[s] = #tabs +set.create = table.tohash + +function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k end - return nums[s] - else - return 0 end - end - - function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs end + return nums[s] + else + return 0 end +end - function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] - end +function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } end +end +function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end end --~ local c = set.create{'aap','noot','mies'} @@ -1481,16 +1507,19 @@ end --- filename : l-os.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure +do -- create closure to overcome 200 locals limit ---~ print(table.serialize(os.uname())) +if not modules then modules = { } end modules ['l-os'] = { + version = 1.001, + comment = "companion to luat-lub.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-os'] = 1.001 +local find = string.find function os.resultof(command) return io.popen(command,"r"):read("*all") @@ -1503,7 +1532,7 @@ if not os.spawn then os.spawn = os.execute end --~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then + if find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" else io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" @@ -1541,11 +1570,10 @@ end os.gettimeofday = os.gettimeofday or os.clock -do - local startuptime = os.gettimeofday() - function os.runtime() - return os.gettimeofday() - startuptime - end +local startuptime = os.gettimeofday() + +function os.runtime() + return os.gettimeofday() - startuptime end --~ print(os.gettimeofday()-os.time()) @@ -1554,47 +1582,92 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) +os.arch = os.arch or function() + local a = os.resultof("uname -m") or "linux" + os.arch = function() + return a + end + return a +end --- filename : l-md5.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-md5'] = 1.001 - -if md5 then do +local platform - local function convert(str,fmt) - return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) +function os.currentplatform(name,default) + if not platform then + local name = os.name or os.platform or name -- os.name is built in, os.platform is mine + if not name then + platform = default or "linux" + elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then + if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then + platform = "mswin-64" + else + platform = "mswin" + end + else + local architecture = os.arch() + if name == "linux" then + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + elseif name == "macosx" then + if find(architecture,"i386") then + platform = "osx-intel" + else + platform = "osx-ppc" + end + elseif name == "sunos" then + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + elseif name == "freebsd" then + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + else + platform = default or name + end + end + function os.currentplatform() + return platform + end end + return platform +end - if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end - if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end - if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end -end end +end -- of closure +do -- create closure to overcome 200 locals limit --- filename : l-file.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +if not modules then modules = { } end modules ['l-file'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-file'] = 1.001 +-- needs a cleanup -if not file then file = { } end +file = file or { } local concat = table.concat +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub function file.removesuffix(filename) - return (filename:gsub("%.[%a%d]+$","")) + return (gsub(filename,"%.[%a%d]+$","")) end -file.stripsuffix = file.removesuffix - function file.addsuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then + if not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1602,23 +1675,23 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - return (filename:gsub("%.[%a%d]+$","")) .. "." .. suffix + return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end function file.dirname(name) - return name:match("^(.+)[/\\].-$") or "" + return match(name,"^(.+)[/\\].-$") or "" end function file.basename(name) - return name:match("^.+[/\\](.-)$") or name + return match(name,"^.+[/\\](.-)$") or name end function file.nameonly(name) - return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) + return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end function file.extname(name) - return name:match("^.+%.([^/\\]-)$") or "" + return match(name,"^.+%.([^/\\]-)$") or "" end file.suffix = file.extname @@ -1631,66 +1704,47 @@ file.suffix = file.extname function file.join(...) local pth = concat({...},"/") - pth = pth:gsub("\\","/") - local a, b = pth:match("^(.*://)(.*)$") + pth = gsub(pth,"\\","/") + local a, b = match(pth,"^(.*://)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - a, b = pth:match("^(//)(.*)$") + a, b = match(pth,"^(//)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - return (pth:gsub("//+","/")) + return (gsub(pth,"//+","/")) end -function file.is_writable(name) - local f = io.open(name, 'w') - if f then - f:close() - return true - else - return false - end -end - -function file.is_readable(name) - local f = io.open(name,'r') - if f then - f:close() +function file.iswritable(name) + local a = lfs.attributes(name) + if a and a.permissions:sub(2,2) == "w" then return true else - return false + name = file.dirname(name) or "." + if name == "" then name = "." end + a = lfs.attributes(name) + return a and a.permissions:sub(2,2) == "w" end end -function file.iswritable(name) - local a = lfs.attributes(name) - return a and a.permissions:sub(2,2) == "w" -end - function file.isreadable(name) local a = lfs.attributes(name) return a and a.permissions:sub(1,1) == "r" end ---~ function file.split_path(str) ---~ if str:find(';') then ---~ return str:splitchr(";") ---~ else ---~ return str:splitchr(io.pathseparator) ---~ end ---~ end +file.is_readable = file.isreadable +file.is_writable = file.iswritable -- todo: lpeg function file.split_path(str) local t = { } - str = str:gsub("\\", "/") - str = str:gsub("(%a):([;/])", "%1\001%2") - for name in str:gmatch("([^;:]+)") do + str = gsub(str,"\\", "/") + str = gsub(str,"(%a):([;/])", "%1\001%2") + for name in gmatch(str,"([^;:]+)") do if name ~= "" then - name = name:gsub("\001",":") - t[#t+1] = name + t[#t+1] = gsub(name,"\001",":") end end return t @@ -1701,15 +1755,15 @@ function file.join_path(tab) end function file.collapse_path(str) - str = str:gsub("/%./","/") + str = gsub(str,"/%./","/") local n, m = 1, 1 while n > 0 or m > 0 do - str, n = str:gsub("[^/%.]+/%.%.$","") - str, m = str:gsub("[^/%.]+/%.%./","") + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") end - str = str:gsub("([^/])/$","%1") - str = str:gsub("^%./","") - str = str:gsub("/%.$","") + str = gsub(str,"([^/])/$","%1") + str = gsub(str,"^%./","") + str = gsub(str,"/%.$","") if str == "" then str = "." end return str end @@ -1722,7 +1776,7 @@ end --~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) - return (str:gsub("[^%a%d%/%-%.\\]+","-")) + return (gsub(str,"[^%a%d%/%-%.\\]+","-")) end file.readdata = io.loaddata @@ -1752,8 +1806,6 @@ end --~ return pattern:match(name) --~ end ---~ file.stripsuffix = file.removesuffix - --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) @@ -1807,7 +1859,6 @@ end --~ end --~ local test = file.extname ---~ local test = file.stripsuffix --~ local test = file.basename --~ local test = file.dirname --~ local test = file.addsuffix @@ -1824,14 +1875,117 @@ end --~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) +-- also rewrite previous --- filename : l-url.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local letter = lpeg.R("az","AZ") + lpeg.S("_-+") +local separator = lpeg.P("://") -if not versions then versions = { } end versions['l-url'] = 1.001 -if not url then url = { } end +local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") +local rootbased = lpeg.P("/") + letter*lpeg.P(":") + +-- ./name ../name /name c: :// name/name + +function file.is_qualified_path(filename) + return qualified:match(filename) +end + +function file.is_rootbased_path(filename) + return rootbased:match(filename) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-md5'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This also provides file checksums and checkers. + +local gsub, format, byte = string.gsub, string.format, string.byte + +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end + +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +--~ if not md5.HEX then +--~ local function remap(chr) return format("%02X",byte(chr)) end +--~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.hex then +--~ local function remap(chr) return format("%02x",byte(chr)) end +--~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.dec then +--~ local function remap(chr) return format("%03i",byte(chr)) end +--~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end +--~ end + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.checksum(name) + if md5 then + local data = io.loaddata(name) + if data then + return md5.HEXsum(data) + end + end + return nil +end + +function file.loadchecksum(name) + if md5 then + local data = io.loaddata(name .. ".md5") + return data and data:gsub("%s","") + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.checksum(name) end + if checksum then + io.savedata(name .. ".md5",checksum) + return checksum + end + return nil +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-url'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local char, gmatch = string.char, string.gmatch +local tonumber, type = tonumber, type -- from the spec (on the web): -- @@ -1843,29 +1997,28 @@ if not url then url = { } end -- / \ / \ -- urn:example:animal:ferret:nose -do +url = url or { } - local function tochar(s) - return string.char(tonumber(s,16)) - end +local function tochar(s) + return char(tonumber(s,16)) +end - local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) +local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) - local hexdigit = lpeg.R("09","AF","af") - local escaped = percent * lpeg.C(hexdigit * hexdigit) / tochar +local hexdigit = lpeg.R("09","AF","af") +local plus = lpeg.P("+") +local escaped = (plus / " ") + (percent * lpeg.C(hexdigit * hexdigit) / tochar) - local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") - local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") - local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") - local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") - local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") +local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") +local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") +local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") +local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") +local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") - local parser = lpeg.Ct(scheme * authority * path * query * fragment) - - function url.split(str) - return (type(str) == "string" and parser:match(str)) or str - end +local parser = lpeg.Ct(scheme * authority * path * query * fragment) +function url.split(str) + return (type(str) == "string" and parser:match(str)) or str end function url.hashed(str) @@ -1888,7 +2041,7 @@ end function url.query(str) if type(str) == "string" then local t = { } - for k, v in str:gmatch("([^&=]*)=([^&=]*)") do + for k, v in gmatch(str,"([^&=]*)=([^&=]*)") do t[k] = v end return t @@ -1903,12 +2056,12 @@ end --~ print(url.filename("file:///etc/test.txt")) --~ print(url.filename("/oeps.txt")) --- from the spec on the web (sort of): +--~ from the spec on the web (sort of): --~ --~ function test(str) --~ print(table.serialize(url.hashed(str))) --~ end ----~ +--~ --~ test("%56pass%20words") --~ test("file:///c:/oeps.txt") --~ test("file:///c|/oeps.txt") @@ -1930,205 +2083,210 @@ end --~ test("zip:///oeps/oeps.zip?bla/bla.tex") --- filename : l-dir.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-dir'] = 1.001 +do -- create closure to overcome 200 locals limit -dir = { } +if not modules then modules = { } end modules ['l-dir'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --- optimizing for no string.find (*) does not save time +local type = type +local find, gmatch = string.find, string.gmatch -if lfs then do +dir = dir or { } - local attributes = lfs.attributes - local walkdir = lfs.dir +-- optimizing for no string.find (*) does not save time - local function glob_pattern(path,patt,recurse,action) - local ok, scanner - if path == "/" then - ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe - end - if ok and type(scanner) == "function" then - if not path:find("/$") then path = path .. '/' end - for name in scanner do - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if full:find(patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - glob_pattern(full,patt,recurse,action) +local attributes = lfs.attributes +local walkdir = lfs.dir + +local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if find(full,patt) then + action(full) end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) end end end +end - dir.glob_pattern = glob_pattern +dir.glob_pattern = glob_pattern - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V +local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) - } +local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) +} - local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) - )^0 ) - - local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for _, s in ipairs(str) do - glob(s,t) - end - return t - elseif lfs.isfile(str) then +local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) +)^0 ) + +local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t + else + local split = pattern:match(str) + if split then local t = t or { } - t[#t+1] = str + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) return t else - local split = pattern:match(str) - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = base:find("%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) - return t - else - return { } - end + return { } end end +end - dir.glob = glob +dir.glob = glob - --~ list = dir.glob("**/*.tif") - --~ list = dir.glob("/**/*.tif") - --~ list = dir.glob("./**/*.tif") - --~ list = dir.glob("oeps/**/*.tif") - --~ list = dir.glob("/oeps/**/*.tif") +--~ list = dir.glob("**/*.tif") +--~ list = dir.glob("/**/*.tif") +--~ list = dir.glob("./**/*.tif") +--~ list = dir.glob("oeps/**/*.tif") +--~ list = dir.glob("/oeps/**/*.tif") - local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func -- alas, we need this indirect way - func = function(name) return name:find(s) end - end - files = files or { } - for name in walkdir(path) do - if name:find("^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if func then - if func(name) then - files[#files+1] = path .. "/" .. name - end - else +local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return find(name,s) end + end + files = files or { } + for name in walkdir(path) do + if find(name,"^%.") then + --- skip + else + local mode = attributes(name,'mode') + if mode == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif mode == "file" then + if func then + if func(name) then files[#files+1] = path .. "/" .. name end + else + files[#files+1] = path .. "/" .. name end end end - return files end + return files +end - dir.globfiles = globfiles +dir.globfiles = globfiles - -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") - -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") - -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") - -- t = dir.glob("f:/minimal/tex/**/*") - -- print(dir.ls("f:/minimal/tex/**/*")) - -- print(dir.ls("*.tex")) +-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") +-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") +-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") +-- t = dir.glob("f:/minimal/tex/**/*") +-- print(dir.ls("f:/minimal/tex/**/*")) +-- print(dir.ls("*.tex")) - function dir.ls(pattern) - return table.concat(glob(pattern),"\n") - end +function dir.ls(pattern) + return table.concat(glob(pattern),"\n") +end - --~ mkdirs("temp") - --~ mkdirs("a/b/c") - --~ mkdirs(".","/a/b/c") - --~ mkdirs("a","b","c") +--~ mkdirs("temp") +--~ mkdirs("a/b/c") +--~ mkdirs(".","/a/b/c") +--~ mkdirs("a","b","c") - local make_indeed = true -- false +local make_indeed = true -- false - if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - local first, middle, last - local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") if first then - -- empty network path == local path + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end else - first, last = str:match("^(//)/*(.-)$") + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end + pth, drive = first .. middle, true else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = str:match("^(/*)(.-)$") - if not middle then - last = str - end + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str end end end - for s in last:gmatch("[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + for s in gmatch(last,"[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2142,79 +2300,79 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") - if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") - end - if not first then - first, last = str:match("^(//)/*(.*)$") - end - if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") - if first and not last:find("^/") then - local d = lfs.currentdir() - if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") - end - lfs.chdir(d) + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") end + lfs.chdir(d) end - if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") - end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") - if last == "" then - return first - else - return first .. "/" .. last - end end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end - else +else - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - str = str:gsub("/+","/") - if str:find("^/") then - pth = "/" - for s in str:gmatch("[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not lfs.isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in str:gmatch("[^/]+") do + end + str = str:gsub("/+","/") + if find(str,"^/") then + pth = "/" + for s in gmatch(str,"[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else pth = pth .. "/" .. s - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in gmatch(str,"[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2224,30 +2382,35 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - if not str:find("^/") then - str = lfs.currentdir() .. "/" .. str - end - str = str:gsub("//","/") - str = str:gsub("/%./","/") - return str + function dir.expand_name(str) + if not find(str,"^/") then + str = lfs.currentdir() .. "/" .. str end - + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str end - dir.makedirs = dir.mkdirs +end -end end +dir.makedirs = dir.mkdirs --- filename : l-boolean.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-boolean'] = 1.001 -if not boolean then boolean = { } end +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-boolean'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +boolean = boolean or { } + +local type, tonumber = type, tonumber function boolean.tonumber(b) if b then return 1 else return 0 end @@ -2294,24 +2457,24 @@ function boolean.falsetrue() end --- filename : l-unicode.lua --- comment : split off from luat-inp --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-unicode'] = 1.001 -if not unicode then unicode = { } end +do -- create closure to overcome 200 locals limit -local concat, utfchar, utfgsub = table.concat, unicode.utf8.char, unicode.utf8.gsub -local char, byte = string.char, string.byte +if not modules then modules = { } end modules ['l-unicode'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not garbagecollector then - garbagecollector = { - push = function() collectgarbage("stop") end, - pop = function() collectgarbage("restart") end, - } -end +utf = utf or unicode.utf8 + +local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub +local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs + +unicode = unicode or { } -- 0 EF BB BF UTF-8 -- 1 FF FE UTF-16-little-endian @@ -2332,17 +2495,17 @@ function unicode.utftype(f) -- \000 fails ! if not str then f:seek('set') return 0 - elseif str:find("^%z%z\254\255") then + elseif find(str,"^%z%z\254\255") then return 4 - elseif str:find("^\255\254%z%z") then + elseif find(str,"^\255\254%z%z") then return 3 - elseif str:find("^\254\255") then + elseif find(str,"^\254\255") then f:seek('set',2) return 2 - elseif str:find("^\255\254") then + elseif find(str,"^\255\254") then f:seek('set',2) return 1 - elseif str:find("^\239\187\191") then + elseif find(str,"^\239\187\191") then f:seek('set',3) return 0 else @@ -2352,18 +2515,17 @@ function unicode.utftype(f) -- \000 fails ! end function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg ---~ garbagecollector.push() local result, tmp, n, m, p = { }, { }, 0, 0, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = n else @@ -2371,7 +2533,7 @@ function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg p = 0 end end - for l,r in str:bytepairs() do + for l,r in bytepairs(str) do if r then if endian then n = l*256 + r @@ -2390,26 +2552,24 @@ function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg end end if #tmp > 0 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) end ---~ garbagecollector.pop() return result end function unicode.utf32_to_utf8(str, endian) ---~ garbagecollector.push() local result = { } local tmp, n, m, p = { }, 0, -1, 0 -- lf | cr | crlf / (cr:13, lf:10) local function doit() if n == 10 then if p ~= 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = 0 end elseif n == 13 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) tmp = { } p = n else @@ -2417,7 +2577,7 @@ function unicode.utf32_to_utf8(str, endian) p = 0 end end - for a,b in str:bytepairs() do + for a,b in bytepairs(str) do if a and b then if m < 0 then if endian then @@ -2439,48 +2599,55 @@ function unicode.utf32_to_utf8(str, endian) end end if #tmp > 0 then - result[#result+1] = concat(tmp,"") + result[#result+1] = concat(tmp) end ---~ garbagecollector.pop() return result end +local function little(c) + local b = byte(c) -- b = c:byte() + if b < 0x10000 then + return char(b%256,b/256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end + +local function big(c) + local b = byte(c) + if b < 0x10000 then + return char(b/256,b%256) + else + b = b - 0x10000 + local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end + function unicode.utf8_to_utf16(str,littleendian) if littleendian then - return char(255,254) .. utfgsub(str,".",function(c) - local b = byte(c) - if b < 0x10000 then - return char(b%256,b/256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1%256,b1/256,b2%256,b2/256) - end - end) + return char(255,254) .. utfgsub(str,".",little) else - return char(254,255) .. utfgsub(str,".",function(c) - local b = byte(c) - if b < 0x10000 then - return char(b/256,b%256) - else - b = b - 0x10000 - local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00 - return char(b1/256,b1%256,b2/256,b2%256) - end - end) + return char(254,255) .. utfgsub(str,".",big) end end --- filename : l-math.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-math'] = 1.001 +do -- create closure to overcome 200 locals limit -local floor = math.floor +if not modules then modules = { } end modules ['l-math'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan if not math.round then function math.round(x) @@ -2500,14 +2667,34 @@ if not math.mod then end end +local pipi = 2*math.pi/360 + +function math.sind(d) + return sin(d*pipi) +end + +function math.cosd(d) + return cos(d*pipi) +end + +function math.tand(d) + return tan(d*pipi) +end + --- filename : l-utils.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-utils'] = 1.001 +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-utils'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- hm, quite unreadable if not utils then utils = { } end if not utils.merger then utils.merger = { } end @@ -2572,24 +2759,52 @@ function utils.merger._self_swap_(data,code) end end +--~ stripper: +--~ +--~ data = string.gsub(data,"%-%-~[^\n]*\n","") +--~ data = string.gsub(data,"\n\n+","\n") + function utils.merger._self_libs_(libs,list) - local result, f = { }, nil + local result, f, frozen = { }, nil, false + result[#result+1] = "\n" if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end + local foundpath = nil for _, lib in ipairs(libs) do for _, pth in ipairs(list) do - local name = string.gsub(pth .. "/" .. lib,"\\","/") - f = io.open(name) - if f then - utils.report("merging library %s",name) - result[#result+1] = f:read("*all") - f:close() - list = { pth } -- speed up the search - break + pth = string.gsub(pth,"\\","/") -- file.clean_path + utils.report("checking library path %s",pth) + local name = pth .. "/" .. lib + if lfs.isfile(name) then + foundpath = pth + end + end + if foundpath then break end + end + if foundpath then + utils.report("using library path %s",foundpath) + local right, wrong = { }, { } + for _, lib in ipairs(libs) do + local fullname = foundpath .. "/" .. lib + if lfs.isfile(fullname) then + -- right[#right+1] = lib + utils.report("merging library %s",fullname) + result[#result+1] = "do -- create closure to overcome 200 locals limit" + result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = "end -- of closure" else - utils.report("no library %s",name) + -- wrong[#wrong+1] = lib + utils.report("no library %s",fullname) end end + if #right > 0 then + utils.report("merged libraries: %s",table.concat(right," ")) + end + if #wrong > 0 then + utils.report("skipped libraries: %s",table.concat(wrong," ")) + end + else + utils.report("no valid library path found") end return table.concat(result, "\n\n") end @@ -2643,15 +2858,257 @@ end -if not modules then modules = { } end modules ['luat-lib'] = { +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { version = 1.001, + comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +debugger = debugger or { } + +local counters = { } +local names = { } +local getinfo = debug.getinfo +local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in pairs(counters) do + if count > threshold then + local name = getname(func) + if not name:find("for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in pairs(counters) do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +trackers = trackers or { } + +local data, done = { }, { } + +local function set(what,value) + for w in gmatch(lower(what),"[^, ]+") do + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](value) + end + end + end + end +end + +local function reset() + for d, f in next, data do + for i=1,#f do + f[i](false) + end + end +end + +function trackers.register(what,...) + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(fnc,value,nesting) end + end + end +end + +function trackers.enable(what) + done = { } + set(what,true) +end + +function trackers.disable(what) + done = { } + if not what or what == "" then + trackers.reset(what) + else + set(what,false) + end +end + +function trackers.reset(what) + done = { } + reset() +end + +function trackers.list() -- pattern + local list = table.sortedkeys(data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-env'] = { + version = 1.001, comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" } --- most code already moved to the l-*.lua and other luat-*.lua files +-- A former version provided functionality for non embeded core +-- scripts i.e. runtime library loading. Given the amount of +-- Lua code we use now, this no longer makes sense. Much of this +-- evolved before bytecode arrays were available and so a lot of +-- code has disappeared already. + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + +local format = string.format + +-- precautions os.setlocale(nil,nil) -- useless feature and even dangerous in luatex @@ -2659,15 +3116,27 @@ function os.setlocale() -- no way you can mess with it end +-- dirty tricks + if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end +if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then + profiler.start("luatex-profile.log") +end + +-- environment + environment = environment or { } environment.arguments = { } environment.files = { } environment.sortedflags = nil +if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end +if not environment.version or environment.version == "" then environment.version = "unknown" end +if not environment.jobname then environment.jobname = "unknown" end + function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil @@ -2689,24 +3158,20 @@ function environment.initialize_arguments(arg) environment.ownname = environment.ownname or arg[0] or 'unknown.lua' end -function environment.showarguments() - for k,v in pairs(environment.arguments) do - print(k .. " : " .. tostring(v)) - end - if #environment.files > 0 then - print("files : " .. table.concat(environment.files, " ")) - end -end - function environment.setargument(name,value) environment.arguments[name] = value end -function environment.argument(name) -- todo: default (plus typecheck on default) +-- todo: defaults, better checks e.g on type (boolean versus string) +-- +-- tricky: too many hits when we support partials unless we add +-- a registration of arguments so from now on we have 'partial' + +function environment.argument(name,partial) local arguments, sortedflags = environment.arguments, environment.sortedflags if arguments[name] then return arguments[name] - else + elseif partial then if not sortedflags then sortedflags = { } for _,v in pairs(table.sortedkeys(arguments)) do @@ -2714,6 +3179,7 @@ function environment.argument(name) -- todo: default (plus typecheck on default) end environment.sortedflags = sortedflags end + -- example of potential clash: ^mode ^modefile for _,v in ipairs(sortedflags) do if name:find(v) then return arguments[v:sub(2,#v)] @@ -2737,43 +3203,17 @@ function environment.split_arguments(separator) -- rather special, cut-off befor return before, after end ---~ function environment.reconstruct_commandline(arg) ---~ if not arg then arg = environment.original_arguments end ---~ local result = { } ---~ for _,a in ipairs(arg) do -- ipairs 1 .. #n ---~ local kk, vv = a:match("^(%-+.-)=(.+)$") ---~ if kk and vv then ---~ if vv:find(" ") then ---~ vv = vv:unquote() ---~ vv = vv:gsub('"','\\"') ---~ result[#result+1] = kk .. "=" .. vv:quote() ---~ else ---~ a = a:unquote() ---~ a = a:gsub('"','\\"') ---~ result[#result+1] = a ---~ end ---~ elseif a:find(" ") then ---~ a = a:unquote() ---~ a = a:gsub('"','\\"') ---~ result[#result+1] = a:quote() ---~ else ---~ result[#result+1] = a ---~ end ---~ end ---~ return table.join(result," ") ---~ end - function environment.reconstruct_commandline(arg,noquote) - if not arg then arg = environment.original_arguments end + arg = arg or environment.original_arguments if noquote and #arg == 1 then local a = arg[1] - a = input.resolve(a) + a = resolvers.resolve(a) a = a:unquote() return a - elseif #arg == 1 then + elseif next(arg) then local result = { } for _,a in ipairs(arg) do -- ipairs 1 .. #n - a = input.resolve(a) + a = resolvers.resolve(a) a = a:unquote() a = a:gsub('"','\\"') -- tricky if a:find(" ") then @@ -2783,6 +3223,8 @@ function environment.reconstruct_commandline(arg,noquote) end end return table.join(result," ") + else + return "" end end @@ -2818,8 +3260,563 @@ if arg then end +-- weird place ... depends on a not yet loaded module + +function environment.texfile(filename) + return resolvers.find_file(filename,'tex') +end + +function environment.luafile(filename) + local resolved = resolvers.find_file(filename,'tex') or "" + if resolved ~= "" then + return resolved + end + resolved = resolvers.find_file(filename,'texmfscripts') or "" + if resolved ~= "" then + return resolved + end + return resolvers.find_file(filename,'luatexlibs') or "" +end + +environment.loadedluacode = loadfile -- can be overloaded + +--~ function environment.loadedluacode(name) +--~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then +--~ local chunk = loadstring(io.loaddata("texluac.luc")) +--~ os.remove("texluac.luc") +--~ return chunk +--~ else +--~ environment.loadedluacode = loadfile -- can be overloaded +--~ return loadfile(name) +--~ end +--~ end + +function environment.luafilechunk(filename) -- used for loading lua bytecode in the format + filename = file.replacesuffix(filename, "lua") + local fullname = environment.luafile(filename) + if fullname and fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading file %s", fullname) + end + return environment.loadedluacode(fullname) + else + if trace_verbose then + logs.report("fileio","unknown file %s", filename) + end + return nil + end +end + +-- the next ones can use the previous ones / combine + +function environment.loadluafile(filename, version) + local lucname, luaname, chunk + local basename = file.removesuffix(filename) + if basename == filename then + lucname, luaname = basename .. ".luc", basename .. ".lua" + else + lucname, luaname = nil, basename -- forced suffix + end + -- when not overloaded by explicit suffix we look for a luc file first + local fullname = (lucname and environment.luafile(lucname)) or "" + if fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + end + if chunk then + assert(chunk)() + if version then + -- we check of the version number of this chunk matches + local v = version -- can be nil + if modules and modules[filename] then + v = modules[filename].version -- new method + elseif versions and versions[filename] then + v = versions[filename] -- old method + end + if v == version then + return true + else + if trace_verbose then + logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) + end + environment.loadluafile(filename) + end + else + return true + end + end + fullname = (luaname and environment.luafile(luaname)) or "" + if fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + if not chunk then + if verbose then + logs.report("fileio","unknown file %s", filename) + end + else + assert(chunk)() + return true + end + end + return false +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-inf'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format = string.format + +local statusinfo, n, registered = { }, 0, { } + +statistics = statistics or { } + +statistics.enable = true +statistics.threshold = 0.05 + +-- timing functions + +local clock = os.gettimeofday or os.clock + +function statistics.hastimer(instance) + return instance and instance.starttime +end + +function statistics.starttiming(instance) + if instance then + local it = instance.timing + if not it then + it = 0 + end + if it == 0 then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end + end + instance.timing = it + 1 + end +end + +function statistics.stoptiming(instance, report) + if instance then + local it = instance.timing + if it > 1 then + instance.timing = it - 1 + else + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + instance.timing = 0 + return loadtime + end + end + end + return 0 +end + +function statistics.elapsedtime(instance) + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function statistics.elapsedindeed(instance) + local t = (instance and instance.loadtime) or 0 + return t > statistics.threshold +end + +-- general function + +function statistics.register(tag,fnc) + if statistics.enable and type(fnc) == "function" then + local rt = registered[tag] or (#statusinfo + 1) + statusinfo[rt] = { tag, fnc } + registered[tag] = rt + if #tag > n then n = #tag end + end +end + +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end + -- this code will move + local register = statistics.register + register("luatex banner", function() + return string.lower(status.banner) + end) + register("control sequences", function() + return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) + end) + register("callbacks", function() + local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 + return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) + end) + register("current memory usage", statistics.memused) + register("runtime",statistics.runtime) +-- -- + for i=1,#statusinfo do + local s = statusinfo[i] + local r = s[2]() + if r then + reporter(s[1],r,n) + end + end + statistics.enable = false + end +end + +function statistics.show_job_stat(tag,data,n) + texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data)) +end + +function statistics.memused() -- no math.round yet -) + local round = math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +end + +if statistics.runtime then + -- already loaded and set +elseif luatex and luatex.starttime then + statistics.starttime = luatex.starttime + statistics.loadtime = 0 + statistics.timing = 0 +else + statistics.starttiming(statistics) +end + +function statistics.runtime() + statistics.stoptiming(statistics) + return statistics.formatruntime(statistics.elapsedtime(statistics)) +end + +function statistics.formatruntime(runtime) + return format("%s seconds", statistics.elapsedtime(statistics)) +end + +function statistics.timed(action,report) + local timer = { } + report = report or logs.simple + statistics.starttiming(timer) + action() + statistics.stoptiming(timer) + report("total runtime: %s",statistics.elapsedtime(timer)) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- this is old code that needs an overhaul + +local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + +if texlua then + write_nl = print + write = io.write +end + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +logs = logs or { } +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4, +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', + 'start_run', 'stop_run', + 'start_page_number', 'stop_page_number', + 'report_output_pages', 'report_output_log', + 'report_tex_stat', 'report_job_stat', + 'show_open', 'show_close', 'show_load', +} + +logs.tracers = { +} + +logs.level = 0 +logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in next, logs.functions do + logs[v] = logs[method][v] or function() end + end +end + +-- tex logging + +function logs.tex.report(category,fmt,...) -- new + if fmt then + write_nl(category .. " | " .. format(fmt,...)) + else + write_nl(category .. " |") + end +end + +function logs.tex.line(fmt,...) -- new + if fmt then + write_nl(format(fmt,...)) + else + write_nl("") + end +end + +local texcount = tex and tex.count + +function logs.tex.start_page_number() + local real, user, sub = texcount[0], texcount[1], texcount[2] + if real > 0 then + if user > 0 then + if sub > 0 then + write(format("[%s.%s.%s",real,user,sub)) + else + write(format("[%s.%s",real,user)) + end + else + write(format("[%s",real)) + end + else + write("[-") + end +end + +function logs.tex.stop_page_number() + write("]") +end + +logs.tex.report_job_stat = statistics.show_job_stat + +-- xml logging + +function logs.xml.report(category,fmt,...) -- new + if fmt then + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) + else + write_nl(format("<r category='%s'/>",category)) + end +end +function logs.xml.line(fmt,...) -- new + if fmt then + write_nl(format("<r>%s</r>",format(fmt,...))) + else + write_nl("<r/>") + end +end + +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + +function logs.xml.start_run() + write_nl("<?xml version='1.0' standalone='yes'?>") + write_nl("<job>") -- xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng' + write_nl("") +end + +function logs.xml.stop_run() + write_nl("</job>") +end + +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", texcount[0], texcount[1], texcount[2])) +end + +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end + +function logs.xml.report_tex_stat(k,v) + texiowrite_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") +end + +local level = 0 + +function logs.xml.show_open(name) + level = level + 1 + texiowrite_nl(format("<f l='%s' n='%s'>",level,name)) +end + +function logs.xml.show_close(name) + texiowrite("</f> ") + level = level - 1 +end + +function logs.xml.show_load(name) + texiowrite_nl(format("<f l='%s' n='%s'/>",level+1,name)) +end + +-- + +local name, banner = 'report', 'context' + +local function report(category,fmt,...) + if fmt then + write_nl(format("%s | %s: %s",name,category,format(fmt,...))) + elseif category then + write_nl(format("%s | %s",name,category)) + else + write_nl(format("%s |",name)) + end +end + +local function simple(fmt,...) + if fmt then + write_nl(format("%s | %s",name,format(fmt,...))) + else + write_nl(format("%s |",name)) + end +end + +function logs.setprogram(_name_,_banner_,_verbose_) + name, banner = _name_, _banner_ + if _verbose_ then + trackers.enable("resolvers.verbose") + end + logs.set_method("tex") + logs.report = report -- also used in libraries + logs.simple = simple -- only used in scripts ! + if utils then + utils.report = simple + end + logs.verbose = _verbose_ +end + +function logs.setverbose(what) + if what then + trackers.enable("resolvers.verbose") + else + trackers.disable("resolvers.verbose") + end + logs.verbose = what or false +end + +function logs.extendbanner(_banner_,_verbose_) + banner = banner .. " | ".. _banner_ + if _verbose_ ~= nil then + logs.setverbose(what) + end +end + +logs.verbose = false +logs.report = logs.tex.report +logs.simple = logs.tex.report + +function logs.reportlines(str) -- todo: <lines></lines> + for line in str:gmatch("(.-)[\n\r]") do + logs.report(line) + end +end + +function logs.reportline() -- for scripts too + logs.report() +end + +logs.simpleline = logs.reportline + +function logs.help(message,option) + logs.report(banner) + logs.reportline() + logs.reportlines(message) + local moreinfo = logs.moreinfo or "" + if moreinfo ~= "" and option ~= "nomoreinfo" then + logs.reportline() + logs.reportlines(moreinfo) + end +end + +logs.set_level('error') +logs.set_method('tex') + +function logs.system(whereto,process,jobname,category,...) + for i=1,10 do + local f = io.open(whereto,"a") + if f then + f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...))) + f:close() + break + else + sleep(0.1) + end + end +end + +--~ local syslogname = "oeps.xxx" +--~ +--~ for i=1,10 do +--~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") +--~ end + -if not modules then modules = { } end modules ['luat-inp'] = { +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", @@ -2827,12 +3824,20 @@ if not modules then modules = { } end modules ['luat-inp'] = { comment = "companion to luat-lib.tex", } +-- After a few years using the code the large luat-inp.lua file +-- has been split up a bit. In the process some functionality was +-- dropped: +-- +-- * support for reading lsr files +-- * selective scanning (subtrees) +-- * some public auxiliary functions were made private +-- -- TODO: os.getenv -> os.env[] -- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this +-- additional functionality becomes available. We will split thislogs.report("fileio", -- module in components once we're done with prototyping. This is the -- first code I wrote for LuaTeX, so it needs some cleanup. Before changing -- something in this module one can best check with Taco or Hans first; there @@ -2846,82 +3851,205 @@ if not modules then modules = { } end modules ['luat-inp'] = { -- Beware, loading and saving is overloaded in luat-tmp! -if not input then input = { } end -if not input.suffixes then input.suffixes = { } end -if not input.formats then input.formats = { } end -if not input.aux then input.aux = { } end - -if not input.suffixmap then input.suffixmap = { } end - -if not input.locators then input.locators = { } end -- locate databases -if not input.hashers then input.hashers = { } end -- load databases -if not input.generators then input.generators = { } end -- generate databases -if not input.filters then input.filters = { } end -- conversion filters - -local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys - -input.locators.notfound = { nil } -input.hashers.notfound = { nil } -input.generators.notfound = { nil } - -input.cacheversion = '1.0.1' -input.banner = nil -input.verbose = false -input.debug = false -input.cnfname = 'texmf.cnf' -input.luaname = 'texmfcnf.lua' -input.lsrname = 'ls-R' -input.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' - ---~ input.luasuffix = 'tma' ---~ input.lucsuffix = 'tmc' - --- for the moment we have .local but this will disappear -input.cnfdefault = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - --- chances are low that the cnf file is in the bin path -input.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - --- we use a cleaned up list / format=any is a wildcard, as is *name - -input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } -input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } -input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } -input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } -input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } -input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } -input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } -input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' -input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } -input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } -input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } -input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } -input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } -input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } -input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } -input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } -input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } - -input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } -input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } - -input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new -input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' - -input.formats ['lua'] = 'LUAINPUTS' -- new -input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } +local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch +local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys +local next, type = next, type + +local trace_locating, trace_detail, trace_verbose = false, false, false + +trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) + +if not resolvers then + resolvers = { + suffixes = { }, + formats = { }, + dangerous = { }, + suffixmap = { }, + alternatives = { }, + locators = { }, -- locate databases + hashers = { }, -- load databases + generators = { }, -- generate databases + } +end + +local resolvers = resolvers + +resolvers.locators .notfound = { nil } +resolvers.hashers .notfound = { nil } +resolvers.generators.notfound = { nil } + +resolvers.cacheversion = '1.0.1' +resolvers.cnfname = 'texmf.cnf' +resolvers.luaname = 'texmfcnf.lua' +resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +local dummy_path_expr = "^!*unset/*$" + +local formats = resolvers.formats +local suffixes = resolvers.suffixes +local dangerous = resolvers.dangerous +local suffixmap = resolvers.suffixmap +local alternatives = resolvers.alternatives + +formats['afm'] = 'AFMFONTS' suffixes['afm'] = { 'afm' } +formats['enc'] = 'ENCFONTS' suffixes['enc'] = { 'enc' } +formats['fmt'] = 'TEXFORMATS' suffixes['fmt'] = { 'fmt' } +formats['map'] = 'TEXFONTMAPS' suffixes['map'] = { 'map' } +formats['mp'] = 'MPINPUTS' suffixes['mp'] = { 'mp' } +formats['ocp'] = 'OCPINPUTS' suffixes['ocp'] = { 'ocp' } +formats['ofm'] = 'OFMFONTS' suffixes['ofm'] = { 'ofm', 'tfm' } +formats['otf'] = 'OPENTYPEFONTS' suffixes['otf'] = { 'otf' } -- 'ttf' +formats['opl'] = 'OPLFONTS' suffixes['opl'] = { 'opl' } +formats['otp'] = 'OTPINPUTS' suffixes['otp'] = { 'otp' } +formats['ovf'] = 'OVFFONTS' suffixes['ovf'] = { 'ovf', 'vf' } +formats['ovp'] = 'OVPFONTS' suffixes['ovp'] = { 'ovp' } +formats['tex'] = 'TEXINPUTS' suffixes['tex'] = { 'tex' } +formats['tfm'] = 'TFMFONTS' suffixes['tfm'] = { 'tfm' } +formats['ttf'] = 'TTFONTS' suffixes['ttf'] = { 'ttf', 'ttc' } +formats['pfb'] = 'T1FONTS' suffixes['pfb'] = { 'pfb', 'pfa' } +formats['vf'] = 'VFFONTS' suffixes['vf'] = { 'vf' } + +formats['fea'] = 'FONTFEATURES' suffixes['fea'] = { 'fea' } +formats['cid'] = 'FONTCIDMAPS' suffixes['cid'] = { 'cid', 'cidmap' } + +formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +formats ['lua'] = 'LUAINPUTS' -- new +suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- backward compatible ones + +alternatives['map files'] = 'map' +alternatives['enc files'] = 'enc' +alternatives['cid files'] = 'cid' +alternatives['fea files'] = 'fea' +alternatives['opentype fonts'] = 'otf' +alternatives['truetype fonts'] = 'ttf' +alternatives['truetype collections'] = 'ttc' +alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +formats ['misc fonts'] = '' +suffixes['misc fonts'] = { } + +formats ['sfd'] = 'SFDFONTS' +suffixes ['sfd'] = { 'sfd' } +alternatives['subfont definition files'] = 'sfd' + +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. -- here we catch a few new thingies (todo: add these paths to context.tmf) -- -- FONTFEATURES = .;$TEXMF/fonts/fea// -- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata() -- not yet ok, no time for debugging now - local instance = input.instance +-- we always have one instance active + +resolvers.instance = resolvers.instance or nil -- the current one (slow access) +local instance = resolvers.instance or nil -- the current one (fast access) + +function resolvers.newinstance() + + -- store once, freeze and faster (once reset we can best use + -- instance.environment) maybe better have a register suffix + -- function + + for k, v in next, suffixes do + for i=1,#v do + local vi = v[i] + if vi then + suffixmap[vi] = k + end + end + end + + -- because vf searching is somewhat dangerous, we want to prevent + -- too liberal searching esp because we do a lookup on the current + -- path anyway; only tex (or any) is safe + + for k, v in next, formats do + dangerous[k] = true + end + dangerous.tex = nil + + -- the instance + + local newinstance = { + rootpath = '', + treepath = '', + progname = 'context', + engine = 'luatex', + format = '', + environment = { }, + variables = { }, + expansions = { }, + files = { }, + remap = { }, + configuration = { }, + setup = { }, + order = { }, + found = { }, + foundintrees = { }, + kpsevars = { }, + hashes = { }, + cnffiles = { }, + luafiles = { }, + lists = { }, + remember = true, + diskcache = true, + renewcache = false, + scandisk = true, + cachepath = nil, + loaderror = false, + sortdata = false, + savelists = true, + cleanuppaths = true, + allresults = false, + pattern = nil, -- lists + data = { }, -- only for loading + force_suffixes = true, + fakepaths = { }, + } + + local ne = newinstance.environment + + for k,v in next, os.env do + ne[k] = resolvers.bare_variable(v) + end + + return newinstance + +end + +function resolvers.setinstance(someinstance) + instance = someinstance + resolvers.instance = someinstance + return someinstance +end + +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) +end + +local function reset_hashes() + instance.lists = { } + instance.found = { } +end + +local function check_configuration() -- not yet ok, no time for debugging now + local ie = instance.environment local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" - local p = instance.environment[proname] - local v = instance.environment[varname] + local p, v = ie[proname], ie[varname] if not ((p and p ~= "") or (v and v ~= "")) then instance.variables[varname] = default -- or environment? end @@ -2937,207 +4065,198 @@ function input.checkconfigdata() -- not yet ok, no time for debugging now fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end --- backward compatible ones - -input.alternatives = { } - -input.alternatives['map files'] = 'map' -input.alternatives['enc files'] = 'enc' -input.alternatives['cid files'] = 'cid' -input.alternatives['fea files'] = 'fea' -input.alternatives['opentype fonts'] = 'otf' -input.alternatives['truetype fonts'] = 'ttf' -input.alternatives['truetype collections'] = 'ttc' -input.alternatives['type1 fonts'] = 'pfb' - --- obscure ones +function resolvers.bare_variable(str) -- assumes str is a string + return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2")) +end -input.formats ['misc fonts'] = '' -input.suffixes['misc fonts'] = { } +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) + end +end -input.formats ['sfd'] = 'SFDFONTS' -input.suffixes ['sfd'] = { 'sfd' } -input.alternatives['subfont definition files'] = 'sfd' +resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical pusposes we now avoid this and use a --- instance variable. - -function input.newinstance() - - local instance = { } - - instance.rootpath = '' - instance.treepath = '' - instance.progname = 'context' - instance.engine = 'luatex' - instance.format = '' - instance.environment = { } - instance.variables = { } - instance.expansions = { } - instance.files = { } - instance.remap = { } - instance.configuration = { } - instance.setup = { } - instance.order = { } - instance.found = { } - instance.foundintrees = { } - instance.kpsevars = { } - instance.hashes = { } - instance.cnffiles = { } - instance.luafiles = { } - instance.lists = { } - instance.remember = true - instance.diskcache = true - instance.renewcache = false - instance.scandisk = true - instance.cachepath = nil - instance.loaderror = false - instance.smallcache = false - instance.sortdata = false - instance.savelists = true - instance.cleanuppaths = true - instance.allresults = false - instance.pattern = nil -- lists - instance.kpseonly = false -- lists - instance.loadtime = 0 - instance.starttime = 0 - instance.stoptime = 0 - instance.validfile = function(path,name) return true end - instance.data = { } -- only for loading - instance.force_suffixes = true - instance.dummy_path_expr = "^!*unset/*$" - instance.fakepaths = { } - instance.lsrmode = false - - -- store once, freeze and faster (once reset we can best use instance.environment) - - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) - end - - -- cross referencing, delayed because we can add suffixes - - for k, v in pairs(input.suffixes) do - for _, vv in pairs(v) do - if vv then - input.suffixmap[vv] = k - end +function resolvers.osenv(key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = resolvers.bare_variable(e) end + ie[key] = value end - - return instance - + return value or "" end -input.instance = input.instance or nil - -function input.reset() - input.instance = input.newinstance() - return input.instance +function resolvers.env(key) + return instance.environment[key] or resolvers.osenv(key) end -function input.reset_hashes() - input.instance.lists = { } - input.instance.found = { } -end +-- -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +local function expand_vars(lst) -- simple vars + local variables, env = instance.variables, resolvers.env + local function resolve(a) + return variables[a] or env(a) + end + for k=1,#lst do + lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve) + end end -function input.settrace(n) - input.trace = tonumber(n or 0) - if input.trace > 0 then - input.verbose = true +local function expanded_var(var) -- simple vars + local function resolve(a) + return instance.variables[a] or resolvers.env(a) end + return (gsub(var,"%$([%a%d%_%-]+)",resolve)) end -input.log = (texio and texio.write_nl) or print - -function input.report(...) - if input.verbose then - input.log("<<"..format(...)..">>") +local function entry(entries,name) + if name and (name ~= "") then + name = gsub(name,'%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = resolvers.env(name) + if result then + instance.variables[name] = result + resolvers.expand_variables() + return instance.expansions[name] or "" + end + end end + return "" end -function input.report(...) - if input.trace > 0 then -- extra test - input.log("<<"..format(...)..">>") +local function is_entry(entries,name) + if name and name ~= "" then + name = gsub(name,'%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false end end -input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) - --- These functions can be used to test the performance, especially --- loading the database files. +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -do - local clock = os.gettimeofday or os.clock +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively - function input.starttiming(instance) - if instance then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 +local function splitpathexpr(str, t, validate) + -- no need for further optimization as it is only called a + -- few times, we can use lpeg for the sub; we could move + -- the local functions outside the body + t = t or { } + str = gsub(str,",}",",@}") + str = gsub(str,"{,","{@,") + -- str = "@" .. str .. "@" + local ok, done + local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" + end + local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" + end + local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb end end + return "{" .. concat(t,",") .. "}" end - - function input.stoptiming(instance, report) - if instance then - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - input.report("load time %0.3f",loadtime) - end - return loadtime - end + local function do_three(a,b,c) + return a .. b.. c + end + while true do + done = false + while true do + str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first) + if ok > 0 then done = true else break end end - return 0 + while true do + str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second) + if ok > 0 then done = true else break end + end + while true do + str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both) + if ok > 0 then done = true else break end + end + str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three) + if ok > 0 then done = true end + if not done then break end end - -end - -function input.elapsedtime(instance) - return format("%0.3f",(instance and instance.loadtime) or 0) -end - -function input.report_loadtime(instance) - if instance then - input.report('total load time %s', input.elapsedtime(instance)) + str = gsub(str,"[{}]", "") + str = gsub(str,"@","") + if validate then + for s in gmatch(str,"[^,]+") do + s = validate(s) + if s then t[#t+1] = s end + end + else + for s in gmatch(str,"[^,]+") do + t[#t+1] = s + end end + return t end -input.loadtime = input.elapsedtime - -function input.env(key) - return input.instance.environment[key] or input.osenv(key) -end - -function input.osenv(key) - local ie = input.instance.environment - local value = ie[key] - if value == nil then - -- local e = os.getenv(key) - local e = os.env[key] - if e == nil then - -- value = "" -- false - else - value = input.bare_variable(e) +local function expanded_path_from_list(pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for k=1,#pathlist do + if find(pathlist[k],"[{}]") then + ok = true + break end - ie[key] = value end - return value or "" + if ok then + local function validate(s) + s = file.collapse_path(s) + return s ~= "" and not find(s,dummy_path_expr) and s + end + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + else + for k=1,#pathlist do + for p in gmatch(pathlist[k],"([^,]+)") do + p = file.collapse_path(p) + if p ~= "" then newlist[#newlist+1] = p end + end + end + end + return newlist end -- we follow a rather traditional approach: @@ -3148,20 +4267,20 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -input.ownpath = input.ownpath or nil -input.ownbin = input.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -input.autoselfdir = true -- false may be handy for debugging +resolvers.ownpath = resolvers.ownpath or nil +resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +resolvers.autoselfdir = true -- false may be handy for debugging -function input.getownpath() - if not input.ownpath then - if input.autoselfdir and os.selfdir then - input.ownpath = os.selfdir +function resolvers.getownpath() + if not resolvers.ownpath then + if resolvers.autoselfdir and os.selfdir then + resolvers.ownpath = os.selfdir else - local binary = input.ownbin + local binary = resolvers.ownbin if os.platform == "windows" then binary = file.replacesuffix(binary,"exe") end - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) if lfs.isfile(b) then -- we assume that after changing to the path the currentdir function @@ -3171,162 +4290,91 @@ function input.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if input.verbose and p ~= pp then - input.report("following symlink %s to %s",p,pp) + if trace_verbose and p ~= pp then + logs.report("fileio","following symlink %s to %s",p,pp) end - input.ownpath = pp + resolvers.ownpath = pp lfs.chdir(olddir) else - if input.verbose then - input.report("unable to check path %s",p) + if trace_verbose then + logs.report("fileio","unable to check path %s",p) end - input.ownpath = p + resolvers.ownpath = p end break end end end - if not input.ownpath then input.ownpath = '.' end + if not resolvers.ownpath then resolvers.ownpath = '.' end end - return input.ownpath + return resolvers.ownpath end -function input.identify_own() - local instance = input.instance - local ownpath = input.getownpath() or lfs.currentdir() +local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } + +local function identify_own() + local ownpath = resolvers.getownpath() or lfs.currentdir() local ie = instance.environment if ownpath then - if input.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end - if input.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end - if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end + if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if resolvers.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end else - input.verbose = true - input.report("error: unable to locate ownpath") + logs.report("fileio","error: unable to locate ownpath") os.exit() end - if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end - if input.env('TEXOS') == "" then os.env['TEXOS'] = input.env('SELFAUTODIR') end - if input.env('TEXROOT') == "" then os.env['TEXROOT'] = input.env('SELFAUTOPARENT') end - if input.verbose then - for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do - input.report("variable %s set to %s",v,input.env(v) or "unknown") + if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end + if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end + if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end + if trace_verbose then + for i=1,#own_places do + local v = own_places[i] + logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") end end - function input.identify_own() end + identify_own = function() end end -function input.identify_cnf() - local instance = input.instance +function resolvers.identify_cnf() if #instance.cnffiles == 0 then -- fallback - input.identify_own() + identify_own() -- the real search - input.expand_variables() - local t = input.split_path(input.env('TEXMFCNF')) - t = input.aux.expanded_path(t) - input.aux.expand_vars(t) -- redundant + resolvers.expand_variables() + local t = resolvers.split_path(resolvers.env('TEXMFCNF')) + t = expanded_path_from_list(t) + expand_vars(t) -- redundant local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) + for i=1,#t do + local ti = t[i] + local texmfcnf = file.collapse_path(file.join(ti,filename)) if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) - end - end - end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) - end -end - -function input.load_cnf() - local instance = input.instance - local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(fname) - end - end - -- instance.cnffiles contain complete names now ! - if #instance.cnffiles == 0 then - input.report("no cnf files found (TEXMFCNF may not be set/known)") - else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = input.normalize_name(instance.rootpath) - if instance.lsrmode then - loadoldconfigdata() - elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance.cnffiles) - if instance.loaderror then - loadoldconfigdata() - input.saveoldconfig() - end - else - loadoldconfigdata() - if instance.renewcache then - input.saveoldconfig() - end - end - input.aux.collapse_cnf_data() - end - input.checkconfigdata() -end - -function input.load_lua() - local instance = input.instance - if #instance.luafiles == 0 then - -- yet harmless - else - instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = input.normalize_name(instance.rootpath) - input.loadnewconfig() - input.aux.collapse_cnf_data() - end - input.checkconfigdata() -end - -function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - local instance = input.instance - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) + list[#list+1] = texmfcnf end end end + locate(resolvers.luaname,instance.luafiles) + locate(resolvers.cnfname,instance.cnffiles) end end -function input.aux.load_cnf(fname) - local instance = input.instance - fname = input.clean_path(fname) +local function load_cnf_file(fname) + fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') local f = io.open(lname) if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_configuration(dname,lname) + resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading %s", fname) + if trace_verbose then + logs.report("fileio","loading %s", fname) + end local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -3338,17 +4386,19 @@ function input.aux.load_cnf(fname) local line, n = f:read(), 0 if line then while true do -- join lines - line, n = line:gsub("\\%s*$", "") + line, n = gsub(line,"\\%s*$", "") if n > 0 then line = line .. f:read() else break end end - if not line:find("^[%%#]") then - local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if not find(line,"^[%%#]") then + local l = gsub(line,"%s*%%.*$","") + local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$") if k and v and not data[k] then - data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + v = gsub(v,"[%%#].*",'') + data[k] = gsub(v,"~","$HOME") instance.kpsevars[k] = true end end @@ -3357,53 +4407,119 @@ function input.aux.load_cnf(fname) end end f:close() + elseif trace_verbose then + logs.report("fileio","skipping %s", fname) + end + end +end + +local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in next, c do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = resolvers.bare_variable(v) + end + end + end + end +end + +function resolvers.load_cnf() + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + load_cnf_file(fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + if trace_verbose then + logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") + end + else + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + if instance.diskcache and not instance.renewcache then + resolvers.loadoldconfig(instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + resolvers.saveoldconfig() + end else - input.report("skipping %s", fname) + loadoldconfigdata() + if instance.renewcache then + resolvers.saveoldconfig() + end end + collapse_cnf_data() end + check_configuration() +end + +function resolvers.load_lua() + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + resolvers.loadnewconfig() + collapse_cnf_data() + end + check_configuration() end -- database loading -function input.load_hash() - local instance = input.instance - input.locatelists() - if instance.lsrmode then - input.loadlists() - elseif instance.diskcache and not instance.renewcache then - input.loadfiles() +function resolvers.load_hash() + resolvers.locatelists() + if instance.diskcache and not instance.renewcache then + resolvers.loadfiles() if instance.loaderror then - input.loadlists() - input.savefiles() + resolvers.loadlists() + resolvers.savefiles() end else - input.loadlists() + resolvers.loadlists() if instance.renewcache then - input.savefiles() + resolvers.savefiles() end end end -function input.aux.append_hash(type,tag,name) - if input.trace > 0 then - input.logger("= hash append: %s",tag) +function resolvers.append_hash(type,tag,name) + if trace_locating then + logs.report("fileio","= hash append: %s",tag) end - table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) + insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(type,tag,name) - if input.trace > 0 then - input.logger("= hash prepend: %s",tag) +function resolvers.prepend_hash(type,tag,name) + if trace_locating then + logs.report("fileio","= hash prepend: %s",tag) end - table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) + insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash - local instance = input.instance --- local t = input.expanded_path_list('TEXMF') -- full expansion - local t = input.split_path(input.env('TEXMF')) - table.insert(t,1,specification) - local newspec = table.join(t,";") +function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash +-- local t = resolvers.expanded_path_list('TEXMF') -- full expansion + local t = resolvers.split_path(resolvers.env('TEXMF')) + insert(t,1,specification) + local newspec = concat(t,";") if instance.environment["TEXMF"] then instance.environment["TEXMF"] = newspec elseif instance.variables["TEXMF"] then @@ -3411,182 +4527,142 @@ function input.aux.extend_texmf_var(specification) -- crap, we could better prep else -- weird end - input.expand_variables() - input.reset_hashes() + resolvers.expand_variables() + reset_hashes() end -- locators -function input.locatelists() - local instance = input.instance - for _, path in pairs(input.clean_path_list('TEXMF')) do - input.report("locating list of %s",path) - input.locatedatabase(input.normalize_name(path)) +function resolvers.locatelists() + for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do + if trace_verbose then + logs.report("fileio","locating list of %s",path) + end + resolvers.locatedatabase(file.collapse_path(path)) end end -function input.locatedatabase(specification) - return input.methodhandler('locators', specification) +function resolvers.locatedatabase(specification) + return resolvers.methodhandler('locators', specification) end -function input.locators.tex(specification) +function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - if input.trace > 0 then - input.logger('! tex locator found: %s',specification) + if trace_locating then + logs.report("fileio",'! tex locator found: %s',specification) end - input.aux.append_hash('file',specification,filename) - elseif input.trace > 0 then - input.logger('? tex locator not found: %s',specification) + resolvers.append_hash('file',specification,filename) + elseif trace_locating then + logs.report("fileio",'? tex locator not found: %s',specification) end end -- hashers -function input.hashdatabase(tag,name) - return input.methodhandler('hashers',tag,name) +function resolvers.hashdatabase(tag,name) + return resolvers.methodhandler('hashers',tag,name) end -function input.loadfiles() - local instance = input.instance +function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then for _, hash in ipairs(instance.hashes) do - input.hashdatabase(hash.tag,hash.name) + resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(tag,name) - input.aux.load_files(tag) +function resolvers.hashers.tex(tag,name) + resolvers.load_data(tag,'files') end -- generators: -function input.loadlists() - for _, hash in ipairs(input.instance.hashes) do - input.generatedatabase(hash.tag) +function resolvers.loadlists() + for _, hash in ipairs(instance.hashes) do + resolvers.generatedatabase(hash.tag) end end -function input.generatedatabase(specification) - return input.methodhandler('generators', specification) +function resolvers.generatedatabase(specification) + return resolvers.methodhandler('generators', specification) end -local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +-- starting with . or .. etc or funny char + +local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -function input.generators.tex(specification) - local instance = input.instance +function resolvers.generators.tex(specification) local tag = specification - if not instance.lsrmode and lfs.dir then - input.report("scanning path %s",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' - else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == 'directory' then - m = m + 1 - if path then - action(path..'/'..name) - else - action(name) - end - elseif path and mode == 'file' then + if trace_verbose then + logs.report("fileio","scanning path %s",specification) + end + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local function action(path) + local full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if not weird:match(name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + if path then n = n + 1 local f = files[name] if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path end - else + else -- probably unique anyway files[name] = path - local lower = name:lower() + local lower = lower(name) if name ~= lower then files["remap:"..lower] = name r = r + 1 end end end - end - end - end - action() - input.report("%s files found on %s directories with %s uppercase remappings",n,m,r) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file %s",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then - if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table - else - fl[#fl+1] = path - end - end + elseif mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line - end + action(name) end - else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line end end - f:close() end end + action() + if trace_verbose then + logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) + end end -- savers, todo -function input.savefiles() - input.aux.save_data('files', function(k,v) - return input.instance.validfile(k,v) -- path, name - end) +function resolvers.savefiles() + resolvers.save_data('files') end -- A config (optionally) has the paths split in tables. Internally -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig() - for i,c in ipairs(input.instance) do +function resolvers.splitconfig() + for i,c in ipairs(instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -3598,23 +4674,23 @@ function input.splitconfig() end end -function input.joinconfig() - for i,c in ipairs(input.instance.order) do - for k,v in pairs(c) do +function resolvers.joinconfig() + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do -- ipairs? if type(v) == 'table' then c[k] = file.join_path(v) end end end end -function input.split_path(str) +function resolvers.split_path(str) if type(str) == 'table' then return str else return file.split_path(str) end end -function input.join_path(str) +function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) else @@ -3622,11 +4698,11 @@ function input.join_path(str) end end -function input.splitexpansions() - local ie = input.instance.expansions - for k,v in pairs(ie) do +function resolvers.splitexpansions() + local ie = instance.expansions + for k,v in next, ie do local t, h = { }, { } - for _,vv in pairs(file.split_path(v)) do + for _,vv in ipairs(file.split_path(v)) do if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -3642,27 +4718,27 @@ end -- end of split/join code -function input.saveoldconfig() - input.splitconfig() - input.aux.save_data('configuration', nil) - input.joinconfig() +function resolvers.saveoldconfig() + resolvers.splitconfig() + resolvers.save_data('configuration') + resolvers.joinconfig() end -input.configbanner = [[ +resolvers.configbanner = [[ -- This is a Luatex configuration file created by 'luatools.lua' or -- 'luatex.exe' directly. For comment, suggestions and questions you can -- contact the ConTeXt Development Team. This configuration file is -- not copyrighted. [HH & TH] ]] -function input.serialize(files) +function resolvers.serialize(files) -- This version is somewhat optimized for the kind of -- tables that we deal with, so it's much faster than -- the generic serializer. This makes sense because -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local function dump(k,v,m) + local function dump(k,v,m) -- could be moved inline if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," elseif #v == 1 then @@ -3672,12 +4748,12 @@ function input.serialize(files) end end t[#t+1] = "return {" - if input.instance.sortdata then - for _, k in pairs(sortedkeys(files)) do + if instance.sortdata then + for _, k in pairs(sortedkeys(files)) do -- ipairs local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do + for _, kk in pairs(sortedkeys(fk)) do -- ipairs t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -3686,10 +4762,10 @@ function input.serialize(files) end end else - for k, v in pairs(files) do + for k, v in next, files do if type(v) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do + for kk,vv in next, v do t[#t+1] = dump(kk,vv,"\t\t") end t[#t+1] = "\t}," @@ -3702,62 +4778,67 @@ function input.serialize(files) return concat(t,"\n") end -if not texmf then texmf = {} end -- no longer needed, at least not here - -function input.aux.save_data(dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(input.instance[dataname]) do +function resolvers.save_data(dataname, makename) -- untested without cache overload + for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - input.report("preparing %s for %s",dataname,cachename) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false + if trace_verbose then + logs.report("fileio","preparing %s for %s",dataname,cachename) + end + for k, v in next, files do + if type(v) == "table" and #v == 1 then + files[k] = v[1] end end local data = { type = dataname, root = cachename, - version = input.cacheversion, + version = resolvers.cacheversion, date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, } - local ok = io.savedata(luaname,input.serialize(data)) + local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - input.report("%s saved in %s",dataname,luaname) + if trace_verbose then + logs.report("fileio","%s saved in %s",dataname,luaname) + end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - input.report("%s compiled to %s",dataname,lucname) + if trace_verbose then + logs.report("fileio","%s compiled to %s",dataname,lucname) + end else - input.report("compiling failed for %s, deleting file %s",dataname,lucname) + if trace_verbose then + logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + end os.remove(lucname) end - else - input.report("unable to save %s in %s (access error)",dataname,luaname) + elseif trace_verbose then + logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) end end end -function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload - local instance = input.instance +function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading %s for %s from %s",dataname,pathname,filename) + if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then + if trace_verbose then + logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + end instance[dataname][pathname] = data.content else - input.report("skipping %s for %s from %s",dataname,pathname,filename) + if trace_verbose then + logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + end instance[dataname][pathname] = { } instance.loaderror = true end - else - input.report("skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_verbose then + logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) end end @@ -3770,195 +4851,139 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(dataname,pathname) - local instance = input.instance - local filename = file.join(pathname,input.luaname) - local blob = loadfile(filename) - if blob then - local data = blob() - if data then - input.report("loading configuration file %s",filename) - if true then - -- flatten to variable.progname - local t = { } - for k, v in pairs(data) do -- v = progname - if type(v) == "string" then - t[k] = v - else - for kk, vv in pairs(v) do -- vv = variable - if type(vv) == "string" then - t[vv.."."..v] = kk +function resolvers.resetconfig() + identify_own() + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function resolvers.loadnewconfig() + for _, cnf in ipairs(instance.luafiles) do + local pathname = file.dirname(cnf) + local filename = file.join(pathname,resolvers.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + if trace_verbose then + logs.report("fileio","loading configuration file %s",filename) + end + if true then + -- flatten to variable.progname + local t = { } + for k, v in next, data do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in next, v do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end end end end + instance['setup'][pathname] = t + else + instance['setup'][pathname] = data end - instance[dataname][pathname] = t else - instance[dataname][pathname] = data + if trace_verbose then + logs.report("fileio","skipping configuration file %s",filename) + end + instance['setup'][pathname] = { } + instance.loaderror = true end - else - input.report("skipping configuration file %s",filename) - instance[dataname][pathname] = { } - instance.loaderror = true + elseif trace_verbose then + logs.report("fileio","skipping configuration file %s",filename) end - else - input.report("skipping configuration file %s",filename) - end -end - -function input.aux.load_configuration(dname,lname) - input.aux.load_data(dname,'configuration',lname and file.basename(lname)) -end -function input.aux.load_files(tag) - input.aux.load_data(tag,'files') -end - -function input.resetconfig() - input.identify_own() - local instance = input.instance - instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false -end - -function input.loadnewconfig() - local instance = input.instance - for _, cnf in ipairs(instance.luafiles) do - local dname = file.dirname(cnf) - input.aux.load_texmfcnf('setup',dname) - instance.order[#instance.order+1] = instance.setup[dname] + instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end end end -function input.loadoldconfig() - local instance = input.instance +function resolvers.loadoldconfig() if not instance.renewcache then for _, cnf in ipairs(instance.cnffiles) do local dname = file.dirname(cnf) - input.aux.load_configuration(dname) + resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig() + resolvers.joinconfig() end -function input.expand_variables() - local instance = input.instance +function resolvers.expand_variables() local expansions, environment, variables = { }, instance.environment, instance.variables - local env = input.env + local env = resolvers.env instance.expansions = expansions if instance.engine ~= "" then environment['engine'] = instance.engine end if instance.progname ~= "" then environment['progname'] = instance.progname end - for k,v in pairs(environment) do - local a, b = k:match("^(%a+)%_(.*)%s*$") + for k,v in next, environment do + local a, b = match(k,"^(%a+)%_(.*)%s*$") if a and b then expansions[a..'.'..b] = v else expansions[k] = v end end - for k,v in pairs(environment) do -- move environment to expansions + for k,v in next, environment do -- move environment to expansions if not expansions[k] then expansions[k] = v end end - for k,v in pairs(variables) do -- move variables to expansions + for k,v in next, variables do -- move variables to expansions if not expansions[k] then expansions[k] = v end end + local busy = false + local function resolve(a) + busy = true + return expansions[a] or env(a) + end while true do - local busy = false - for k,v in pairs(expansions) do - local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) - busy = true - return expansions[a] or env(a) - end) - local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) - busy = true - return expansions[a] or env(a) - end) + busy = false + for k,v in next, expansions do + local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve) + local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve) if n > 0 or m > 0 then expansions[k]= s end end if not busy then break end end - for k,v in pairs(expansions) do - expansions[k] = v:gsub("\\", '/') + for k,v in next, expansions do + expansions[k] = gsub(v,"\\", '/') end end -function input.aux.expand_vars(lst) -- simple vars - local instance = input.instance - local variables, env = instance.variables, input.env - for k,v in pairs(lst) do - lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return variables[a] or env(a) - end) - end +function resolvers.variable(name) + return entry(instance.variables,name) end -function input.aux.expanded_var(var) -- simple vars - local instance = input.instance - return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(a) - end) +function resolvers.expansion(name) + return entry(instance.expansions,name) end -function input.aux.entry(entries,name) - if name and (name ~= "") then - local instance = input.instance - name = name:gsub('%$','') - local result = entries[name..'.'..instance.progname] or entries[name] - if result then - return result - else - result = input.env(name) - if result then - instance.variables[name] = result - input.expand_variables() - return instance.expansions[name] or "" - end - end - end - return "" -end -function input.variable(name) - return input.aux.entry(input.instance.variables,name) -end -function input.expansion(name) - return input.aux.entry(input.instance.expansions,name) +function resolvers.is_variable(name) + return is_entry(instance.variables,name) end -function input.aux.is_entry(entries,name) - if name and name ~= "" then - name = name:gsub('%$','') - return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil - else - return false - end +function resolvers.is_expansion(name) + return is_entry(instance.expansions,name) end -function input.is_variable(name) - return input.aux.is_entry(input.instance.variables,name) +function resolvers.unexpanded_path_list(str) + local pth = resolvers.variable(str) + local lst = resolvers.split_path(pth) + return expanded_path_from_list(lst) end -function input.is_expansion(name) - return input.aux.is_entry(input.instance.expansions,name) +function resolvers.unexpanded_path(str) + return file.join_path(resolvers.unexpanded_path_list(str)) end -function input.unexpanded_path_list(str) - local pth = input.variable(str) - local lst = input.split_path(pth) - return input.aux.expanded_path(lst) -end - -function input.unexpanded_path(str) - return file.join_path(input.unexpanded_path_list(str)) -end +do -- no longer needed -do local done = { } - function input.reset_extra_path() - local instance = input.instance + function resolvers.reset_extra_path() local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -3968,26 +4993,25 @@ do end end - function input.register_extra_path(paths,subpaths) - local instance = input.instance + function resolvers.register_extra_path(paths,subpaths) local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then if subpaths and subpaths ~= "" then - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = p .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end end else - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do if not done[p] then - ep[#ep+1] = input.clean_path(p) + ep[#ep+1] = resolvers.clean_path(p) done[p] = true end end @@ -3995,10 +5019,10 @@ do elseif subpaths and subpaths ~= "" then for i=1,n do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = ep[i] .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end @@ -4014,306 +5038,196 @@ do end -function input.expanded_path_list(str) - local instance = input.instance - local function made_list(list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new = { }, { } - -- honour . .. ../.. but only when at the start - for k, v in ipairs(list) do - if not done[v] then - if v:find("^[%.%/]$") then - done[v] = true - new[#new+1] = v - else - break - end - end - end - -- first the extra paths - for k, v in ipairs(ep) do - if not done[v] then +local function made_list(instance,list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k=1,#list do + local v = list[k] + if not done[v] then + if find(v,"^[%.%/]$") then done[v] = true new[#new+1] = v + else + break end end - -- next the formal paths - for k, v in ipairs(list) do - if not done[v] then - done[v] = true - new[#new+1] = v - end + end + -- first the extra paths + for k=1,#ep do + local v = ep[k] + if not done[v] then + done[v] = true + new[#new+1] = v end - return new end - end - if not str then - return ep or { } - elseif instance.savelists then - -- engine+progname hash - str = str:gsub("%$","") - if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(str))) - instance.lists[str] = input.aux.expanded_path(lst) + -- next the formal paths + for k=1,#list do + local v = list[k] + if not done[v] then + done[v] = true + new[#new+1] = v + end end - return instance.lists[str] - else - local lst = input.split_path(input.expansion(str)) - return made_list(input.aux.expanded_path(lst)) + return new end end - -function input.clean_path_list(str) - local t = input.expanded_path_list(str) +function resolvers.clean_path_list(str) + local t = resolvers.expanded_path_list(str) if t then for i=1,#t do - t[i] = file.collapse_path(input.clean_path(t[i])) + t[i] = file.collapse_path(resolvers.clean_path(t[i])) end end return t end -function input.expand_path(str) - return file.join_path(input.expanded_path_list(str)) +function resolvers.expand_path(str) + return file.join_path(resolvers.expanded_path_list(str)) +end + +function resolvers.expanded_path_list(str) + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = gsub(str,"%$","") + if not instance.lists[str] then -- cached + local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str))) + instance.lists[str] = expanded_path_from_list(lst) + end + return instance.lists[str] + else + local lst = resolvers.split_path(resolvers.expansion(str)) + return made_list(instance,expanded_path_from_list(lst)) + end end -function input.expanded_path_list_from_var(str) -- brrr - local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) +function resolvers.expanded_path_list_from_var(str) -- brrr + local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return input.expanded_path_list(str) + return resolvers.expanded_path_list(str) else - return input.expanded_path_list(tmp) + return resolvers.expanded_path_list(tmp) end end -function input.expand_path_from_var(str) - return file.join_path(input.expanded_path_list_from_var(str)) + +function resolvers.expand_path_from_var(str) + return file.join_path(resolvers.expanded_path_list_from_var(str)) end -function input.format_of_var(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.format_of_var(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.format_of_suffix(str) - return input.suffixmap[file.extname(str)] or 'tex' +function resolvers.format_of_suffix(str) + return suffixmap[file.extname(str)] or 'tex' end -function input.variable_of_format(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.variable_of_format(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.var_of_format_or_suffix(str) - local v = input.formats[str] +function resolvers.var_of_format_or_suffix(str) + local v = formats[str] if v then return v end - v = input.formats[input.alternatives[str]] + v = formats[alternatives[str]] if v then return v end - v = input.suffixmap[file.extname(str)] + v = suffixmap[file.extname(str)] if v then - return input.formats[isf] + return formats[isf] end return '' end -function input.expand_braces(str) -- output variable and brace expansion of STRING - local ori = input.variable(str) - local pth = input.aux.expanded_path(input.split_path(ori)) +function resolvers.expand_braces(str) -- output variable and brace expansion of STRING + local ori = resolvers.variable(str) + local pth = expanded_path_from_list(resolvers.split_path(ori)) return file.join_path(pth) end --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} - --- this one is better and faster, but it took me a while to realize --- that this kind of replacement is cleaner than messy parsing and --- fuzzy concatenating we can probably gain a bit with selectively --- applying lpeg, but experiments with lpeg parsing this proved not to --- work that well; the parsing is ok, but dealing with the resulting --- table is a pain because we need to work inside-out recursively - -function input.aux.splitpathexpr(str, t, validate) - -- no need for optimization, only called a few times, we can use lpeg for the sub - t = t or { } - str = str:gsub(",}",",@}") - str = str:gsub("{,","{@,") - -- str = "@" .. str .. "@" - while true do - local done = false - while true do - local ok = false - str = str:gsub("([^{},]+){([^{}]+)}", function(a,b) - local t = { } - for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b) - local t = { } - for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b) - local t = { } - for sa in a:gmatch("[^,]+") do - for sb in b:gmatch("[^,]+") do - t[#t+1] = sa .. sb - end - end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c) - done = true - return a .. b.. c - end) - if not done then break end - end - str = str:gsub("[{}]", "") - str = str:gsub("@","") - if validate then - for s in str:gmatch("[^,]+") do - s = validate(s) - if s then t[#t+1] = s end - end - else - for s in str:gmatch("[^,]+") do - t[#t+1] = s - end - end - return t -end - -function input.aux.expanded_path(pathlist) -- maybe not a list, just a path - local instance = input.instance - -- a previous version fed back into pathlist - local newlist, ok = { }, false - for _,v in ipairs(pathlist) do - if v:find("[{}]") then - ok = true - break - end - end - if ok then - for _, v in ipairs(pathlist) do - input.aux.splitpathexpr(v, newlist, function(s) - s = file.collapse_path(s) - return s ~= "" and not s:find(instance.dummy_path_expr) and s - end) - end - else - for _,v in ipairs(pathlist) do - for vv in string.gmatch(v..',',"(.-),") do - vv = file.collapse_path(v) - if vv ~= "" then newlist[#newlist+1] = vv end - end - end - end - return newlist -end +resolvers.isreadable = { } -input.is_readable = { } - -function input.aux.is_readable(readable, name) - if input.trace > 2 then +function resolvers.isreadable.file(name) + local readable = lfs.isfile(name) -- brrr + if trace_detail then if readable then - input.logger("+ readable: %s",name) + logs.report("fileio","+ readable: %s",name) else - input.logger("- readable: %s", name) + logs.report("fileio","- readable: %s", name) end end return readable end -function input.is_readable.file(name) - return input.aux.is_readable(lfs.isfile(name), name) -end - -input.is_readable.tex = input.is_readable.file +resolvers.isreadable.tex = resolvers.isreadable.file -- name -- name/name -function input.aux.collect_files(names) - local instance = input.instance +local function collect_files(names) local filelist = { } - for _, fname in pairs(names) do - if fname then - if input.trace > 2 then - input.logger("? blobpath asked: %s",fname) - end - local bname = file.basename(fname) - local dname = file.dirname(fname) - if dname == "" or dname:find("^%.") then - dname = false - else - dname = "/" .. dname .. "$" - end - for _, hash in ipairs(instance.hashes) do - local blobpath = hash.tag - local files = blobpath and instance.files[blobpath] - if files then - if input.trace > 2 then - input.logger('? blobpath do: %s (%s)',blobpath,bname) + for k=1,#names do + local fname = names[k] + if trace_detail then + logs.report("fileio","? blobpath asked: %s",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or find(dname,"^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + local hashes = instance.hashes + for h=1,#hashes do + local hash = hashes[h] + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if trace_detail then + logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or find(blobfile,dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result + } end - end - if blobfile then - if type(blobfile) == 'string' then - if not dname or blobfile:find(dname) then + else + for kk=1,#blobfile do + local vv = blobfile[kk] + if not dname or find(vv,dname) then filelist[#filelist+1] = { hash.type, - file.join(blobpath,blobfile,bname), -- search - input.concatinators[hash.type](blobpath,blobfile,bname) -- result + file.join(blobpath,vv,bname), -- search + resolvers.concatinators[hash.type](blobpath,vv,bname) -- result } end - else - for _, vv in pairs(blobfile) do - if not dname or vv:find(dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,vv,bname), -- search - input.concatinators[hash.type](blobpath,vv,bname) -- result - } - end - end end end - elseif input.trace > 1 then - input.logger('! blobpath no: %s (%s)',blobpath,bname) end + elseif trace_locating then + logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) end end end @@ -4324,102 +5238,95 @@ function input.aux.collect_files(names) end end -function input.suffix_of_format(str) - if input.suffixes[str] then - return input.suffixes[str][1] +function resolvers.suffix_of_format(str) + if suffixes[str] then + return suffixes[str][1] else return "" end end -function input.suffixes_of_format(str) - if input.suffixes[str] then - return input.suffixes[str] +function resolvers.suffixes_of_format(str) + if suffixes[str] then + return suffixes[str] else return {} end end -do - - -- called about 700 times for an empty doc (font initializations etc) - -- i need to weed the font files for redundant calls - - local letter = lpeg.R("az","AZ") - local separator = lpeg.P("://") - - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local rootbased = lpeg.P("/") + letter*lpeg.P(":") - - -- ./name ../name /name c: :// - function input.aux.qualified_path(filename) - return qualified:match(filename) - end - function input.aux.rootbased_path(filename) - return rootbased:match(filename) - end - - function input.normalize_name(original) - return original +function resolvers.register_in_trees(name) + if not find(name,"^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end - - input.normalize_name = file.collapse_path - end -function input.aux.register_in_trees(name) - if not name:find("^%.") then - local instance = input.instance - instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one +-- split the next one up for readability (bu this module needs a cleanup anyway) + +local function can_be_dir(name) -- can become local + local fakepaths = instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name] = 1 -- directory + else + fakepaths[name] = 2 -- no directory + end end + return (fakepaths[name] == 1) end --- split the next one up, better for jit - -function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) - local instance = input.instance - local result = { } +local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) + local result = collected or { } local stamp = nil - filename = input.normalize_name(filename) -- elsewhere - filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- elsewhere + filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - if input.trace > 0 then - input.logger('! remembered: %s',filename) + if trace_locating then + logs.report("fileio",'! remembered: %s',filename) end return instance.found[stamp] end end - if filename:find('%*') then - if input.trace > 0 then - input.logger('! wildcard: %s', filename) + if not dangerous[instance.format or "?"] then + if resolvers.isreadable.file(filename) then + if trace_detail then + logs.report("fileio",'= found directly: %s',filename) + end + instance.found[stamp] = { filename } + return { filename } end - result = input.find_wildcard_files(filename) - elseif input.aux.qualified_path(filename) then - if input.is_readable.file(filename) then - if input.trace > 0 then - input.logger('! qualified: %s', filename) + end + if find(filename,'%*') then + if trace_locating then + logs.report("fileio",'! wildcard: %s', filename) + end + result = resolvers.find_wildcard_files(filename) + elseif file.is_qualified_path(filename) then + if resolvers.isreadable.file(filename) then + if trace_locating then + logs.report("fileio",'! qualified: %s', filename) end result = { filename } else - local forcedname, ok = "", false - if file.extname(filename) == "" then + local forcedname, ok, suffix = "", false, file.extname(filename) + if suffix == "" then -- why if instance.format == "" then forcedname = filename .. ".tex" - if input.is_readable.file(forcedname) then - if input.trace > 0 then - input.logger('! no suffix, forcing standard filetype: tex') + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio",'! no suffix, forcing standard filetype: tex') end result, ok = { forcedname }, true end else - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do forcedname = filename .. "." .. s - if input.is_readable.file(forcedname) then - if input.trace > 0 then - input.logger('! no suffix, forcing format filetype: %s', s) + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio",'! no suffix, forcing format filetype: %s', s) end result, ok = { forcedname }, true break @@ -4427,8 +5334,49 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) end end end - if not ok and input.trace > 0 then - input.logger('? qualified: %s', filename) + if not ok and suffix ~= "" then + -- try to find in tree (no suffix manipulation), here we search for the + -- matching last part of the name + local basename = file.basename(filename) + local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local savedformat = instance.format + local format = savedformat or "" + if format == "" then + instance.format = resolvers.format_of_suffix(suffix) + end + if not format then + instance.format = "othertextfiles" -- kind of everything, maybe texinput is better + end + -- + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end + end + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if rr:find(pattern) then + result[#result+1], ok = rr, true + end + end + -- a real wildcard: + -- + -- if not ok then + -- local filelist = collect_files({basename}) + -- for f=1,#filelist do + -- local ff = filelist[f][3] or "" + -- if ff:find(pattern) then + -- result[#result+1], ok = ff, true + -- end + -- end + -- end + end + if not ok and trace_locating then + logs.report("fileio",'? qualified: %s', filename) end end else @@ -4445,45 +5393,47 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) if ext == "" then local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname - filetype = input.format_of_suffix(forcedname) - if input.trace > 0 then - input.logger('! forcing filetype: %s',filetype) - end + filetype = resolvers.format_of_suffix(forcedname) + if trace_locating then + logs.report("fileio",'! forcing filetype: %s',filetype) + end else - filetype = input.format_of_suffix(filename) - if input.trace > 0 then - input.logger('! using suffix based filetype: %s',filetype) + filetype = resolvers.format_of_suffix(filename) + if trace_locating then + logs.report("fileio",'! using suffix based filetype: %s',filetype) end end else if ext == "" then - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do wantedfiles[#wantedfiles+1] = filename .. "." .. s end end filetype = instance.format - if input.trace > 0 then - input.logger('! using given filetype: %s',filetype) + if trace_locating then + logs.report("fileio",'! using given filetype: %s',filetype) end end - local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(typespec) + local typespec = resolvers.variable_of_format(filetype) + local pathlist = resolvers.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard - if input.trace > 2 then - input.logger('? filename: %s',filename) - input.logger('? filetype: %s',filetype or '?') - input.logger('? wanted files: %s',concat(wantedfiles," | ")) - end - for _, fname in pairs(wantedfiles) do - if fname and input.is_readable.file(fname) then + if trace_detail then + logs.report("fileio",'? filename: %s',filename) + logs.report("fileio",'? filetype: %s',filetype or '?') + logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + end + for k=1,#wantedfiles do + local fname = wantedfiles[k] + if fname and resolvers.isreadable.file(fname) then filename, done = fname, true result[#result+1] = file.join('.',fname) break end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(wantedfiles) + local filelist = collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -4492,56 +5442,53 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) end else -- list search - local filelist = input.aux.collect_files(wantedfiles) + local filelist = collect_files(wantedfiles) local doscan, recurse - if input.trace > 2 then - input.logger('? filename: %s',filename) - -- if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end - -- if filelist then input.logger('? file list: %s',concat(filelist," | ")) end + if trace_detail then + logs.report("fileio",'? filename: %s',filename) end -- a bit messy ... esp the doscan setting here - for _, path in pairs(pathlist) do - if path:find("^!!") then doscan = false else doscan = true end - if path:find("//$") then recurse = true else recurse = false end - local pathname = path:gsub("^!+", '') + for k=1,#pathlist do + local path = pathlist[k] + if find(path,"^!!") then doscan = false else doscan = true end + if find(path,"//$") then recurse = true else recurse = false end + local pathname = gsub(path,"^!+", '') done = false -- using file list if filelist and not (done and not instance.allresults) and recurse then -- compare list entries with permitted pattern - pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences - pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless + pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences + pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname + pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless local expr = "^" .. pathname - -- input.debug('?',expr) - for _, fl in ipairs(filelist) do + for k=1,#filelist do + local fl = filelist[k] local f = fl[2] - if f:find(expr) then - -- input.debug('T',' '..f) - if input.trace > 2 then - input.logger('= found in hash: %s',f) + if find(f,expr) then + if trace_detail then + logs.report("fileio",'= found in hash: %s',f) end --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(f) -- for tracing used files + resolvers.register_in_trees(f) -- for tracing used files done = true if not instance.allresults then break end - else - -- input.debug('F',' '..f) end end end if not done and doscan then -- check if on disk / unchecked / does not work at all / also zips - if input.method_is_file(pathname) then -- ? - local pname = pathname:gsub("%.%*$",'') - if not pname:find("%*") then - local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(ppname) then - for _, w in pairs(wantedfiles) do + if resolvers.splitmethod(pathname).scheme == 'file' then -- ? + local pname = gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + local ppname = gsub(pname,"/+$","") + if can_be_dir(ppname) then + for k=1,#wantedfiles do + local w = wantedfiles[k] local fname = file.join(ppname,w) - if input.is_readable.file(fname) then - if input.trace > 2 then - input.logger('= found by scanning: %s',fname) + if resolvers.isreadable.file(fname) then + if trace_detail then + logs.report("fileio",'= found by scanning: %s',fname) end result[#result+1] = fname done = true @@ -4561,8 +5508,8 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) end end end - for k,v in pairs(result) do - result[k] = file.collapse_path(v) + for k=1,#result do + result[k] = file.collapse_path(result[k]) end if instance.remember then instance.found[stamp] = result @@ -4570,38 +5517,12 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) return result end -input.aux._find_file_ = input.aux.find_file -- frozen variant +if not resolvers.concatinators then resolvers.concatinators = { } end -function input.aux.find_file(filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(filename) - if #result == 0 then - local lowered = filename:lower() - if filename ~= lowered then - return input.aux._find_file_(lowered) - end - end - return result -end - -function input.aux.can_be_dir(name) - local instance = input.instance - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end - end - return (instance.fakepaths[name] == 1) -end - -if not input.concatinators then input.concatinators = { } end - -input.concatinators.tex = file.join -input.concatinators.file = input.concatinators.tex +resolvers.concatinators.tex = file.join +resolvers.concatinators.file = resolvers.concatinators.tex -function input.find_files(filename,filetype,mustexist) - local instance = input.instance +function resolvers.find_files(filename,filetype,mustexist) if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -4610,19 +5531,26 @@ function input.find_files(filename,filetype,mustexist) filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(filename,true) + local result = collect_instance_files(filename) + if #result == 0 then + local lowered = lower(filename) + if filename ~= lowered then + return collect_instance_files(lowered) + end + end instance.format = '' - return t + return result end -function input.find_file(filename,filetype,mustexist) - return (input.find_files(filename,filetype,mustexist)[1] or "") +function resolvers.find_file(filename,filetype,mustexist) + return (resolvers.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(filename) - local instance = input.instance +function resolvers.find_given_files(filename) local bname, result = file.basename(filename), { } - for k, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] local files = instance.files[hash.tag] local blist = files[bname] if not blist then @@ -4635,11 +5563,12 @@ function input.find_given_files(filename) end if blist then if type(blist) == 'string' then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or "" if not instance.allresults then break end else - for kk,vv in pairs(blist) do - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + for kk=1,#blist do + local vv = blist[kk] + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or "" if not instance.allresults then break end end end @@ -4648,61 +5577,68 @@ function input.find_given_files(filename) return result end -function input.find_given_file(filename) - return (input.find_given_files(filename)[1] or "") +function resolvers.find_given_file(filename) + return (resolvers.find_given_files(filename)[1] or "") +end + +local function doit(path,blist,bname,tag,kind,result,allresults) + local done = false + if blist and kind then + if type(blist) == 'string' then + -- make function and share code + if find(lower(blist),path) then + result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or "" + done = true + end + else + for kk=1,#blist do + local vv = blist[kk] + if find(lower(vv),path) then + result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done end -function input.find_wildcard_files(filename) -- todo: remap: - local instance = input.instance +function resolvers.find_wildcard_files(filename) -- todo: remap: local result = { } local bname, dname = file.basename(filename), file.dirname(filename) - local path = dname:gsub("^*/","") - path = path:gsub("*",".*") - path = path:gsub("-","%%-") + local path = gsub(dname,"^*/","") + path = gsub(path,"*",".*") + path = gsub(path,"-","%%-") if dname == "" then path = ".*" end local name = bname - name = name:gsub("*",".*") - name = name:gsub("-","%%-") - path = path:lower() - name = name:lower() - local function doit(blist,bname,hash,allresults) - local done = false - if blist then - if type(blist) == 'string' then - -- make function and share code - if (blist:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" - done = true - end - else - for kk,vv in pairs(blist) do - if (vv:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" - done = true - if not allresults then break end - end - end - end - end - return done - end + name = gsub(name,"*",".*") + name = gsub(name,"-","%%-") + path = lower(path) + name = lower(name) local files, allresults, done = instance.files, instance.allresults, false - if name:find("%*") then - for k, hash in ipairs(instance.hashes) do - for kk, hh in pairs(files[hash.tag]) do - if not kk:find("^remap:") then - if (kk:lower()):find(name) then - if doit(hh,kk,hash,allresults) then done = true end + if find(name,"%*") then + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + for kk, hh in next, files[hash.tag] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end end end else - for k, hash in ipairs(instance.hashes) do - if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end @@ -4711,67 +5647,49 @@ function input.find_wildcard_files(filename) -- todo: remap: return result end -function input.find_wildcard_file(filename) - return (input.find_wildcard_files(filename)[1] or "") +function resolvers.find_wildcard_file(filename) + return (resolvers.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(filename,jobname) - local instance = input.instance - if not filename then filename = 'luatex.jlg' end - local f = io.open(filename,'w') - if f then - f:write("<?xml version='1.0' standalone='yes'?>\n") - f:write("<rl:job>\n") - if jobname then - f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") - end - f:write("\t<rl:files>\n") - for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs - f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") - end - f:write("\t</rl:files>\n") - f:write("</rl:usedfiles>\n") - f:close() - end -end - -function input.automount() +function resolvers.automount() -- implemented later end -function input.load() - input.starttiming(input.instance) - input.resetconfig() - input.identify_cnf() - input.load_lua() - input.expand_variables() - input.load_cnf() - input.expand_variables() - input.load_hash() - input.automount() - input.stoptiming(input.instance) +function resolvers.load(option) + statistics.starttiming(instance) + resolvers.resetconfig() + resolvers.identify_cnf() + resolvers.load_lua() + resolvers.expand_variables() + resolvers.load_cnf() + resolvers.expand_variables() + if option ~= "nofiles" then + resolvers.load_hash() + resolvers.automount() + end + statistics.stoptiming(instance) end -function input.for_files(command, files, filetype, mustexist) +function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if input.verbose then - input.report(str) -- has already verbose + if trace_verbose then + logs.report("fileio",str) -- has already verbose else print(str) end end - if input.verbose then + if trace_verbose then report('') end - for _, file in pairs(files) do + for _, file in ipairs(files) do local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in pairs(result) do + for _,v in ipairs(result) do report(v) end end @@ -4781,19 +5699,19 @@ end -- strtab -input.var_value = input.variable -- output the value of variable $STRING. -input.expand_var = input.expansion -- output variable expansion of STRING. +resolvers.var_value = resolvers.variable -- output the value of variable $STRING. +resolvers.expand_var = resolvers.expansion -- output variable expansion of STRING. -function input.show_path(str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(input.format_of_var(str))) +function resolvers.show_path(str) -- output search path for file type NAME + return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str))) end --- input.find_file(filename) --- input.find_file(filename, filetype, mustexist) --- input.find_file(filename, mustexist) --- input.find_file(filename, filetype) +-- resolvers.find_file(filename) +-- resolvers.find_file(filename, filetype, mustexist) +-- resolvers.find_file(filename, mustexist) +-- resolvers.find_file(filename, filetype) -function input.aux.register_file(files, name, path) +function resolvers.register_file(files, name, path) if files[name] then if type(files[name]) == 'string' then files[name] = { files[name], path } @@ -4805,170 +5723,77 @@ function input.aux.register_file(files, name, path) end end -if not input.finders then input.finders = { } end -if not input.openers then input.openers = { } end -if not input.loaders then input.loaders = { } end - -input.finders.notfound = { nil } -input.openers.notfound = { nil } -input.loaders.notfound = { false, nil, 0 } - -function input.splitmethod(filename) +function resolvers.splitmethod(filename) if not filename then return { } -- safeguard elseif type(filename) == "table" then return filename -- already split - elseif not filename:find("://") then + elseif not find(filename,"://") then return { scheme="file", path = filename, original=filename } -- quick hack else return url.hashed(filename) end end -function input.method_is_file(filename) - return input.splitmethod(filename).scheme == 'file' -end - function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do + for k, v in pairs(t) do -- pairs? s[#s+1] = k .. "=" .. v end return concat(s, sep or " | ") end -function input.methodhandler(what, filename, filetype) -- ... - local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb +function resolvers.methodhandler(what, filename, filetype) -- ... + local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme - if input[what][scheme] then - if input.trace > 0 then - input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + if resolvers[what][scheme] then + if trace_locating then + logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) end - return input[what][scheme](filename,filetype) -- todo: specification + return resolvers[what][scheme](filename,filetype) -- todo: specification else - return input[what].tex(filename,filetype) -- todo: specification - end -end - --- also inside next test? - -function input.findtexfile(filename, filetype) - return input.methodhandler('finders',input.normalize_name(filename), filetype) -end -function input.opentexfile(filename) - return input.methodhandler('openers',input.normalize_name(filename)) -end - -function input.findbinfile(filename, filetype) - return input.methodhandler('finders',input.normalize_name(filename), filetype) -end -function input.openbinfile(filename) - return input.methodhandler('loaders',input.normalize_name(filename)) -end - -function input.loadbinfile(filename, filetype) - local fname = input.findbinfile(input.normalize_name(filename), filetype) - if fname and fname ~= "" then - return input.openbinfile(fname) - else - return unpack(input.loaders.notfound) - end -end - -function input.texdatablob(filename, filetype) - local ok, data, size = input.loadbinfile(filename, filetype) - return data or "" -end - -input.loadtexfile = input.texdatablob - -function input.openfile(filename) - local fullname = input.findtexfile(filename) - if fullname and (fullname ~= "") then - return input.opentexfile(fullname) - else - return nil + return resolvers[what].tex(filename,filetype) -- todo: specification end end -function input.logmode() - return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() -end - --- this is a prelude to engine/progname specific configuration files --- in which case we can omit files meant for other programs and --- packages - ---- ctx - --- maybe texinputs + font paths --- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc - -input.validators = { } -input.validators.visibility = { } - -function input.validators.visibility.default(path, name) - return true -end - -function input.validators.visibility.context(path, name) - path = path[1] or path -- some day a loop - return not ( - path:find("latex") or --- path:find("doc") or - path:find("tex4ht") or - path:find("source") or --- path:find("config") or --- path:find("metafont") or - path:find("lists$") or - name:find("%.tpm$") or - name:find("%.bak$") - ) -end - --- todo: describe which functions are public (maybe input.private. ... ) - --- beware: i need to check where we still need a / on windows: - -function input.clean_path(str) +function resolvers.clean_path(str) if str then - str = str:gsub("\\","/") - str = str:gsub("^!+","") - str = str:gsub("^~",input.homedir) + str = gsub(str,"\\","/") + str = gsub(str,"^!+","") + str = gsub(str,"^~",resolvers.homedir) return str else return nil end end -function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(name)) do - func("^"..input.clean_path(v)) +function resolvers.do_with_path(name,func) + for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? + func("^"..resolvers.clean_path(v)) end end -function input.do_with_var(name,func) - func(input.aux.expanded_var(name)) +function resolvers.do_with_var(name,func) + func(expanded_var(name)) end -function input.with_files(pattern,handle) - local instance = input.instance +function resolvers.with_files(pattern,handle) for _, hash in ipairs(instance.hashes) do local blobpath = hash.tag local blobtype = hash.type if blobpath then local files = instance.files[blobpath] if files then - for k,v in pairs(files) do - if k:find("^remap:") then + for k,v in next, files do + if find(k,"^remap:") then k = files[k] v = files[k] -- chained end - if k:find(pattern) then + if find(k,pattern) then if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do + for _,vv in pairs(v) do -- ipairs? handle(blobtype,blobpath,vv,k) end end @@ -4979,122 +5804,32 @@ function input.with_files(pattern,handle) end end -function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = input.clean_path(oldname) - input.report("to be replaced old script %s", oldscript) - local newscripts = input.find_files(newname) or { } - if #newscripts == 0 then - input.report("unable to locate new script") - else - for _, newscript in ipairs(newscripts) do - newscript = input.clean_path(newscript) - input.report("checking new script %s", newscript) - if oldscript == newscript then - input.report("old and new script are the same") - elseif not newscript:find(scriptpath) then - input.report("new script should come from %s",scriptpath) - elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then - input.report("invalid new script name") - else - local newdata = io.loaddata(newscript) - if newdata then - input.report("old script content replaced by new content") - io.savedata(oldscript,newdata) - break - else - input.report("unable to load new script") - end - end - end +function resolvers.locate_format(name) + local barename, fmtname = name:gsub("%.%a+$",""), "" + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" end -end - - ---~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) - --- command line resolver: - ---~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) - -do - - local resolvers = { } - - resolvers.environment = function(str) - return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") - end - resolvers.relative = function(str,n) - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end - end - return input.clean_path(str) - end - resolvers.locate = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path((fullname ~= "" and fullname) or str) + if fmtname == "" then + fmtname = resolvers.find_files(barename..".fmt")[1] or "" end - resolvers.filename = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) - end - resolvers.pathname = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) - end - - resolvers.env = resolvers.environment - resolvers.rel = resolvers.relative - resolvers.loc = resolvers.locate - resolvers.kpse = resolvers.locate - resolvers.full = resolvers.locate - resolvers.file = resolvers.filename - resolvers.path = resolvers.pathname - - local function resolve(str) - if type(str) == "table" then - for k, v in pairs(str) do - str[k] = resolve(v) or v - end - elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target) - if resolvers[method] then - return resolvers[method](target) - else - return method .. ":" .. target - end - end) + fmtname = resolvers.clean_path(fmtname) + if fmtname ~= "" then + local barename = file.removesuffix(fmtname) + local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" + if lfs.isfile(luiname) then + return barename, luiname + elseif lfs.isfile(lucname) then + return barename, lucname + elseif lfs.isfile(luaname) then + return barename, luaname end - return str end - - if os.uname then - for k, v in pairs(os.uname()) do - if not resolvers[k] then - resolvers[k] = function() return v end - end - end - end - - input.resolve = resolve - + return nil, nil end -function input.boolean_variable(str,default) - local b = input.expansion(str) +function resolvers.boolean_variable(str,default) + local b = resolvers.expansion(str) if b == "" then return default else @@ -5103,165 +5838,20 @@ function input.boolean_variable(str,default) end end +texconfig.kpse_init = false -if not modules then modules = { } end modules ['luat-log'] = { - version = 1.001, - comment = "companion to luat-lib.tex", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} +kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } ) ---[[ldx-- -<p>This is a prelude to a more extensive logging module. For the sake -of parsing log files, in addition to the standard logging we will -provide an <l n='xml'/> structured file. Actually, any logging that -is hooked into callbacks will be \XML\ by default.</p> ---ldx]]-- - --- input.logger -> special tracing, driven by log level (only input) --- input.report -> goes to terminal, depends on verbose, has banner --- logs.report -> module specific tracing and reporting, no banner but class - - -input = input or { } -logs = logs or { } - ---[[ldx-- -<p>This looks pretty ugly but we need to speed things up a bit.</p> ---ldx]]-- - -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4 -} - -logs.functions = { - 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' -} +-- for a while -logs.callbacks = { - 'start_page_number', - 'stop_page_number', - 'report_output_pages', - 'report_output_log' -} +input = resolvers -logs.tracers = { -} -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } +end -- of closure -logs.level = 0 - -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +do -- create closure to overcome 200 locals limit -if texlua then - write_nl = print - write = io.write -end - -function logs.xml.report(category,fmt,...) -- new - write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) -end -function logs.xml.line(fmt,...) -- new - write_nl(format("<r>%s</r>",format(fmt,...))) -end - -function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end -function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end -function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end -function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end - -function logs.tex.report(category,fmt,...) -- new - -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . - write_nl(category .. " | " .. format(fmt,...)) -end -function logs.tex.line(fmt,...) -- new - write_nl(format(fmt,...)) -end - -function logs.set_level(level) - logs.level = logs.levels[level] or level -end - -function logs.set_method(method) - for _, v in pairs(logs.functions) do - logs[v] = logs[method][v] or function() end - end - if callback and input[method] then - for _, cb in pairs(logs.callbacks) do - callback.register(cb, input[method][cb]) - end - end -end - -function logs.xml.start_page_number() - write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) -end - -function logs.xml.stop_page_number() - write("/>") - write_nl("") -end - -function logs.xml.report_output_pages(p,b) - write_nl(format("<v k='pages' v='%s'/>", p)) - write_nl(format("<v k='bytes' v='%s'/>", b)) - write_nl("") -end - -function logs.xml.report_output_log() -end - -function input.logger(...) -- assumes test for input.trace > n - if input.trace > 0 then - logs.report(...) - end -end - -function input.report(fmt,...) - if input.verbose then - logs.report(input.banner or "report",format(fmt,...)) - end -end - -function input.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do - logs.report(input.banner or "report",line) - end -end - -input.moreinfo = [[ -more information about ConTeXt and the tools that come with it can be found at: - -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] - -function input.help(banner,message) - if not input.verbose then - input.verbose = true - -- input.report(banner,"\n") - end - input.report(banner,"\n") - input.report("") - input.reportlines(message) - if input.moreinfo and input.moreinfo ~= "" then - input.report("") - input.reportlines(input.moreinfo) - end -end - -logs.set_level('error') -logs.set_method('tex') - - -if not modules then modules = { } end modules ['luat-tmp'] = { +if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5285,33 +5875,37 @@ being written at the same time is small. We also need to extend luatools with a recache feature.</p> --ldx]]-- -local format = string.format +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) caches = caches or { } -dir = dir or { } -texmf = texmf or { } caches.path = caches.path or nil caches.base = caches.base or "luatex-cache" caches.more = caches.more or "context" caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false caches.tree = false caches.paths = caches.paths or nil caches.force = false caches.defaults = { "TEXMFCACHE", "TMPDIR", "TEMPDIR", "TMP", "TEMP", "HOME", "HOMEPATH" } +function caches.cleanname(name) + return (gsub(lower(name),"[^%w%d]+","-")) +end + function caches.temp() local cachepath = nil local function check(list,isenv) if not cachepath then - for _, v in ipairs(list) do + for k=1,#list do + local v = list[k] cachepath = (isenv and (os.env[v] or "")) or v or "" if cachepath == "" then -- next else - cachepath = input.clean_path(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + cachepath = resolvers.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" break elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then dir.mkdirs(cachepath) @@ -5324,7 +5918,7 @@ function caches.temp() end end end - check(input.clean_path_list("TEXMFCACHE") or { }) + check(resolvers.clean_path_list("TEXMFCACHE") or { }) check(caches.defaults,true) if not cachepath then print("\nfatal error: there is no valid (writable) cache path defined\n") @@ -5333,7 +5927,7 @@ function caches.temp() print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end - cachepath = input.normalize_name(cachepath) + cachepath = file.collapse_path(cachepath) function caches.temp() return cachepath end @@ -5341,24 +5935,13 @@ function caches.temp() end function caches.configpath() - return table.concat(input.instance.cnffiles,";") + return table.concat(resolvers.instance.cnffiles,";") end function caches.hashed(tree) - return md5.hex((tree:lower()):gsub("[\\\/]+","/")) + return md5.hex(gsub(lower(tree),"[\\\/]+","/")) end ---~ tracing: - ---~ function caches.hashed(tree) ---~ tree = (tree:lower()):gsub("[\\\/]+","/") ---~ local hash = md5.hex(tree) ---~ if input.verbose then -- temp message ---~ input.report("hashing %s => %s",tree,hash) ---~ end ---~ return hash ---~ end - function caches.treehash() local tree = caches.configpath() if not tree or tree == "" then @@ -5373,21 +5956,19 @@ function caches.setpath(...) if not caches.path then caches.path = caches.temp() end - caches.path = input.clean_path(caches.path) -- to be sure - if lfs then - caches.tree = caches.tree or caches.treehash() - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) - else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) - end + caches.path = resolvers.clean_path(caches.path) -- to be sure + caches.tree = caches.tree or caches.treehash() + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) end end if not caches.path then caches.path = '.' end - caches.path = input.clean_path(caches.path) - if lfs and not table.is_empty({...}) then + caches.path = resolvers.clean_path(caches.path) + if not table.is_empty({...}) then local pth = dir.mkdirs(caches.path,...) return pth end @@ -5415,9 +5996,14 @@ function caches.loaddata(path,name) end end -function caches.is_writable(filepath,filename) +--~ function caches.loaddata(path,name) +--~ local tmaname, tmcname = caches.setluanames(path,name) +--~ return dofile(tmcname) or dofile(tmaname) +--~ end + +function caches.iswritable(filepath,filename) local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.is_writable(tmaname) + return file.iswritable(tmaname) end function caches.savedata(filepath,filename,data,raw) @@ -5427,23 +6013,81 @@ function caches.savedata(filepath,filename,data,raw) reduce, simplify = false, false end if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true,false)) -- no hex + file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else - table.tofile(tmaname, data,'return',true,true,false) -- maybe not the last true + table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true end - local cleanup = input.boolean_variable("PURGECACHE", false) - local strip = input.boolean_variable("LUACSTRIP", true) + local cleanup = resolvers.boolean_variable("PURGECACHE", false) + local strip = resolvers.boolean_variable("LUACSTRIP", true) utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and input.instance then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath("formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") + texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +resolvers.finders = resolvers.finders or { } +resolvers.openers = resolvers.openers or { } +resolvers.loaders = resolvers.loaders or { } + +resolvers.finders.notfound = { nil } +resolvers.openers.notfound = { nil } +resolvers.loaders.notfound = { false, nil, 0 } + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-out'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +outputs = outputs or { } + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-con'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) +local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + --[[ldx-- <p>Once we found ourselves defining similar cache constructs several times, containers were introduced. Containers are used @@ -5457,126 +6101,141 @@ table structures without bothering about the disk cache.</p> <p>Examples of usage can be found in the font related code.</p> --ldx]]-- -containers = { } -containers.trace = false +containers = containers or { } -do -- local report +containers.usecache = true - local function report(container,tag,name) - if caches.trace or containers.trace or container.trace then - logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') - end +local function report(container,tag,name) + if trace_cache or trace_containers then + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end +end - local allocated = { } +local allocated = { } - -- tracing +-- tracing - function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil - end +function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches and caches.setpath(category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil end end +end + +function containers.is_usable(container, name) + return container.enabled and caches and caches.iswritable(container.path, name) +end - function containers.is_usable(container, name) - return container.enabled and caches.is_writable(container.path, name) +function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false end +end - function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version +function containers.read(container,name) + if container.enabled and caches and not container.storage[name] and containers.usecache then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) else - return false + container.storage[name] = nil end end - - function containers.read(container,name) - if container.enabled and not container.storage[name] then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then - report(container,"loaded",name) - else - container.storage[name] = nil - end - end - if container.storage[name] then - report(container,"reusing",name) - end - return container.storage[name] + if container.storage[name] then + report(container,"reusing",name) end + return container.storage[name] +end - function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data +function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled and caches then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared end - return data - end - - function containers.content(container,name) - return container.storage[name] + report(container,"stored",name) + container.storage[name] = data end + return data +end +function containers.content(container,name) + return container.storage[name] end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-use'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. -local save_data = input.aux.save_data -local load_data = input.aux.load_data +local save_data = resolvers.save_data +local load_data = resolvers.load_data -input.cachepath = nil -- public, for tracing -input.usecache = true -- public, for tracing +resolvers.cachepath = nil -- public, for tracing +resolvers.usecache = true -- public, for tracing -function input.aux.save_data(dataname, check) - save_data(dataname, check, function(cachename,dataname) - input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) - if input.usecache then - input.cachepath = input.cachepath or caches.definepath("trees") - return file.join(input.cachepath(),caches.hashed(cachename)) +function resolvers.save_data(dataname) + save_data(dataname, function(cachename,dataname) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) end end) end -function input.aux.load_data(pathname,dataname,filename) +function resolvers.load_data(pathname,dataname,filename) load_data(pathname,dataname,filename,function(dataname,filename) - input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) - if input.usecache then - input.cachepath = input.cachepath or caches.definepath("trees") - return file.join(input.cachepath(),caches.hashed(pathname)) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then filename = dataname @@ -5588,15 +6247,15 @@ end -- we will make a better format, maybe something xml or just text or lua -input.automounted = input.automounted or { } +resolvers.automounted = resolvers.automounted or { } -function input.automount(usecache) - local mountpaths = input.clean_path_list(input.expansion('TEXMFMOUNT')) +function resolvers.automount(usecache) + local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) if table.is_empty(mountpaths) and usecache then mountpaths = { caches.setpath("mount") } end if not table.is_empty(mountpaths) then - input.starttiming(input.instance) + statistics.starttiming(resolvers.instance) for k, root in pairs(mountpaths) do local f = io.open(root.."/url.tmi") if f then @@ -5605,1007 +6264,302 @@ function input.automount(usecache) if line:find("^[%%#%-]") then -- or %W -- skip elseif line:find("^zip://") then - input.report("mounting %s",line) - table.insert(input.automounted,line) - input.usezipfile(line) + if trace_locating then + logs.report("fileio","mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end end end f:close() end end - input.stoptiming(input.instance) + statistics.stoptiming(resolvers.instance) end end --- store info in format +-- status info -input.storage = { } -input.storage.data = { } -input.storage.min = 0 -- 500 -input.storage.max = input.storage.min - 1 -input.storage.trace = false -- true -input.storage.done = input.storage.done or 0 -input.storage.evaluators = { } --- (evaluate,message,names) +statistics.register("used config path", function() return caches.configpath() end) +statistics.register("used cache path", function() return caches.temp() or "?" end) -function input.storage.register(...) - input.storage.data[#input.storage.data+1] = { ... } -end +-- experiment (code will move) -function input.storage.evaluate(name) - input.storage.evaluators[#input.storage.evaluators+1] = name -end - -function input.storage.finalize() -- we can prepend the string with "evaluate:" - for _, t in ipairs(input.storage.evaluators) do - for i, v in pairs(t) do - if type(v) == "string" then - t[i] = loadstring(v)() - elseif type(v) == "table" then - for _, vv in pairs(v) do - if type(vv) == "string" then - t[i] = loadstring(vv)() - end - end - end - end +function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname + local enginebanner = status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname = file.replacesuffix(texname,"luv") + local luvdata = { + enginebanner = enginebanner, + formatbanner = formatbanner, + sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcefile = sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) end end -function input.storage.dump() - for name, data in ipairs(input.storage.data) do - local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] - local name, initialize, finalize, code = nil, "", "", "" - for str in target:gmatch("([^%.]+)") do - if name then - name = name .. "." .. str +function statistics.check_fmt_status(texname) + local enginebanner = status.list().banner + if enginebanner and texname then + local luvname = file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv = dofile(luvname) + if luv and luv.sourcefile then + local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + if luv.enginebanner and luv.enginebanner ~= enginebanner then + return "engine mismatch" + end + if luv.sourcehash and luv.sourcehash ~= sourcehash then + return "source mismatch" + end else - name = str + return "invalid status file" end - initialize = format("%s %s = %s or {} ", initialize, name, name) - end - if evaluate then - finalize = "input.storage.evaluate(" .. name .. ")" - end - input.storage.max = input.storage.max + 1 - if input.storage.trace then - logs.report('storage','saving %s in slot %s',message,input.storage.max) - code = - initialize .. - format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. - table.serialize(original,name) .. - finalize else - code = initialize .. table.serialize(original,name) .. finalize + return "missing status file" end - lua.bytecode[input.storage.max] = loadstring(code) end + return true end --- we also need to count at generation time (nicer for message) - -if lua.bytecode then -- from 0 upwards - local i = input.storage.min - while lua.bytecode[i] do - lua.bytecode[i]() - lua.bytecode[i] = nil - i = i + 1 - end - input.storage.done = i -end - - --- filename : luat-zip.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-zip'] = 1.001 - -local format = string.format - -if zip and input then - zip.supported = true -else - zip = { } - zip.supported = false -end - -if not zip.supported then - if not input then input = { } end -- will go away +end -- of closure - function zip.openarchive (...) return nil end -- needed ? - function zip.closenarchive (...) end -- needed ? - function input.usezipfile (...) end -- needed ? +do -- create closure to overcome 200 locals limit -else - - -- zip:///oeps.zip?name=bla/bla.tex - -- zip:///oeps.zip?tree=tex/texmf-local +if not modules then modules = { } end modules ['luat-kps'] = { + version = 1.001, + comment = "companion to luatools.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - local function validzip(str) - if not str:find("^zip://") then - return "zip:///" .. str - else - return str - end - end +--[[ldx-- +<p>This file is used when we want the input handlers to behave like +<type>kpsewhich</type>. What to do with the following:</p> - zip.archives = { } - zip.registeredfiles = { } +<typing> +{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} +$SELFAUTOLOC : /usr/tex/bin/platform +$SELFAUTODIR : /usr/tex/bin +$SELFAUTOPARENT : /usr/tex +</typing> - function zip.openarchive(name) - if not name or name == "" then - return nil - else - local arch = zip.archives[name] - if arch then - return arch - else - local full = input.find_file(name) or "" - local arch = (full ~= "" and zip.open(full)) or false - zip.archives[name] = arch - return arch - end - end - end +<p>How about just forgetting about them?</p> +--ldx]]-- - function zip.closearchive(name) - if not name or name == "" and zip.archives[name] then - zip.close(zip.archives[name]) - zip.archives[name] = nil - end - end +local suffixes = resolvers.suffixes +local formats = resolvers.formats + +suffixes['gf'] = { '<resolution>gf' } +suffixes['pk'] = { '<resolution>pk' } +suffixes['base'] = { 'base' } +suffixes['bib'] = { 'bib' } +suffixes['bst'] = { 'bst' } +suffixes['cnf'] = { 'cnf' } +suffixes['mem'] = { 'mem' } +suffixes['mf'] = { 'mf' } +suffixes['mfpool'] = { 'pool' } +suffixes['mft'] = { 'mft' } +suffixes['mppool'] = { 'pool' } +suffixes['graphic/figure'] = { 'eps', 'epsi' } +suffixes['texpool'] = { 'pool' } +suffixes['PostScript header'] = { 'pro' } +suffixes['ist'] = { 'ist' } +suffixes['web'] = { 'web', 'ch' } +suffixes['cweb'] = { 'w', 'web', 'ch' } +suffixes['cmap files'] = { 'cmap' } +suffixes['lig files'] = { 'lig' } +suffixes['bitmap font'] = { } +suffixes['MetaPost support'] = { } +suffixes['TeX system documentation'] = { } +suffixes['TeX system sources'] = { } +suffixes['dvips config'] = { } +suffixes['type42 fonts'] = { } +suffixes['web2c files'] = { } +suffixes['other text files'] = { } +suffixes['other binary files'] = { } +suffixes['opentype fonts'] = { 'otf' } + +suffixes['fmt'] = { 'fmt' } +suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +suffixes['pdftex config'] = { } +suffixes['Troff fonts'] = { } + +suffixes['ls-R'] = { } - -- zip:///texmf.zip?tree=/tex/texmf - -- zip:///texmf.zip?tree=/tex/texmf-local - -- zip:///texmf-mine.zip?tree=/tex/texmf-projects +--[[ldx-- +<p>If you wondered abou tsome of the previous mappings, how about +the next bunch:</p> +--ldx]]-- - function input.locators.zip(specification) -- where is this used? startup zips (untested) - specification = input.splitmethod(specification) - local zipfile = specification.path - local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree - if input.trace > 0 then - if zfile then - input.logger('! zip locator, found: %s',specification.original) - else - input.logger('? zip locator, not found: %s',specification.original) - end - end - end +formats['bib'] = '' +formats['bst'] = '' +formats['mft'] = '' +formats['ist'] = '' +formats['web'] = '' +formats['cweb'] = '' +formats['MetaPost support'] = '' +formats['TeX system documentation'] = '' +formats['TeX system sources'] = '' +formats['Troff fonts'] = '' +formats['dvips config'] = '' +formats['graphic/figure'] = '' +formats['ls-R'] = '' +formats['other text files'] = '' +formats['other binary files'] = '' + +formats['gf'] = '' +formats['pk'] = '' +formats['base'] = 'MFBASES' +formats['cnf'] = '' +formats['mem'] = 'MPMEMS' +formats['mf'] = 'MFINPUTS' +formats['mfpool'] = 'MFPOOL' +formats['mppool'] = 'MPPOOL' +formats['texpool'] = 'TEXPOOL' +formats['PostScript header'] = 'TEXPSHEADERS' +formats['cmap files'] = 'CMAPFONTS' +formats['type42 fonts'] = 'T42FONTS' +formats['web2c files'] = 'WEB2C' +formats['pdftex config'] = 'PDFTEXCONFIG' +formats['texmfscripts'] = 'TEXMFSCRIPTS' +formats['bitmap font'] = '' +formats['lig files'] = 'LIGFONTS' + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-aux'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - function input.hashers.zip(tag,name) - input.report("loading zip file %s as %s",name,tag) - input.usezipfile(tag .."?tree=" .. name) - end +local find = string.find - function input.concatinators.zip(tag,path,name) - if not path or path == "" then - return tag .. '?name=' .. name - else - return tag .. '?name=' .. path .. "/" .. name - end - end +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) - function input.is_readable.zip(name) - return true +function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = resolvers.clean_path(oldname) + if trace_verbose then + logs.report("fileio","to be replaced old script %s", oldscript) end - - function input.finders.zip(specification,filetype) - specification = input.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) - if zfile then - if input.trace > 0 then - input.logger('! zip finder, path: %s',specification.path) - end - local dfile = zfile:open(q.name) - if dfile then - dfile = zfile:close() - if input.trace > 0 then - input.logger('+ zip finder, name: %s',q.name) - end - return specification.original - end - elseif input.trace > 0 then - input.logger('? zip finder, path %s',specification.path) - end - end - end - if input.trace > 0 then - input.logger('- zip finder, name: %s',filename) + local newscripts = resolvers.find_files(newname) or { } + if #newscripts == 0 then + if trace_verbose then + logs.report("fileio","unable to locate new script") end - return unpack(input.finders.notfound) - end - - function input.openers.zip(specification) - local zipspecification = input.splitmethod(specification) - if zipspecification.path then - local q = url.query(zipspecification.query) - if q.name then - local zfile = zip.openarchive(zipspecification.path) - if zfile then - if input.trace > 0 then - input.logger('+ zip starter, path: %s',zipspecification.path) - end - local dfile = zfile:open(q.name) - if dfile then - input.show_open(specification) - return input.openers.text_opener(specification,dfile,'zip') - end - elseif input.trace > 0 then - input.logger('- zip starter, path %s',zipspecification.path) - end + else + for i=1,#newscripts do + local newscript = resolvers.clean_path(newscripts[i]) + if trace_verbose then + logs.report("fileio","checking new script %s", newscript) end - end - if input.trace > 0 then - input.logger('- zip opener, name: %s',filename) - end - return unpack(input.openers.notfound) - end - - function input.loaders.zip(specification) - specification = input.splitmethod(specification) - if specification.path then - local q = url.query(specification.query) - if q.name then - local zfile = zip.openarchive(specification.path) - if zfile then - if input.trace > 0 then - input.logger('+ zip starter, path: %s',specification.path) - end - local dfile = zfile:open(q.name) - if dfile then - input.show_load(filename) - if input.trace > 0 then - input.logger('+ zip loader, name: %s',filename) - end - local s = dfile:read("*all") - dfile:close() - return true, s, #s - end - elseif input.trace > 0 then - input.logger('- zip starter, path: %s',specification.path) + if oldscript == newscript then + if trace_verbose then + logs.report("fileio","old and new script are the same") end - end - end - if input.trace > 0 then - input.logger('- zip loader, name: %s',filename) - end - return unpack(input.openers.notfound) - end - - -- zip:///somefile.zip - -- zip:///somefile.zip?tree=texmf-local -> mount - - function input.usezipfile(zipname) - zipname = validzip(zipname) - if input.trace > 0 then - input.logger('! zip use, file: %s',zipname) - end - local specification = input.splitmethod(zipname) - local zipfile = specification.path - if zipfile and not zip.registeredfiles[zipname] then - local tree = url.query(specification.query).tree or "" - if input.trace > 0 then - input.logger('! zip register, file: %s',zipname) - end - local z = zip.openarchive(zipfile) - if z then - local instance = input.instance - if input.trace > 0 then - input.logger("= zipfile, registering: %s",zipname) + elseif not find(newscript,scriptpath) then + if trace_verbose then + logs.report("fileio","new script should come from %s",scriptpath) end - input.starttiming(instance) - input.aux.prepend_hash('zip',zipname,zipfile) - input.aux.extend_texmf_var(zipname) -- resets hashes too - zip.registeredfiles[zipname] = z - instance.files[zipname] = input.aux.register_zip_file(z,tree or "") - input.stoptiming(instance) - elseif input.trace > 0 then - input.logger("? zipfile, unknown: %s",zipname) - end - elseif input.trace > 0 then - input.logger('! zip register, no file: %s',zipname) - end - end - - function input.aux.register_zip_file(z,tree) - local files, filter = { }, "" - if tree == "" then - filter = "^(.+)/(.-)$" - else - filter = "^"..tree.."/(.+)/(.-)$" - end - if input.trace > 0 then - input.logger('= zip filter: %s',filter) - end - local register, n = input.aux.register_file, 0 - for i in z:files() do - local path, name = i.filename:match(filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_verbose then + logs.report("fileio","invalid new script name") end else - register(files, i.filename, '') - n = n + 1 - end - end - input.logger('= zip entries: %s',n) - return files - end - -end - - --- filename : luat-zip.lua --- comment : companion to luat-lib.tex --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['luat-tex'] = 1.001 - --- special functions that deal with io - -local format = string.format - -if texconfig and not texlua then - - input.level = input.level or 0 - - if input.logmode() == 'xml' then - function input.show_open(name) - input.level = input.level + 1 - texio.write_nl("<f l='"..input.level.."' n='"..name.."'>") - end - function input.show_close(name) - texio.write("</f> ") - input.level = input.level - 1 - end - function input.show_load(name) - texio.write_nl("<f l='"..(input.level+1).."' n='"..name.."'/>") -- level? - end - else - function input.show_open () end - function input.show_close() end - function input.show_load () end - end - - function input.finders.generic(tag,filename,filetype) - local foundname = input.find_file(filename,filetype) - if foundname and foundname ~= "" then - if input.trace > 0 then - input.logger('+ finder: %s, file: %s', tag,filename) - end - return foundname - else - if input.trace > 0 then - input.logger('- finder: %s, file: %s', tag,filename) - end - return unpack(input.finders.notfound) - end - end - - input.filters.dynamic_translator = nil - input.filters.frozen_translator = nil - input.filters.utf_translator = nil - - function input.openers.text_opener(filename,file_handle,tag) - local u = unicode.utftype(file_handle) - local t = { } - if u > 0 then - if input.trace > 0 then - input.logger('+ opener: %s (%s), file: %s',tag,unicode.utfname[u],filename) - end - local l - if u > 2 then - l = unicode.utf32_to_utf8(file_handle:read("*a"),u==4) - else - l = unicode.utf16_to_utf8(file_handle:read("*a"),u==2) - end - file_handle:close() - t = { - utftype = u, -- may go away - lines = l, - current = 0, -- line number, not really needed - handle = nil, - noflines = #l, - close = function() - if input.trace > 0 then - input.logger('= closer: %s (%s), file: %s',tag,unicode.utfname[u],filename) - end - input.show_close(filename) - t = nil - end, ---~ getline = function(n) ---~ local line = t.lines[n] ---~ if not line or line == "" then ---~ return "" ---~ else ---~ local translator = input.filters.utf_translator ---~ return (translator and translator(line)) or line ---~ end ---~ end, - reader = function(self) - self = self or t - local current, lines = self.current, self.lines - if current >= #lines then - return nil - else - current = current + 1 - self.current = current - local line = lines[current] - if line == "" then - return "" - else - local translator = input.filters.utf_translator - -- return (translator and translator(line)) or line - if translator then - return translator(line) - else - return line - end - end - end - end - } - else - if input.trace > 0 then - input.logger('+ opener: %s, file: %s',tag,filename) - end - -- todo: file;name -> freeze / eerste regel scannen -> freeze - local filters = input.filters - t = { - reader = function(self) - local line = file_handle:read() - if line == "" then - return "" - end - local translator = filters.utf_translator - if translator then - return translator(line) - end - translator = filters.dynamic_translator - if translator then - return translator(line) - end - return line - end, - close = function() - if input.trace > 0 then - input.logger('= closer: %s, file: %s',tag,filename) + local newdata = io.loaddata(newscript) + if newdata then + if trace_verbose then + logs.report("fileio","old script content replaced by new content") end - input.show_close(filename) - file_handle:close() - t = nil - end, - handle = function() - return file_handle - end, - noflines = function() - t.noflines = io.noflines(file_handle) - return t.noflines - end - } - end - return t - end - - function input.openers.generic(tag,filename) - if filename and filename ~= "" then - local f = io.open(filename,"r") - if f then - input.show_open(filename) - return input.openers.text_opener(filename,f,tag) - end - end - if input.trace > 0 then - input.logger('- opener: %s, file: %s',tag,filename) - end - return unpack(input.openers.notfound) - end - - function input.loaders.generic(tag,filename) - if filename and filename ~= "" then - local f = io.open(filename,"rb") - if f then - input.show_load(filename) - if input.trace > 0 then - input.logger('+ loader: %s, file: %s',tag,filename) - end - local s = f:read("*a") - garbagecollector.check(s) - f:close() - if s then - return true, s, #s + io.savedata(oldscript,newdata) + break + elseif trace_verbose then + logs.report("fileio","unable to load new script") end end end - if input.trace > 0 then - input.logger('- loader: %s, file: %s',tag,filename) - end - return unpack(input.loaders.notfound) - end - - function input.finders.tex(filename,filetype) - return input.finders.generic('tex',filename,filetype) end - function input.openers.tex(filename) - return input.openers.generic('tex',filename) - end - function input.loaders.tex(filename) - return input.loaders.generic('tex',filename) - end - end --- callback into the file io and related things; disabling kpse - - -if texconfig and not texlua then do - -- this is not the right place, because we refer to quite some not yet defined tables, but who cares ... +end -- of closure - ctx = ctx or { } +do -- create closure to overcome 200 locals limit - function ctx.writestatus(a,b,c,...) - if c then - texio.write_nl(("%-15s: %s\n"):format(a,b:format(c,...))) - else - texio.write_nl(("%-15s: %s\n"):format(a,b)) -- b can have %'s - end - end +if not modules then modules = { } end modules ['data-lst'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} - -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc +-- used in mtxrun - local statusinfo, n = { }, 0 +local find, concat, upper, format = string.find, table.concat, string.upper, string.format - function ctx.register_statistics(tag,pattern,fnc) - statusinfo[#statusinfo+1] = { tag, pattern, fnc } - if #tag > n then n = #tag end - end +resolvers.listers = resolvers.listers or { } - function ctx.memused() - -- collectgarbage("collect") - return string.format("%s MB (ctx: %s MB)",math.round(collectgarbage("count")), math.round(status.luastate_bytes/1000)) - end - - function ctx.show_statistics() -- todo: move calls - local loadtime, register_statistics = input.loadtime, ctx.register_statistics - if caches then - register_statistics("used config path", "%s", function() return caches.configpath() end) - register_statistics("used cache path", "%s", function() return caches.temp() or "?" end) - end - if status.luabytecodes > 0 and input.storage and input.storage.done then - register_statistics("modules/dumps/instances", "%s/%s/%s", function() return status.luabytecodes-500, input.storage.done, status.luastates end) - end - if input.instance then - register_statistics("input load time", "%s seconds", function() return loadtime(input.instance) end) - end - if fonts then - register_statistics("fonts load time","%s seconds", function() return loadtime(fonts) end) - end - if xml then - register_statistics("xml load time", "%s seconds, lpath calls: %s, cached calls: %s", function() - local stats = xml.statistics() - return loadtime(xml), stats.lpathcalls, stats.lpathcached - end) - register_statistics("lxml load time", "%s seconds preparation, backreferences: %i", function() - return loadtime(lxml), #lxml.self - end) - end - if mptopdf then - register_statistics("mps conversion time", "%s seconds", function() return loadtime(mptopdf) end) - end - if nodes then - register_statistics("node processing time", "%s seconds including kernel", function() return loadtime(nodes) end) - end - if kernel then - register_statistics("kernel processing time", "%s seconds", function() return loadtime(kernel) end) - end - if attributes then - register_statistics("attribute processing time", "%s seconds", function() return loadtime(attributes) end) - end - if languages then - register_statistics("language load time", "%s seconds, n=%s", function() return loadtime(languages), languages.hyphenation.n() end) - end - if figures then - register_statistics("graphics processing time", "%s seconds including tex, n=%s", function() return loadtime(figures), figures.n or "?" end) - end - if metapost then - register_statistics("metapost processing time", "%s seconds, loading: %s seconds, execution: %s seconds, n: %s", function() return loadtime(metapost), loadtime(mplib), loadtime(metapost.exectime), metapost.n end) - end - if status.luastate_bytes and ctx.memused then - register_statistics("current memory usage", "%s", ctx.memused) - end - if nodes then - register_statistics("cleaned up reserved nodes", "%s nodes, %s lists of %s", function() return nodes.cleanup_reserved(tex.count[24]) end) -- \topofboxstack - end - if status.node_mem_usage then - register_statistics("node memory usage", "%s", function() return status.node_mem_usage end) - end - if languages then - register_statistics("loaded patterns", "%s", function() return languages.logger.report() end) - end - if fonts then - register_statistics("loaded fonts", "%s", function() return fonts.logger.report() end) - end - if xml then -- so we are in mkiv, we need a different check - register_statistics("runtime", "%s seconds, %i processed pages, %i shipped pages, %.3f pages/second", function() - input.stoptiming(input.instance) - local runtime = loadtime(input.instance) - local shipped = tex.count['nofshipouts'] - local pages = tex.count['realpageno'] - 1 - local persecond = shipped / runtime - return runtime, pages, shipped, persecond - end) - end - for _, t in ipairs(statusinfo) do - local tag, pattern, fnc = t[1], t[2], t[3] - ctx.writestatus("mkiv lua stats", "%s - %s", tag:rpadd(n," "), pattern:format(fnc())) - end-- input.expanded_path_list("osfontdir") +local function tabstr(str) + if type(str) == 'table' then + return concat(str," | ") + else + return str end +end -end end - -if texconfig and not texlua then - - texconfig.kpse_init = false - texconfig.trace_file_names = input.logmode() == 'tex' - texconfig.max_print_line = 100000 - - -- if still present, we overload kpse (put it off-line so to say) - - input.starttiming(input.instance) - - if not input.instance then - - if not input.instance then -- prevent a second loading - - input.instance = input.reset() - input.instance.progname = 'context' - input.instance.engine = 'luatex' - input.instance.validfile = input.validctxfile - - input.load() - - end - - if callback then - callback.register('find_read_file' , function(id,name) return input.findtexfile(name) end) - callback.register('open_read_file' , function( name) return input.opentexfile(name) end) - end - - if callback then - callback.register('find_data_file' , function(name) return input.findbinfile(name,"tex") end) - callback.register('find_enc_file' , function(name) return input.findbinfile(name,"enc") end) - callback.register('find_font_file' , function(name) return input.findbinfile(name,"tfm") end) - callback.register('find_format_file' , function(name) return input.findbinfile(name,"fmt") end) - callback.register('find_image_file' , function(name) return input.findbinfile(name,"tex") end) - callback.register('find_map_file' , function(name) return input.findbinfile(name,"map") end) - callback.register('find_ocp_file' , function(name) return input.findbinfile(name,"ocp") end) - callback.register('find_opentype_file' , function(name) return input.findbinfile(name,"otf") end) - callback.register('find_output_file' , function(name) return name end) - callback.register('find_pk_file' , function(name) return input.findbinfile(name,"pk") end) - callback.register('find_sfd_file' , function(name) return input.findbinfile(name,"sfd") end) - callback.register('find_truetype_file' , function(name) return input.findbinfile(name,"ttf") end) - callback.register('find_type1_file' , function(name) return input.findbinfile(name,"pfb") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(name,"vf") end) - - callback.register('read_data_file' , function(file) return input.loadbinfile(file,"tex") end) - callback.register('read_enc_file' , function(file) return input.loadbinfile(file,"enc") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(file,"tfm") end) - -- format - -- image - callback.register('read_map_file' , function(file) return input.loadbinfile(file,"map") end) - callback.register('read_ocp_file' , function(file) return input.loadbinfile(file,"ocp") end) - callback.register('read_opentype_file' , function(file) return input.loadbinfile(file,"otf") end) - -- output - callback.register('read_pk_file' , function(file) return input.loadbinfile(file,"pk") end) - callback.register('read_sfd_file' , function(file) return input.loadbinfile(file,"sfd") end) - callback.register('read_truetype_file' , function(file) return input.loadbinfile(file,"ttf") end) - callback.register('read_type1_file' , function(file) return input.loadbinfile(file,"pfb") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(file,"vf" ) end) - end - - if input.aleph_mode == nil then environment.aleph_mode = true end -- some day we will drop omega font support - - if callback and input.aleph_mode then - callback.register('find_font_file' , function(name) return input.findbinfile(name,"ofm") end) - callback.register('read_font_file' , function(file) return input.loadbinfile(file,"ofm") end) - callback.register('find_vf_file' , function(name) return input.findbinfile(name,"ovf") end) - callback.register('read_vf_file' , function(file) return input.loadbinfile(file,"ovf") end) - end - - if callback then - callback.register('find_write_file' , function(id,name) return name end) - end - - if callback and (not config or (#config == 0)) then - callback.register('find_format_file' , function(name) return name end) - end - - if callback and false then - for k, v in pairs(callback.list()) do - if not v then texio.write_nl("<w>callback "..k.." is not set</w>") end - end - end - - if callback then - - input.start_actions = { } - input.stop_actions = { } - - function input.register_start_actions(f) table.insert(input.start_actions, f) end - function input.register_stop_actions (f) table.insert(input.stop_actions, f) end - - --~ callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) - --~ callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end end) - - end - - if callback then - - if input.logmode() == 'xml' then - - function input.start_page_number() - texio.write_nl("<p real='" .. tex.count[0] .. "' page='"..tex.count[1].."' sub='"..tex.count[2].."'") - end - function input.stop_page_number() - texio.write("/>") - texio.write_nl("") - end - - callback.register('start_page_number' , input.start_page_number) - callback.register('stop_page_number' , input.stop_page_number ) - - function input.report_output_pages(p,b) - texio.write_nl("<v k='pages'>"..p.."</v>") - texio.write_nl("<v k='bytes'>"..b.."</v>") - texio.write_nl("") - end - function input.report_output_log() - end - - callback.register('report_output_pages', input.report_output_pages) - callback.register('report_output_log' , input.report_output_log ) - - function input.start_run() - texio.write_nl("<?xml version='1.0' standalone='yes'?>") - texio.write_nl("<job xmlns='www.tug.org/luatex/schemas/context-job.rng'>") - texio.write_nl("") - end - function input.stop_run() - texio.write_nl("</job>") - end - function input.show_statistics() - for k,v in pairs(status.list()) do - texio.write_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") - end +local function list(list,report) + local instance = resolvers.instance + local pat = upper(pattern or "","") + local report = report or texio.write_nl + for _,key in pairs(table.sortedkeys(list)) do + if instance.pattern == "" or find(upper(key),pat) then + if instance.kpseonly then + if instance.kpsevars[key] then + report(format("%s=%s",key,tabstr(list[key]))) end - - table.insert(input.start_actions, input.start_run) - table.insert(input.stop_actions , input.show_statistics) - table.insert(input.stop_actions , input.stop_run) - else - table.insert(input.stop_actions , input.show_statistics) + report(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) end - - callback.register('start_run', function() for _, a in pairs(input.start_actions) do a() end end) - callback.register('stop_run' , function() for _, a in pairs(input.stop_actions ) do a() end ctx.show_statistics() end) - - end - - end - - if kpse then - - function kpse.find_file(filename,filetype,mustexist) - return input.find_file(filename,filetype,mustexist) - end - function kpse.expand_path(variable) - return input.expand_path(variable) - end - function kpse.expand_var(variable) - return input.expand_var(variable) end - function kpse.expand_braces(variable) - return input.expand_braces(variable) - end - end - end --- program specific configuration (memory settings and alike) - -if texconfig and not texlua then +function resolvers.listers.variables () list(resolvers.instance.variables ) end +function resolvers.listers.expansions() list(resolvers.instance.expansions) end - luatex = luatex or { } - - luatex.variablenames = { - 'main_memory', 'extra_mem_bot', 'extra_mem_top', - 'buf_size','expand_depth', - 'font_max', 'font_mem_size', - 'hash_extra', 'max_strings', 'pool_free', 'pool_size', 'string_vacancies', - 'obj_tab_size', 'pdf_mem_size', 'dest_names_size', - 'nest_size', 'param_size', 'save_size', 'stack_size', - 'trie_size', 'hyph_size', 'max_in_open', - 'ocp_stack_size', 'ocp_list_size', 'ocp_buf_size' - } - - function luatex.variables() - local t, x = { }, nil - for _,v in pairs(luatex.variablenames) do - x = input.var_value(v) - if x and x:find("^%d+$") then - t[v] = tonumber(x) +function resolvers.listers.configurations(report) + local report = report or texio.write_nl + local instance = resolvers.instance + for _,key in ipairs(table.sortedkeys(instance.kpsevars)) do + if not instance.pattern or (instance.pattern=="") or find(key,instance.pattern) then + report(format("%s\n",key)) + for i,c in ipairs(instance.order) do + local str = c[key] + if str then + report(format("\t%s\t%s",i,str)) + end end - end - return t - end - - function luatex.setvariables(tab) - for k,v in pairs(luatex.variables()) do - tab[k] = v - end - end - - if not luatex.variables_set then - luatex.setvariables(texconfig) - luatex.variables_set = true - end - - texconfig.max_print_line = 100000 - texconfig.max_in_open = 127 - -end - --- some tex basics, maybe this will move to ctx - -if tex then - - local texsprint, texwrite = tex.sprint, tex.write - - if not cs then cs = { } end - - function cs.def(k,v) - texsprint(tex.texcatcodes, "\\def\\" .. k .. "{" .. v .. "}") - end - - function cs.chardef(k,v) - texsprint(tex.texcatcodes, "\\chardef\\" .. k .. "=" .. v .. "\\relax") - end - - function cs.boolcase(b) - if b then texwrite(1) else texwrite(0) end - end - - function cs.testcase(b) - if b then - texsprint(tex.texcatcodes, "\\firstoftwoarguments") - else - texsprint(tex.texcatcodes, "\\secondoftwoarguments") + report("") end end - end -if not modules then modules = { } end modules ['luat-kps'] = { - version = 1.001, - comment = "companion to luatools.lua", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -<p>This file is used when we want the input handlers to behave like -<type>kpsewhich</type>. What to do with the following:</p> - -<typing> -{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} -$SELFAUTOLOC : /usr/tex/bin/platform -$SELFAUTODIR : /usr/tex/bin -$SELFAUTOPARENT : /usr/tex -</typing> - -<p>How about just forgetting abou them?</p> ---ldx]]-- - -input = input or { } -input.suffixes = input.suffixes or { } -input.formats = input.formats or { } - -input.suffixes['gf'] = { '<resolution>gf' } -input.suffixes['pk'] = { '<resolution>pk' } -input.suffixes['base'] = { 'base' } -input.suffixes['bib'] = { 'bib' } -input.suffixes['bst'] = { 'bst' } -input.suffixes['cnf'] = { 'cnf' } -input.suffixes['mem'] = { 'mem' } -input.suffixes['mf'] = { 'mf' } -input.suffixes['mfpool'] = { 'pool' } -input.suffixes['mft'] = { 'mft' } -input.suffixes['mppool'] = { 'pool' } -input.suffixes['graphic/figure'] = { 'eps', 'epsi' } -input.suffixes['texpool'] = { 'pool' } -input.suffixes['PostScript header'] = { 'pro' } -input.suffixes['ist'] = { 'ist' } -input.suffixes['web'] = { 'web', 'ch' } -input.suffixes['cweb'] = { 'w', 'web', 'ch' } -input.suffixes['cmap files'] = { 'cmap' } -input.suffixes['lig files'] = { 'lig' } -input.suffixes['bitmap font'] = { } -input.suffixes['MetaPost support'] = { } -input.suffixes['TeX system documentation'] = { } -input.suffixes['TeX system sources'] = { } -input.suffixes['dvips config'] = { } -input.suffixes['type42 fonts'] = { } -input.suffixes['web2c files'] = { } -input.suffixes['other text files'] = { } -input.suffixes['other binary files'] = { } -input.suffixes['opentype fonts'] = { 'otf' } - -input.suffixes['fmt'] = { 'fmt' } -input.suffixes['texmfscripts'] = { 'rb','lua','py','pl' } - -input.suffixes['pdftex config'] = { } -input.suffixes['Troff fonts'] = { } - -input.suffixes['ls-R'] = { } - ---[[ldx-- -<p>If you wondered abou tsome of the previous mappings, how about -the next bunch:</p> ---ldx]]-- - -input.formats['bib'] = '' -input.formats['bst'] = '' -input.formats['mft'] = '' -input.formats['ist'] = '' -input.formats['web'] = '' -input.formats['cweb'] = '' -input.formats['MetaPost support'] = '' -input.formats['TeX system documentation'] = '' -input.formats['TeX system sources'] = '' -input.formats['Troff fonts'] = '' -input.formats['dvips config'] = '' -input.formats['graphic/figure'] = '' -input.formats['ls-R'] = '' -input.formats['other text files'] = '' -input.formats['other binary files'] = '' - -input.formats['gf'] = '' -input.formats['pk'] = '' -input.formats['base'] = 'MFBASES' -input.formats['cnf'] = '' -input.formats['mem'] = 'MPMEMS' -input.formats['mf'] = 'MFINPUTS' -input.formats['mfpool'] = 'MFPOOL' -input.formats['mppool'] = 'MPPOOL' -input.formats['texpool'] = 'TEXPOOL' -input.formats['PostScript header'] = 'TEXPSHEADERS' -input.formats['cmap files'] = 'CMAPFONTS' -input.formats['type42 fonts'] = 'T42FONTS' -input.formats['web2c files'] = 'WEB2C' -input.formats['pdftex config'] = 'PDFTEXCONFIG' -input.formats['texmfscripts'] = 'TEXMFSCRIPTS' -input.formats['bitmap font'] = '' -input.formats['lig files'] = 'LIGFONTS' - +end -- of closure -- end library merge -- We initialize some characteristics of this program. We need to @@ -6623,21 +6577,33 @@ own.libs = { -- todo: check which ones are really needed 'l-number.lua', 'l-set.lua', 'l-os.lua', - 'l-md5.lua', 'l-file.lua', + 'l-md5.lua', 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-unicode.lua', 'l-math.lua', 'l-utils.lua', - 'luat-lib.lua', - 'luat-inp.lua', - 'luat-log.lua', - 'luat-tmp.lua', - 'luat-zip.lua', - 'luat-tex.lua', - 'luat-kps.lua', + 'trac-tra.lua', + 'luat-env.lua', + 'trac-inf.lua', + 'trac-log.lua', + 'data-res.lua', + 'data-tmp.lua', +-- 'data-pre.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-tex.lua', +-- 'data-bin.lua', +-- 'data-zip.lua', +-- 'data-crl.lua', +-- 'data-lua.lua', + 'data-kps.lua', -- so that we can replace kpsewhich + 'data-aux.lua', -- updater + 'data-lst.lua', -- lister } -- We need this hack till luatex is fixed. @@ -6674,11 +6640,11 @@ function locate_libs() end end -if not input then +if not resolvers then locate_libs() end -if not input then +if not resolvers then print("") print("Luatools is unable to start up due to lack of libraries. You may") print("try to run 'lua luatools.lua --selfmerge' in the path where this") @@ -6687,63 +6653,65 @@ if not input then os.exit() end -input.instance = input.reset() -input.verbose = environment.arguments["verbose"] or false -input.banner = 'LuaTools' -utils.report = input.report - -input.defaultlibs = { -- not all are needed - 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', - 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-dimen.lua', - 'luat-lib.lua', 'luat-inp.lua', 'luat-env.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' -} +logs.setprogram('LuaTools',"TDS Management Tool 1.31",environment.arguments["verbose"] or false) --- todo: use environment.argument() instead of environment.arguments[] +local instance = resolvers.reset() -local instance = input.instance +resolvers.defaultlibs = { -- not all are needed + 'l-string.lua', + 'l-lpeg.lua', + 'l-table.lua', + 'l-boolean.lua', + 'l-number.lua', + 'l-unicode.lua', + 'l-os.lua', + 'l-io.lua', + 'l-file.lua', + 'l-md5.lua', + 'l-url.lua', + 'l-dir.lua', + 'l-utils.lua', + 'l-dimen.lua', + 'trac-inf.lua', + 'trac-tra.lua', + 'trac-log.lua', + 'luat-env.lua', -- here ? + 'data-res.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-tmp.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-pre.lua', + 'data-tex.lua', + 'data-bin.lua', +-- 'data-zip.lua', +-- 'data-clr.lua', + 'data-lua.lua', + 'data-ctx.lua', + 'luat-fio.lua', + 'luat-cnf.lua', +} instance.engine = environment.arguments["engine"] or 'luatex' instance.progname = environment.arguments["progname"] or 'context' instance.luaname = environment.arguments["luafile"] or "" -- environment.ownname or "" -instance.lualibs = environment.arguments["lualibs"] or table.concat(input.defaultlibs,",") +instance.lualibs = environment.arguments["lualibs"] or table.concat(resolvers.defaultlibs,",") instance.allresults = environment.arguments["all"] or false instance.pattern = environment.arguments["pattern"] or nil instance.sortdata = environment.arguments["sort"] or false instance.kpseonly = not environment.arguments["all"] or false instance.my_format = environment.arguments["format"] or instance.format -instance.lsrmode = environment.arguments["lsr"] or false if type(instance.pattern) == 'boolean' then - input.report("invalid pattern specification") -- toto, force verbose for one message + logs.simple("invalid pattern specification") instance.pattern = nil end -if environment.arguments["trace"] then input.settrace(environment.arguments["trace"]) end +if environment.arguments["trace"] then resolvers.settrace(environment.arguments["trace"]) end -if environment.arguments["minimize"] then - if input.validators.visibility[instance.progname] then - instance.validfile = input.validators.visibility[instance.progname] - end -end - -function input.my_prepare_a() - input.resetconfig() - input.identify_cnf() - input.load_lua() - input.expand_variables() - input.load_cnf() - input.expand_variables() -end - -function input.my_prepare_b() - input.my_prepare_a() - input.load_hash() - input.automount() -end - --- barename - -if not messages then messages = { } end +runners = runners or { } +messages = messages or { } messages.no_ini_file = [[ There is no lua initialization file found. This file can be forced by the @@ -6769,21 +6737,18 @@ messages.help = [[ --luafile=str lua inifile (default is <progname>.lua) --lualibs=list libraries to assemble (optional when --compile) --compile assemble and compile lua inifile ---mkii force context mkii mode (only for testing, not usable!) --verbose give a bit more info ---minimize optimize lists for format --all show all found files --sort sort cached data --engine=str target engine --progname=str format or backend --pattern=str filter variables ---lsr use lsr and cnf directly ]] -function input.my_make_format(texname) - local instance = input.instance +function runners.make_format(texname) + local instance = resolvers.instance if texname and texname ~= "" then - if input.usecache then + if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform if path and lfs then lfs.chdir(path) @@ -6793,22 +6758,22 @@ function input.my_make_format(texname) if barename == texname then texname = texname .. ".tex" end - local fullname = input.find_files(texname)[1] or "" + local fullname = resolvers.find_files(texname)[1] or "" if fullname == "" then - input.report("no tex file with name: %s",texname) + logs.simple("no tex file with name: %s",texname) else local luaname, lucname, luapath, lualibs = "", "", "", { } -- the following is optional, since context.lua can also -- handle this collect and compile business if environment.arguments["compile"] then if luaname == "" then luaname = barename end - input.report("creating initialization file: %s",luaname) + logs.simple("creating initialization file: %s",luaname) luapath = file.dirname(luaname) if luapath == "" then luapath = file.dirname(texname) end if luapath == "" then - luapath = file.dirname(input.find_files(texname)[1] or "") + luapath = file.dirname(resolvers.find_files(texname)[1] or "") end lualibs = string.split(instance.lualibs,",") luaname = file.basename(barename .. ".lua") @@ -6818,83 +6783,86 @@ function input.my_make_format(texname) if lualibs[1] then local firstlib = file.join(luapath,lualibs[1]) if not lfs.isfile(firstlib) then - local foundname = input.find_files(lualibs[1])[1] + local foundname = resolvers.find_files(lualibs[1])[1] if foundname then - input.report("located library path: %s",luapath) + logs.simple("located library path: %s",luapath) luapath = file.dirname(foundname) end end end - input.report("using library path: %s",luapath) - input.report("using lua libraries: %s",table.join(lualibs," ")) + logs.simple("using library path: %s",luapath) + logs.simple("using lua libraries: %s",table.join(lualibs," ")) utils.merger.selfcreate(lualibs,luapath,luaname) - local strip = input.boolean_variable("LUACSTRIP", true) + local strip = resolvers.boolean_variable("LUACSTRIP", true) if utils.lua.compile(luaname,lucname,false,strip) and io.exists(lucname) then luaname = lucname - input.report("using compiled initialization file: %s",lucname) + logs.simple("using compiled initialization file: %s",lucname) else - input.report("using uncompiled initialization file: %s",luaname) + logs.simple("using uncompiled initialization file: %s",luaname) end else for _, v in pairs({instance.luaname, instance.progname, barename}) do v = string.gsub(v..".lua","%.lua%.lua$",".lua") if v and (v ~= "") then - luaname = input.find_files(v)[1] or "" + luaname = resolvers.find_files(v)[1] or "" if luaname ~= "" then break end end end end + if environment.arguments["noluc"] then + luaname = luaname:gsub("%.luc$",".lua") -- make this an option + end if luaname == "" then - input.reportlines(messages.no_ini_file) - input.report("texname : %s",texname) - input.report("luaname : %s",instance.luaname) - input.report("progname: %s",instance.progname) - input.report("barename: %s",barename) + if logs.verbose then + logs.simplelines(messages.no_ini_file) + logs.simple("texname : %s",texname) + logs.simple("luaname : %s",instance.luaname) + logs.simple("progname: %s",instance.progname) + logs.simple("barename: %s",barename) + end else - input.report("using lua initialization file: %s",luaname) - local mp = dir.glob(file.stripsuffix(file.basename(luaname)).."-*.mem") + logs.simple("using lua initialization file: %s",luaname) + local mp = dir.glob(file.removesuffix(file.basename(luaname)).."-*.mem") if mp and #mp > 0 then for _, name in ipairs(mp) do - input.report("removing related mplib format %s", file.basename(name)) + logs.simple("removing related mplib format %s", file.basename(name)) os.remove(name) end end - local flags = { "--ini" } - if environment.arguments["mkii"] then - flags[#flags+1] = "--progname=" .. instance.progname - else - flags[#flags+1] = "--lua=" .. string.quote(luaname) - end + local flags = { + "--ini", + "--lua=" .. string.quote(luaname) + } local bs = (os.platform == "unix" and "\\\\") or "\\" -- todo: make a function local command = "luatex ".. table.concat(flags," ") .. " " .. string.quote(fullname) .. " " .. bs .. "dump" - input.report("running command: %s\n",command) + logs.simple("running command: %s\n",command) os.spawn(command) -- todo: do a dummy run that generates the related metafun and mfplain formats end end else - input.report("no tex file given") + logs.simple("no tex file given") end end -function input.my_run_format(name,data,more) +function runners.run_format(name,data,more) -- hm, rather old code here; we can now use the file.whatever functions if name and (name ~= "") then local barename = name:gsub("%.%a+$","") local fmtname = "" - if input.usecache then + if resolvers.usecache then local path = file.join(caches.setpath("formats")) -- maybe platform fmtname = file.join(path,barename..".fmt") or "" end if fmtname == "" then - fmtname = input.find_files(barename..".fmt")[1] or "" + fmtname = resolvers.find_files(barename..".fmt")[1] or "" end - fmtname = input.clean_path(fmtname) + fmtname = resolvers.clean_path(fmtname) barename = fmtname:gsub("%.%a+$","") if fmtname == "" then - input.report("no format with name: %s",name) + logs.simple("no format with name: %s",name) else local luaname = barename .. ".luc" local f = io.open(luaname) @@ -6904,151 +6872,104 @@ function input.my_run_format(name,data,more) end if f then f:close() - local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. string.quote(more) - input.report("running command: %s",command) + local command = "luatex --fmt=" .. string.quote(barename) .. " --lua=" .. string.quote(luaname) .. " " .. string.quote(data) .. " " .. (more ~= "" and string.quote(more) or "") + logs.simple("running command: %s",command) os.spawn(command) else - input.report("using format name: %s",fmtname) - input.report("no luc/lua with name: %s",barename) - end - end - end -end - --- helpers for verbose lists - -input.listers = input.listers or { } - -local function tabstr(str) - if type(str) == 'table' then - return table.concat(str," | ") - else - return str - end -end - -local function list(list) - local instance = input.instance - local pat = string.upper(pattern or "","") - for _,key in pairs(table.sortedkeys(list)) do - if instance.pattern == "" or string.find(key:upper(),pat) then - if instance.kpseonly then - if instance.kpsevars[key] then - print(format("%s=%s",key,tabstr(list[key]))) - end - else - print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) + logs.simple("using format name: %s",fmtname) + logs.simple("no luc/lua with name: %s",barename) end end end end -function input.listers.variables () list(input.instance.variables ) end -function input.listers.expansions() list(input.instance.expansions) end - -function input.listers.configurations() - local instance = input.instance - for _,key in pairs(table.sortedkeys(instance.kpsevars)) do - if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then - print(key.."\n") - for i,c in ipairs(instance.order) do - local str = c[key] - if str then - print(format("\t%s\t%s",i,str)) - end - end - print() - end - end -end - -input.report("%s\n",banner) - local ok = true +-- private option --noluc for testing errors in the stub + if environment.arguments["find-file"] then - input.my_prepare_b() + resolvers.load() instance.format = environment.arguments["format"] or instance.format if instance.pattern then instance.allresults = true - input.for_files(input.find_files, { instance.pattern }, instance.my_format) + resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format) else - input.for_files(input.find_files, environment.files, instance.my_format) + resolvers.for_files(resolvers.find_files, environment.files, instance.my_format) end elseif environment.arguments["find-path"] then - input.my_prepare_b() - local path = input.find_file(environment.files[1], instance.my_format) - if input.verbose then - input.report(file.dirname(path)) + resolvers.load() + local path = resolvers.find_file(environment.files[1], instance.my_format) + if logs.verbose then + logs.simple(file.dirname(path)) else print(file.dirname(path)) end elseif environment.arguments["run"] then - input.my_prepare_a() -- ! no need for loading databases - input.verbose = true - input.my_run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") + resolvers.load("nofiles") -- ! no need for loading databases + logs.setverbose(true) + runners.run_format(environment.files[1] or "",environment.files[2] or "",environment.files[3] or "") elseif environment.arguments["fmt"] then - input.my_prepare_a() -- ! no need for loading databases - input.verbose = true - input.my_run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") + resolvers.load("nofiles") -- ! no need for loading databases + logs.setverbose(true) + runners.run_format(environment.arguments["fmt"], environment.files[1] or "",environment.files[2] or "") elseif environment.arguments["expand-braces"] then - input.my_prepare_a() - input.for_files(input.expand_braces, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_braces, environment.files) elseif environment.arguments["expand-path"] then - input.my_prepare_a() - input.for_files(input.expand_path, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_path, environment.files) elseif environment.arguments["expand-var"] or environment.arguments["expand-variable"] then - input.my_prepare_a() - input.for_files(input.expand_var, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.expand_var, environment.files) elseif environment.arguments["show-path"] or environment.arguments["path-value"] then - input.my_prepare_a() - input.for_files(input.show_path, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.show_path, environment.files) elseif environment.arguments["var-value"] or environment.arguments["show-value"] then - input.my_prepare_a() - input.for_files(input.var_value, environment.files) + resolvers.load("nofiles") + resolvers.for_files(resolvers.var_value, environment.files) elseif environment.arguments["format-path"] then - input.my_prepare_b() - input.report(caches.setpath("format")) + resolvers.load() + logs.simple(caches.setpath("format")) elseif instance.pattern then -- brrr - input.my_prepare_b() + resolvers.load() instance.format = environment.arguments["format"] or instance.format instance.allresults = true - input.for_files(input.find_files, { instance.pattern }, instance.my_format) + resolvers.for_files(resolvers.find_files, { instance.pattern }, instance.my_format) elseif environment.arguments["generate"] then instance.renewcache = true - input.verbose = true - input.my_prepare_b(instance) + logs.setverbose(true) + resolvers.load() elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then - input.my_prepare_b(instance) - input.verbose = true - input.my_make_format(environment.files[1] or "") + resolvers.load() + logs.setverbose(true) + runners.make_format(environment.files[1] or "") elseif environment.arguments["selfmerge"] then utils.merger.selfmerge(own.name,own.libs,own.list) elseif environment.arguments["selfclean"] then utils.merger.selfclean(own.name) elseif environment.arguments["selfupdate"] then - input.my_prepare_b() - input.verbose = true - input.update_script(own.name,"luatools") + resolvers.load() + logs.setverbose(true) + resolvers.update_script(own.name,"luatools") elseif environment.arguments["variables"] or environment.arguments["show-variables"] then - input.my_prepare_a() - input.listers.variables() + resolvers.load("nofiles") + resolvers.listers.variables() elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then - input.my_prepare_a() - input.listers.expansions() + resolvers.load("nofiles") + resolvers.listers.expansions() elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then - input.my_prepare_a() - input.listers.configurations() + resolvers.load("nofiles") + resolvers.listers.configurations() elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then - input.help(banner,messages.help) + logs.help(messages.help) else - input.my_prepare_b() - input.for_files(input.find_files, environment.files, instance.my_format) + resolvers.load() + resolvers.for_files(resolvers.find_files, environment.files, instance.my_format) end -if input.verbose then - input.report("") - input.report("runtime: %0.3f seconds",os.runtime()) +if logs.verbose then + logs.simpleline() + logs.simple("runtime: %0.3f seconds",os.runtime()) end if os.platform == "unix" then diff --git a/scripts/context/lua/mtx-babel.lua b/scripts/context/lua/mtx-babel.lua index 92b1cd597..8d7765643 100644 --- a/scripts/context/lua/mtx-babel.lua +++ b/scripts/context/lua/mtx-babel.lua @@ -385,22 +385,22 @@ do local structure = environment.argument("structure") or "document" converter = converter[structure] if converter then - input.report("converting '%s' using language '%s' with structure '%s'", filename, language, structure) + logs.simple("converting '%s' using language '%s' with structure '%s'", filename, language, structure) data = converter:match(data) local newfilename = filename .. ".utf" io.savedata(newfilename, data) - input.report("converted data saved in '%s'", newfilename) + logs.simple("converted data saved in '%s'", newfilename) else - input.report("unknown structure '%s' language '%s'", structure, language) + logs.simple("unknown structure '%s' language '%s'", structure, language) end else - input.report("no converter for language '%s'", language) + logs.simple("no converter for language '%s'", language) end else - input.report("provide language") + logs.simple("provide language") end else - input.report("no data in '%s'",filename) + logs.simple("no data in '%s'",filename) end end end @@ -413,7 +413,7 @@ do end -banner = banner .. " | babel conversion tools | version 1.2" +logs.extendbanner("Babel Conversion Tools 1.2",true) messages.help = [[ --language=string conversion language (e.g. greek) @@ -421,10 +421,8 @@ messages.help = [[ --convert convert babel codes into utf ]] -input.verbose = true - if environment.argument("convert") then scripts.babel.convert(environment.files[1] or "") else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-cache.lua b/scripts/context/lua/mtx-cache.lua index 0432168ee..a1fbed825 100644 --- a/scripts/context/lua/mtx-cache.lua +++ b/scripts/context/lua/mtx-cache.lua @@ -76,7 +76,7 @@ function scripts.cache.list(all) end) end -banner = banner .. " | cache tools " +logs.extendbanner("Cache Tools 0.10") messages.help = [[ --purge remove not used files @@ -93,5 +93,5 @@ elseif environment.argument("erase") then elseif environment.argument("list") then scripts.cache.list(environment.argument("all")) else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-chars.lua b/scripts/context/lua/mtx-chars.lua index f4f751377..d4330ca30 100644 --- a/scripts/context/lua/mtx-chars.lua +++ b/scripts/context/lua/mtx-chars.lua @@ -6,38 +6,12 @@ if not modules then modules = { } end modules ['mtx-chars'] = { license = "see context related readme files" } -local format, concat = string.format, table.concat +local format, concat, utfchar, upper = string.format, table.concat, unicode.utf8.char, string.upper scripts = scripts or { } scripts.chars = scripts.chars or { } -function scripts.chars.stixtomkiv(inname,outname) - if inname == "" then - logs.report("aquiring math data","invalid datafilename") - end - local f = io.open(inname) - if not f then - logs.report("aquiring math data","invalid datafile") - else - logs.report("aquiring math data","processing " .. inname) - if not outname or outname == "" then - outname = "char-mth.lua" - end - local classes = { - N = "normal", - A = "alphabetic", - D = "diacritic", - P = "punctuation", - B = "binary", - R = "relation", - L = "large", - O = "opening", - C = "closing", - F = "fence" - } - local valid, done = false, { } - local g = io.open(outname,'w') - g:write([[ +local banner = [[ -- filename : char-mth.lua -- comment : companion to char-mth.tex (in ConTeXt) -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -46,38 +20,70 @@ function scripts.chars.stixtomkiv(inname,outname) if not versions then versions = { } end versions['char-mth'] = 1.001 if not characters then characters = { } end - ]]) - g:write(format("\ncharacters.math = {\n")) - for l in f:lines() do - if not valid then - valid = l:find("AMS/TeX name") - end - if valid then - local unicode = l:sub(2,6) - if unicode:sub(1,1) ~= " " and unicode ~= "" and not done[unicode] then - local mathclass, adobename, texname = l:sub(57,57) or "", l:sub(13,36) or "", l:sub(84,109) or "" - texname, adobename = texname:gsub("[\\ ]",""), adobename:gsub("[\\ ]","") - local t = { } - if mathclass ~= "" then t[#t+1] = format("mathclass='%s'", classes[mathclass] or "unknown") end - if adobename ~= "" then t[#t+1] = format("adobename='%s'", adobename ) end - if texname ~= "" then t[#t+1] = format("texname='%s'" , texname ) end - if #t > 0 then - g:write(format("\t[0x%s] = { %s },\n",unicode, concat(t,", "))) - end - done[unicode] = true - end - end - end - if not valid then - g:write("\t-- The data file is corrupt, invalid or maybe the format has changed.\n") - logs.report("aquiring math data","problems with data table") - else - logs.report("aquiring math data","table saved in " .. outname) - end - g:write("}\n") - g:close() - f:close() - end +]] + +--~ function scripts.chars.stixtomkiv(inname,outname) +--~ if inname == "" then +--~ logs.report("aquiring math data","invalid datafilename") +--~ end +--~ local f = io.open(inname) +--~ if not f then +--~ logs.report("aquiring math data","invalid datafile") +--~ else +--~ logs.report("aquiring math data","processing " .. inname) +--~ if not outname or outname == "" then +--~ outname = "char-mth.lua" +--~ end +--~ local classes = { +--~ N = "normal", +--~ A = "alphabetic", +--~ D = "diacritic", +--~ P = "punctuation", +--~ B = "binary", +--~ R = "relation", +--~ L = "large", +--~ O = "opening", +--~ C = "closing", +--~ F = "fence" +--~ } +--~ local valid, done = false, { } +--~ local g = io.open(outname,'w') +--~ g:write(banner) +--~ g:write(format("\ncharacters.math = {\n")) +--~ for l in f:lines() do +--~ if not valid then +--~ valid = l:find("AMS/TeX name") +--~ end +--~ if valid then +--~ local unicode = l:sub(2,6) +--~ if unicode:sub(1,1) ~= " " and unicode ~= "" and not done[unicode] then +--~ local mathclass, adobename, texname = l:sub(57,57) or "", l:sub(13,36) or "", l:sub(84,109) or "" +--~ texname, adobename = texname:gsub("[\\ ]",""), adobename:gsub("[\\ ]","") +--~ local t = { } +--~ if mathclass ~= "" then t[#t+1] = format("mathclass='%s'", classes[mathclass] or "unknown") end +--~ if adobename ~= "" then t[#t+1] = format("adobename='%s'", adobename ) end +--~ if texname ~= "" then t[#t+1] = format("texname='%s'" , texname ) end +--~ if #t > 0 then +--~ g:write(format("\t[0x%s] = { %s },\n",unicode, concat(t,", "))) +--~ end +--~ done[unicode] = true +--~ end +--~ end +--~ end +--~ if not valid then +--~ g:write("\t-- The data file is corrupt, invalid or maybe the format has changed.\n") +--~ logs.report("aquiring math data","problems with data table") +--~ else +--~ logs.report("aquiring math data","table saved in " .. outname) +--~ end +--~ g:write("}\n") +--~ g:close() +--~ f:close() +--~ end +--~ end + +function scripts.chars.stixtomkiv(inname,outname) + logs.report("we no longer use this options but use our own tables instead") end local banner_pdf_1 = [[ @@ -95,7 +101,7 @@ local banner_pdf_2 = [[ ]] function scripts.chars.makepdfr() - local chartable = input.find_file("char-def.lua") or "" + local chartable = resolvers.find_file("char-def.lua") or "" if chartable ~= "" then dofile(chartable) if characters and characters.data then @@ -117,93 +123,189 @@ function scripts.chars.makepdfr() end end -local banner_utf_1 = [[ -% filename : enco-utf.tex -% comment : generated by mtxrun --script chars --utf -% author : Hans Hagen, PRAGMA-ADE, Hasselt NL -% copyright: PRAGMA ADE / ConTeXt Development Team -% license : see context related readme files +local banner_utf_module = [[ +%% filename : %s +%% comment : generated by mtxrun --script chars --xtx +%% author : Hans Hagen, PRAGMA-ADE, Hasselt NL +%% copyright: PRAGMA ADE / ConTeXt Development Team +%% license : see context related readme files +]] -\ifx\setcclcucx\undefined +local banner_utf_mappings = [[ - \def\setcclcucx #1 #2 #3 % - {\global\catcode"#1=11 - \global\lccode "#1="#2 - \global\uccode "#1="#3 } +% lc/uc/catcode mappings -\fi ]] -local banner_utf_2 = [[ +local banner_utf_patch = [[ -% lc/uc/catcode mappings +% patch needed for turkish +\setXTXcharcodes "201C "201C "201C +\setXTXcharcodes "201D "201D "201D ]] -local banner_utf_3 = [[ +local banner_utf_names = [[ % named characters mapped onto utf (\\char is needed for accents) ]] -local banner_utf_4 = [[ +local banner_utf_classes = [[ + +% some character classes for xetex; seems to be rather hard coded, these numbers +% and also a mix of several classes; here we do linebreaks + +]] + +local banner_utf_finish = [[ \endinput ]] +local xtxclasses = { + id = 1, + ex = 3, + is = 3, + cm = 256, + op = 2, + ns = 3, + cl = 3, +} + function scripts.chars.makeencoutf() - local chartable = input.find_file("char-def.lua") or "" + local chartable = resolvers.find_file("char-def.lua") or "" if chartable ~= "" then dofile(chartable) - if characters and characters.data then - local f = io.open("enco-utf.tex", 'w') + local function open(name,banner) + local f = io.open(name,'w') + if f then + logs.simple("writing '%s'",name) + f:write(format(banner_utf_module,name)) + f:write(banner) + f:write() + return f + end + end + local function close(f) + f:write(banner_utf_finish) + f:close() + end + local data = characters and characters.data + if data then + local list = table.sortedkeys(characters.data) + local f = open("xetx-utf.tex",banner_utf_mappings) if f then - f:write(banner_utf_1) - f:write(banner_utf_2) - local list = table.sortedkeys(characters.data) - local length = 0 for i=1,#list do local code = list[i] if code <= 0xFFFF then - local chr = characters.data[code] + local chr = data[code] local cc = chr.category if cc == 'll' or cc == 'lu' or cc == 'lt' then if not chr.lccode then chr.lccode = code end if not chr.uccode then chr.uccode = code end - f:write(format("\\setcclcucx %04X %04X %04X %% %s\n",code,chr.lccode,chr.uccode,chr.description)) + f:write(format('\\setXTXcharcodes "%05X "%05X "%05X %% %s\n',code,chr.lccode,chr.uccode,chr.description)) + end + end + end + f:write("\n") + for i=1,#list do + local code = list[i] + local chr = data[code] + if chr and chr.range then + local cc = chr.category + if cc == 'lo' then + f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharcodes\\recurselevel\\recurselevel\\recurselevel}\n',code,chr.range)) end - if #(chr.contextname or "") > length then + end + end + f:write(banner_utf_patch) + close(f) + end + local f = open("xetx-chr.tex",banner_utf_names) + if f then + local length = 0 + for i=1,#list do + local code = list[i] + if code > 0x5B and code <= 0xFFFF then + local chr = data[code] + if chr and #(chr.contextname or "") > length then length = #chr.contextname end end end - f:write(banner_utf_3) for i=1,#list do local code = list[i] if code > 0x5B and code <= 0xFFFF then - local chr = characters.data[code] - if chr.contextname then - local ch = char(code) ---~ if ch:find("[~#$%%^&{}\\]") then - f:write(format("\\def\\%s{\\char\"%04X } %% %s: %s\n", chr.contextname:rpadd(length," "), code, chr.description, ch)) ---~ else ---~ f:write(format("\\def\\%s{%s} %% %s\n", chr.contextname:rpadd(length," "), ch,chr.description)) ---~ end + local chr = data[code] + if chr and chr.contextname then + local ch = utfchar(code) + f:write(format("\\def\\%s{\\char\"%05X } %% %s: %s\n", chr.contextname:rpadd(length," "), code, chr.description, ch)) end end end - f:write(banner_utf_4) - f:close() + close(f) + end + local f = open("xetx-cls.tex",banner_utf_classes) + if f then + for k, v in pairs(xtxclasses) do + f:write(format("\\defineXTXcharinjectionclass[lb:%s]\n",k)) + end + f:write("\n") + local i_first, i_last, i_clb = nil, nil, nil + local function flush() + if i_first then + if i_first == i_last then + f:write(format('\\dosetXTXcharacterclass{"%05X}{lb:%s}\n',i_first,i_clb)) + else + f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n',i_first,i_last,i_clb)) + end + end + i_first, i_last, i_clb = nil, nil, nil + end + for i=1,#list do + local code = list[i] + local code_next = list[i+1] + local chr = data[code] + local chr_next = data[code_next] + local clb = chr and chr.linebreak + local lbc = xtxclasses[clb] + if not lbc then + flush() + elseif clb == i_clb then + if i_first then + i_last = code + else + i_first, i_last, i_clb = code, code, clb + end + else + flush() + i_first, i_last, i_clb = code, code, clb + end + end + flush() + f:write("\n") + for i=1,#list do + local code = list[i] + local chr = data[code] + if chr and chr.range then + local lbc = chr.linebreak + if xtxclasses[lbc] then + f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n',code,chr.range,lbc)) + end + end + end + close(f) end end end end -banner = banner .. " | character tools " +logs.extendbanner("Character Tools 0.10") messages.help = [[ --stix convert stix table to math table ---utf generate enco-utf.tex (used by xetex) +--xtx generate xetx-*.tex (used by xetex) --pdf generate pdfr-def.tex (used by pdftex) ]] @@ -211,10 +313,10 @@ if environment.argument("stix") then local inname = environment.files[1] or "" local outname = environment.files[2] or "" scripts.chars.stixtomkiv(inname,outname) -elseif environment.argument("utf") then +elseif environment.argument("xtx") then scripts.chars.makeencoutf() elseif environment.argument("pdf") then scripts.chars.makepdfr() else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-check.lua b/scripts/context/lua/mtx-check.lua index 7c44fa855..6be7f2765 100644 --- a/scripts/context/lua/mtx-check.lua +++ b/scripts/context/lua/mtx-check.lua @@ -34,7 +34,7 @@ do end end - local P, S, V, C, CP, CC = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cc + local P, R, S, V, C, CP, CC = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cc local i_m, d_m = P("$"), P("$$") local l_s, r_s = P("["), P("]") @@ -49,22 +49,27 @@ do local line = newline / function() validator.n = validator.n + 1 end - local grammar = P { "tokens", - ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, - ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), - ["grouped"] = CP() * C(l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g) * CC("group") / progress, - ["setup"] = CP() * C(l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s) * CC("setup") / progress, - ["display"] = CP() * C(d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m) * CC("display") / progress, - ["inline"] = CP() * C(i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m) * CC("inline") / progress, - ["errors"] = (V("gerror") + V("serror") + V("derror") + V("ierror")) * true, - ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, - ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, - ["derror"] = CP() * d_m * CC("display math error") / message, - ["ierror"] = CP() * i_m * CC("inline math error") / message, - } + -- local grammar = P { "tokens", + -- ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + -- ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), + -- ["grouped"] = CP() * C(l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g) * CC("group") / progress, + -- ["setup"] = CP() * C(l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s) * CC("setup") / progress, + -- ["display"] = CP() * C(d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m) * CC("display") / progress, + -- ["inline"] = CP() * C(i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m) * CC("inline") / progress, + -- ["errors"] = (V("gerror") + V("serror") + V("derror") + V("ierror")) * true, + -- ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, + -- ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, + -- ["derror"] = CP() * d_m * CC("display math error") / message, + -- ["ierror"] = CP() * i_m * CC("inline math error") / message, + -- } + + local startluacode = P("\\startluacode") + local stopluacode = P("\\stopluacode") + + local somecode = startluacode * (1-stopluacode)^1 * stopluacode local grammar = P { "tokens", - ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + ["tokens"] = (V("ignore") + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, ["setup"] = l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, @@ -75,6 +80,7 @@ do ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, ["derror"] = CP() * d_m * CC("display math error") / message, ["ierror"] = CP() * i_m * CC("inline math error") / message, + ["ignore"] = somecode, } function validator.check(str) @@ -117,19 +123,16 @@ function scripts.checker.check(filename) end end - -banner = banner .. " | tex check tools " +logs.extendbanner("Syntax Checking 0.10",true) messages.help = [[ --convert check tex file for errors ]] -input.verbose = true - if environment.argument("check") then scripts.checker.check(environment.files[1]) elseif environment.argument("help") then - input.help(banner,messages.help) + logs.help(messages.help) elseif environment.files[1] then scripts.checker.check(environment.files[1]) end diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 7d5eb7e80..2bae51501 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -23,30 +23,6 @@ function io.copydata(fromfile,tofile) io.savedata(tofile,io.loaddata(fromfile) or "") end --- luat-inp - -function input.locate_format(name) -- move this to core / luat-xxx - local barename, fmtname = name:gsub("%.%a+$",""), "" - if input.usecache then - local path = file.join(caches.setpath("formats")) -- maybe platform - fmtname = file.join(path,barename..".fmt") or "" - end - if fmtname == "" then - fmtname = input.find_files(barename..".fmt")[1] or "" - end - fmtname = input.clean_path(fmtname) - if fmtname ~= "" then - barename = fmtname:gsub("%.%a+$","") - local luaname, lucname = barename .. ".lua", barename .. ".luc" - if lfs.isfile(lucname) then - return barename, luaname - elseif lfs.isfile(luaname) then - return barename, luaname - end - end - return nil, nil -end - -- ctx ctxrunner = { } @@ -135,15 +111,15 @@ do elseif ctxdata.ctxname then ctlname = file.replacesuffix(ctxdata.ctxname,'ctl') else - input.report("invalid ctl name: %s",ctlname or "?") + logs.simple("invalid ctl name: %s",ctlname or "?") return end end if table.is_empty(ctxdata.prepfiles) then - input.report("nothing prepared, no ctl file saved") + logs.simple("nothing prepared, no ctl file saved") os.remove(ctlname) else - input.report("saving logdata in: %s",ctlname) + logs.simple("saving logdata in: %s",ctlname) f = io.open(ctlname,'w') if f then f:write("<?xml version='1.0' standalone='yes'?>\n\n") @@ -188,8 +164,8 @@ do ctxdata.jobname = file.addsuffix(ctxdata.jobname,'tex') ctxdata.ctxname = file.addsuffix(ctxdata.ctxname,'ctx') - input.report("jobname: %s",ctxdata.jobname) - input.report("ctxname: %s",ctxdata.ctxname) + logs.simple("jobname: %s",ctxdata.jobname) + logs.simple("ctxname: %s",ctxdata.ctxname) -- mtxrun should resolve kpse: and file: @@ -206,7 +182,7 @@ do end end - if not found and defaultname and defaultname ~= "" and file.exists(defaultname) then + if not found and defaultname and defaultname ~= "" and lfs.isfile(defaultname) then usedname, found = defaultname, true end @@ -236,7 +212,7 @@ do ctxdata.flags = ctxrunner.reflag(ctxdata.flags) for _, message in ipairs(ctxdata.messages) do - input.report("ctx comment: %s", xml.tostring(message)) + logs.simple("ctx comment: %s", xml.tostring(message)) end xml.each(ctxdata.xmldata,"ctx:value[@name='job']", function(ek,e,k) @@ -326,10 +302,10 @@ do -- potential optimization: when mtxrun run internal command = xml.text(command) command = ctxrunner.justtext(command) -- command is still xml element here - input.report("command: %s",command) + logs.simple("command: %s",command) local result = os.spawn(command) or 0 if result > 0 then - input.report("error, return code: %s",result) + logs.simple("error, return code: %s",result) end if ctxdata.runlocal then oldfile = file.basename(oldfile) @@ -340,11 +316,11 @@ do file.syncmtimes(oldfile,newfile) ctxdata.prepfiles[oldfile] = true else - input.report("error, check target location of new file: %s", newfile) + logs.simple("error, check target location of new file: %s", newfile) ctxdata.prepfiles[oldfile] = false end else - input.report("old file needs no preprocessing") + logs.simple("old file needs no preprocessing") ctxdata.prepfiles[oldfile] = lfs.isfile(newfile) end end @@ -362,7 +338,8 @@ end -- rest scripts.context.multipass = { - suffixes = { ".tuo", ".tuc" }, +-- suffixes = { ".tuo", ".tuc" }, + suffixes = { ".tuc" }, nofruns = 8, } @@ -394,6 +371,7 @@ scripts.context.backends = { function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,currentrun,finalrun) -- take jobname from ctx + jobname = file.removesuffix(jobname) local f = io.open(jobname..".top","w") if f then local function someflag(flag) @@ -433,74 +411,124 @@ function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,curr local function setalways(format,...) f:write(format:format(...),"\n") end + -- + setalways("%% runtime options files (command line driven)") + -- setalways("\\unprotect") - setvalue('output' , "\\setupoutput[%s]", scripts.context.backends, 'pdftex') - setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun or 0, currentrun or 0) - setalways( "\\setupsystem[\\c!type=%s]",os.platform) + -- + setalways("%% special commands, mostly for the ctx development team") + -- + if environment.argument("dumpdelta") then + setalways("\\tracersdumpdelta") + elseif environment.argument("dumphash") then + setalways("\\tracersdumphash") + end + setalways("%% feedback and basic job control") + if type(environment.argument("track")) == "string" then + setvalue ("track" , "\\enabletrackers[%s]") + end + setfixed ("timing" , "\\usemodule[timing]") setfixed ("batchmode" , "\\batchmode") setfixed ("nonstopmode" , "\\nonstopmode") setfixed ("tracefiles" , "\\tracefilestrue") + setfixed ("nostats" , "\\nomkivstatistics") setfixed ("paranoid" , "\\def\\maxreadlevel{1}") - setvalues("modefile" , "\\readlocfile{%s}{}{}") + -- + setalways("%% handy for special styles") + -- + setalways("\\startluacode") + setalways("document = document or { }") + setalways(table.serialize(environment.arguments, "document.arguments")) + setalways(table.serialize(environment.files, "document.files")) + setalways("\\stopluacode") + -- + setalways("%% process info") + -- + setalways( "\\setupsystem[\\c!n=%s,\\c!m=%s]", kindofrun or 0, currentrun or 0) + setalways( "\\setupsystem[\\c!type=%s]",os.platform) setvalue ("inputfile" , "\\setupsystem[inputfile=%s]") setvalue ("result" , "\\setupsystem[file=%s]") setvalues("path" , "\\usepath[%s]") + setvalue ("setuppath" , "\\setupsystem[\\c!directory={%s}]") + setvalue ("randomseed" , "\\setupsystem[\\c!random=%s]") + setvalue ("arguments" , "\\setupenv[%s]") + setalways("%% modes") + setvalues("modefile" , "\\readlocfile{%s}{}{}") + setvalues("mode" , "\\enablemode[%s]", true) + if ctxdata then + setvalues(ctxdata.modes, "\\enablemode[%s]") + end + -- + setalways("%% options (not that important)") + -- + setalways("\\startsetups *runtime:options") + setvalue ('output' , "\\setupoutput[%s]", scripts.context.backends, 'pdftex') setfixed ("color" , "\\setupcolors[\\c!state=\\v!start]") - -- setfixed ("nompmode" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics - -- setfixed ("nomprun" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics - -- setfixed ("automprun" , "\\runMPgraphicsfalse") -- obsolete, we assume runtime mp graphics - setfixed ("fast" , "\\fastmode\n") - setfixed ("silentmode" , "\\silentmode\n") - setfixed ("nostats" , "\\nomkivstatistics\n") setvalue ("separation" , "\\setupcolors[\\c!split=%s]") - setvalue ("setuppath" , "\\setupsystem[\\c!directory={%s}]") setfixed ("noarrange" , "\\setuparranging[\\v!disable]") if environment.argument('arrange') and not finalrun then setalways( "\\setuparranging[\\v!disable]") end - setvalue ("randomseed" , "\\setupsystem[\\c!random=%s]") - setvalue ("arguments" , "\\setupenv[%s]") - -- singular and plural - setvalues("mode" , "\\enablemode[%s]", true) + setalways("\\stopsetups") + -- + setalways("%% styles and modules") + -- + setalways("\\startsetups *runtime:modules") setvalues("filter" , "\\useXMLfilter[%s]", true) setvalues("usemodule" , "\\usemodule[%s]", true) setvalues("environment" , "\\environment %s ", true) - -- ctx stuff if ctxdata then - setvalues(ctxdata.modes, "\\enablemode[%s]") setvalues(ctxdata.modules, "\\usemodule[%s]") setvalues(ctxdata.environments, "\\environment %s ") end - -- done - setalways("\\protect") - setalways("\\endinput") + setalways("\\stopsetups") + -- + setalways("%% done") + -- + setalways("\\protect \\endinput") f:close() end end function scripts.context.multipass.copyluafile(jobname) - io.savedata(jobname..".tuc",io.loaddata(jobname..".tua") or "") -end - -function scripts.context.multipass.copytuifile(jobname) - local f, g = io.open(jobname..".tui"), io.open(jobname..".tuo",'w') - if f and g then - g:write("% traditional utility file, only commands written by mtxrun/context\n%\n") - for line in f:lines() do - if line:find("^c ") then - g:write((line:gsub("^c ","")),"%\n") - end - end - g:write("\\endinput\n") - f:close() - g:close() +-- io.savedata(jobname..".tuc",io.loaddata(jobname..".tua") or "") + local tuaname, tucname = jobname..".tua", jobname..".tuc" + if lfs.isfile(tuaname) then + os.remove(tucname) + os.rename(tuaname,tucname) end end +-- obsolete: +-- +-- function scripts.context.multipass.copytuifile(jobname) +-- local tuiname, tuoname = jobname .. ".tui", jobname .. ".tuo" +-- if lfs.isfile(tuiname) then +-- local f, g = io.open(tuiname), io.open(tuoname,'w') +-- if f and g then +-- g:write("% traditional utility file, only commands written by mtxrun/context\n%\n") +-- for line in f:lines() do +-- if line:find("^c ") then +-- g:write((line:gsub("^c ","")),"%\n") +-- end +-- end +-- g:write("\\endinput\n") +-- f:close() +-- g:close() +-- end +-- else +-- -- os.remove(tuoname) +-- end +-- end + scripts.context.xmlsuffixes = table.tohash { "xml", } +scripts.context.luasuffixes = table.tohash { + "lua", +} + scripts.context.beforesuffixes = { "tuo", "tuc" } @@ -518,12 +546,17 @@ scripts.context.interfaces = { it = "cont-it", ro = "cont-ro", pe = "cont-pe", + -- for taco and me + -- xp = "cont-xp", } scripts.context.defaultformats = { "cont-en", "cont-nl", +-- "cont-xp", "mptopdf", +-- "metatex", + "metafun", "plain" } @@ -532,7 +565,7 @@ local function analyze(filename) if f then local t = { } local line = f:read("*line") or "" - local preamble = line:match("^%% *(.*)$") + local preamble = line:match("[\254\255]*%%%s+(.+)$") -- there can be an utf bomb in front if preamble then for key, value in preamble:gmatch("(%S+)=(%S+)") do t[key] = value @@ -541,42 +574,53 @@ local function analyze(filename) elseif line:find("^<?xml ") then t.type = "xml" end + if not t.engine then + t.engine = 'luatex' + end f:close() return t end return nil end -function scripts.context.run(ctxdata) - local function makestub(format,filename) - local stubname = file.replacesuffix(file.basename(filename),'run') - local f = io.open(stubname,'w') - if f then - f:write("\\starttext\n") - f:write(string.format(format,filename),"\n") - f:write("\\stoptext\n") - f:close() - filename = stubname - end - return filename +local function makestub(format,filename) + local stubname = file.replacesuffix(file.basename(filename),'run') + local f = io.open(stubname,'w') + if f then + f:write("\\starttext\n") + f:write(string.format(format,filename),"\n") + f:write("\\stoptext\n") + f:close() + filename = stubname end + return filename +end + +function scripts.context.run(ctxdata,filename) + -- filename overloads environment.files + local files = (filename and { filename }) or environment.files if ctxdata then -- todo: interface for k,v in pairs(ctxdata.flags) do environment.setargument(k,v) end end - local files = environment.files if #files > 0 then - input.identify_cnf() - input.load_cnf() - input.expand_variables() + -- local interface = environment.argument("interface") -- todo: environment.argument("interface","en") interface = (type(interface) == "string" and interface) or "en" -- local formatname = scripts.context.interfaces[interface] or "cont-en" - local formatfile, scriptfile = input.locate_format(formatname) + local formatfile, scriptfile = resolvers.locate_format(formatname) + -- this catches the command line + if not formatfile or not scriptfile then + logs.simple("warning: no format found, forcing remake (commandline driven)") + scripts.context.generate() + scripts.context.make(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) + end + -- if formatfile and scriptfile then for _, filename in ipairs(files) do local basename, pathname = file.basename(filename), file.dirname(filename) @@ -584,137 +628,275 @@ function scripts.context.run(ctxdata) if pathname == "" then filename = "./" .. filename end + -- look at the first line local a = analyze(filename) - if a then - if a.interface and a.interface ~= interface then + if a and (a.engine == 'pdftex' or a.engine == 'xetex' or environment.argument("pdftex") or environment.argument("xetex")) then + local texexec = resolvers.find_file("texexec.rb") or "" + if texexec ~= "" then + local command = string.format("ruby %s %s",texexec,environment.reconstruct_commandline(environment.arguments_after)) + os.exec(command) + end + else + if a and a.interface and a.interface ~= interface then formatname = scripts.context.interfaces[a.interface] or formatname - formatfile, scriptfile = input.locate_format(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) end - end - -- we default to mkiv xml ! - if scripts.context.xmlsuffixes[file.extname(filename) or "?"] or environment.argument("forcexml") then - if environment.argument("mkii") then - filename = makestub("\\processXMLfilegrouped{%s}",filename) - else - filename = makestub("\\xmlprocess{\\xmldocument}{%s}{}",filename) + -- this catches the command line + if not formatfile or not scriptfile then + logs.simple("warning: no format found, forcing remake (source driven)") + scripts.context.generate() + scripts.context.make(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) end - end - -- - -- todo: also other stubs - -- - local resultname, oldbase, newbase = environment.argument("result"), "", "" - if type(resultname) == "string" then - oldbase = file.removesuffix(jobname) - newbase = file.removesuffix(resultname) - if oldbase ~= newbase then - for _, suffix in pairs(scripts.context.beforesuffixes) do - local oldname = file.addsuffix(oldbase,suffix) - local newname = file.addsuffix(newbase,suffix) - local tmpname = "keep-"..oldname - os.remove(tmpname) - os.rename(oldname,tmpname) - os.remove(oldname) - os.rename(newname,oldname) + if formatfile and scriptfile then + -- we default to mkiv xml ! + local suffix = file.extname(filename) or "?" + if scripts.context.xmlsuffixes[suffix] or environment.argument("forcexml") then + if environment.argument("mkii") then + filename = makestub("\\processXMLfilegrouped{%s}",filename) + else + filename = makestub("\\xmlprocess{\\xmldocument}{%s}{}",filename) + end + elseif scripts.context.luasuffixes[suffix] then + filename = makestub("\\ctxlua{dofile('%s')}",filename) end - else - resultname = nil - end - else - resultname = nil - end - -- - if environment.argument("autopdf") then - os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) - if resultname then - os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) - end - end - -- - local flags = { } - if environment.argument("batchmode") then - flags[#flags+1] = "--interaction=batchmode" - end - flags[#flags+1] = "--fmt=" .. string.quote(formatfile) - flags[#flags+1] = "--lua=" .. string.quote(scriptfile) - local command = string.format("luatex %s %s", table.concat(flags," "), string.quote(filename)) - local oldhash, newhash = scripts.context.multipass.hashfiles(jobname), { } - local once = environment.argument("once") - local maxnofruns = (once and 1) or scripts.context.multipass.nofruns - for i=1,maxnofruns do - -- 1:first run, 2:successive run, 3:once, 4:last of maxruns - local kindofrun = (once and 3) or (i==1 and 1) or (i==maxnofruns and 4) or 2 - scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,i,false) -- kindofrun, currentrun, final - input.report("run %s: %s",i,command) - local returncode, errorstring = os.spawn(command) - if not returncode then - input.report("fatal error, message: %s",errorstring or "?") - os.exit(1) - break - elseif returncode > 0 then - input.report("fatal error, code: %s",returncode or "?") - os.exit(returncode) - break - else - scripts.context.multipass.copyluafile(jobname) - scripts.context.multipass.copytuifile(jobname) - newhash = scripts.context.multipass.hashfiles(jobname) - if scripts.context.multipass.changed(oldhash,newhash) then - oldhash = newhash + -- + -- todo: also other stubs + -- + local resultname, oldbase, newbase = environment.argument("result"), "", "" + if type(resultname) == "string" then + oldbase = file.removesuffix(jobname) + newbase = file.removesuffix(resultname) + if oldbase ~= newbase then + for _, suffix in pairs(scripts.context.beforesuffixes) do + local oldname = file.addsuffix(oldbase,suffix) + local newname = file.addsuffix(newbase,suffix) + local tmpname = "keep-"..oldname + os.remove(tmpname) + os.rename(oldname,tmpname) + os.remove(oldname) + os.rename(newname,oldname) + end + else + resultname = nil + end else - break + resultname = nil + end + -- + if environment.argument("autopdf") then + os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + if resultname then + os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) + end + end + -- + local okay = statistics.check_fmt_status(formatfile) + if okay ~= true then + logs.simple("warning: %s, forcing remake",tostring(okay)) + scripts.context.generate() + scripts.context.make(formatname) + end + -- + local flags = { } + if environment.argument("batchmode") then + flags[#flags+1] = "--interaction=batchmode" + end + flags[#flags+1] = "--fmt=" .. string.quote(formatfile) + flags[#flags+1] = "--lua=" .. string.quote(scriptfile) + flags[#flags+1] = "--backend=pdf" + local command = string.format("luatex %s %s", table.concat(flags," "), string.quote(filename)) + local oldhash, newhash = scripts.context.multipass.hashfiles(jobname), { } + local once = environment.argument("once") + local maxnofruns = (once and 1) or scripts.context.multipass.nofruns + local arrange = environment.argument("arrange") + for i=1,maxnofruns do + -- 1:first run, 2:successive run, 3:once, 4:last of maxruns + local kindofrun = (once and 3) or (i==1 and 1) or (i==maxnofruns and 4) or 2 + scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,i,false) -- kindofrun, currentrun, final + logs.simple("run %s: %s",i,command) + local returncode, errorstring = os.spawn(command) + --~ if returncode == 3 then + --~ scripts.context.generate() + --~ scripts.context.make(formatname) + --~ returncode, errorstring = os.spawn(command) + --~ if returncode == 3 then + --~ logs.simple("fatal error, return code 3, message: %s",errorstring or "?") + --~ os.exit(1) + --~ end + --~ end + if not returncode then + logs.simple("fatal error, no return code, message: %s",errorstring or "?") + os.exit(1) + break + elseif returncode > 0 then + logs.simple("fatal error, return code: %s",returncode or "?") + os.exit(returncode) + break + else + scripts.context.multipass.copyluafile(jobname) + -- scripts.context.multipass.copytuifile(jobname) + newhash = scripts.context.multipass.hashfiles(jobname) + if scripts.context.multipass.changed(oldhash,newhash) then + oldhash = newhash + else + break + end + end + end + -- + if arrange then + local kindofrun = 3 + scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,i,true) -- kindofrun, currentrun, final + logs.simple("arrange run: %s",command) + local returncode, errorstring = os.spawn(command) + if not returncode then + logs.simple("fatal error, no return code, message: %s",errorstring or "?") + os.exit(1) + elseif returncode > 0 then + logs.simple("fatal error, return code: %s",returncode or "?") + os.exit(returncode) + end + end + -- + if environment.argument("purge") then + scripts.context.purge_job(filename) + elseif environment.argument("purgeall") then + scripts.context.purge_job(filename,true) + end + -- + os.remove(jobname..".top") + -- + if resultname then + for _, suffix in pairs(scripts.context.aftersuffixes) do + local oldname = file.addsuffix(oldbase,suffix) + local newname = file.addsuffix(newbase,suffix) + local tmpname = "keep-"..oldname + os.remove(newname) + os.rename(oldname,newname) + os.rename(tmpname,oldname) + end + logs.simple("result renamed to: %s",newbase) + end + -- + if environment.argument("purge") then + scripts.context.purge_job(resultname) + elseif environment.argument("purgeall") then + scripts.context.purge_job(resultname,true) + end + -- + if environment.argument("autopdf") then + if resultname then + os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) + else + os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + end + end + -- + if environment.argument("timing") then + logs.line() + logs.simple("you can process (timing) statistics with:",jobname) + logs.line() + logs.simple("context --extra=timing '%s'",jobname) + logs.simple("mtxrun --script timing --xhtml [--launch --remove] '%s'",jobname) + logs.line() end - end - end - -- - -- todo: extra arrange run - -- - if environment.argument("purge") then - scripts.context.purge_job(filename) - elseif environment.argument("purgeall") then - scripts.context.purge_job(filename,true) - end - -- - if resultname then - for _, suffix in pairs(scripts.context.aftersuffixes) do - local oldname = file.addsuffix(oldbase,suffix) - local newname = file.addsuffix(newbase,suffix) - local tmpname = "keep-"..oldname - os.remove(newname) - os.rename(oldname,newname) - os.rename(tmpname,oldname) - end - input.report("result renamed to: %s",newbase) - end - if environment.argument("autopdf") then - if resultname then - os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(resultname,"pdf"))) else - os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + if formatname then + logs.simple("error, no format found with name: %s, skipping",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end + break end end - -- end else - input.verbose = true - input.report("error, no format found with name: %s",formatname) + if formatname then + logs.simple("error, no format found with name: %s, aborting",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end end end end -function scripts.context.make() - local list = (environment.files[1] and environment.files) or scripts.context.defaultformats +function scripts.context.pipe() + -- context --pipe + -- context --pipe --purge --dummyfile=whatever.tmp + local interface = environment.argument("interface") + interface = (type(interface) == "string" and interface) or "en" + local formatname = scripts.context.interfaces[interface] or "cont-en" + local formatfile, scriptfile = resolvers.locate_format(formatname) + if not formatfile or not scriptfile then + logs.simple("warning: no format found, forcing remake (commandline driven)") + scripts.context.generate() + scripts.context.make(formatname) + formatfile, scriptfile = resolvers.locate_format(formatname) + end + if formatfile and scriptfile then + local okay = statistics.check_fmt_status(formatfile) + if okay ~= true then + logs.simple("warning: %s, forcing remake",tostring(okay)) + scripts.context.generate() + scripts.context.make(formatname) + end + local flags = { + "--interaction=scrollmode", + "--fmt=" .. string.quote(formatfile), + "--lua=" .. string.quote(scriptfile), + "--backend=pdf", + } + local filename = environment.argument("dummyfile") or "" + if filename == "" then + filename = "\\relax" + logs.simple("entering scrollmode, end job with \\end") + else + filename = file.addsuffix(filename,"tmp") + io.savedata(filename,"\\relax") + scripts.context.multipass.makeoptionfile(filename,{ flags = flags },3,1,false) -- kindofrun, currentrun, final + logs.simple("entering scrollmode using '%s' with optionfile, end job with \\end",filename) + end + local command = string.format("luatex %s %s", table.concat(flags," "), string.quote(filename)) + os.spawn(command) + if environment.argument("purge") then + scripts.context.purge_job(filename) + elseif environment.argument("purgeall") then + scripts.context.purge_job(filename,true) + os.remove(filename) + end + else + if formatname then + logs.simple("error, no format found with name: %s, aborting",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end + end +end + +function scripts.context.make(name) + local runners = { + "luatools --make --compile ", + (environment.argument("pdftex") and "mtxrun texexec.rb --make --pdftex ") or false, + (environment.argument("xetex") and "mtxrun texexec.rb --make --xetex " ) or false, + } + local list = (name and { name }) or (environment.files[1] and environment.files) or scripts.context.defaultformats for _, name in ipairs(list) do name = scripts.context.interfaces[name] or name - local command = "luatools --make --compile " .. name - input.report("running command: %s",command) - os.spawn(command) + for _, runner in ipairs(runners) do + if runner then + local command = runner .. name + logs.simple("running command: %s",command) + os.spawn(command) + end + end end end function scripts.context.generate() -- hack, should also be a shared function local command = "luatools --generate " - input.report("running command: %s",command) + logs.simple("running command: %s",command) os.spawn(command) end @@ -725,23 +907,81 @@ function scripts.context.ctx() scripts.context.run(ctxdata) end +function scripts.context.autoctx() + local ctxdata = nil + local files = (filename and { filename }) or environment.files + local firstfile = #files > 0 and files[1] + if firstfile and file.extname(firstfile) == "xml" then + local f = io.open(firstfile) + if f then + local chunk = f:read(512) or "" + f:close() + local ctxname = string.match(chunk,"<%?context%-directive%s+job%s+ctxfile%s+([^ ]-)%s*?>") + if ctxname then + ctxdata = ctxrunner.new() + ctxdata.jobname = firstfile + ctxrunner.manipulate(ctxdata,ctxname) + end + end + end + scripts.context.run(ctxdata) +end + +-- todo: quite after first image + +local template = [[ + \starttext + \startMPpage %% %s + input "%s" ; + \stopMPpage + \stoptext +]] + +local loaded = false + +function scripts.context.metapost() + local filename = environment.files[1] or "" +--~ local tempname = "mtx-context-metapost.tex" +--~ local tempdata = string.format(template,"metafun",filename) +--~ io.savedata(tempname,tempdata) +--~ environment.files[1] = tempname +--~ environment.setargument("result",file.removesuffix(filename)) +--~ environment.setargument("once",true) +--~ scripts.context.run() + if not loaded then + dofile(resolvers.find_file("mlib-run.lua")) + loaded = true + commands = commands or { } + commands.writestatus = logs.report + end + local formatname = environment.arguments("format") or "metafun" + if formatname == "" or type(format) == "boolean" then + formatname = "metafun" + end + if environment.arguments("svg") then + metapost.directrun(formatname,filename,"svg") + else + metapost.directrun(formatname,filename,"mps") + end +end + function scripts.context.version() - local name = input.find_file("context.tex") + local name = resolvers.find_file("context.tex") if name ~= "" then - input.report("main context file: %s",name) + logs.simple("main context file: %s",name) local data = io.loaddata(name) if data then local version = data:match("\\edef\\contextversion{(.-)}") if version then - input.report("current version: %s",version) + logs.simple("current version: %s",version) else - input.report("context version: unknown, no timestamp found") + logs.simple("context version: unknown, no timestamp found") end else - input.report("context version: unknown, load error") + logs.simple("context version: unknown, load error") end else - input.report("main context file: unknown, 'context.tex' not found") + logs.simple("main context file: unknown, 'context.tex' not found") end end @@ -780,6 +1020,7 @@ local function purge_file(dfile,cfile) end function scripts.context.purge_job(jobname,all) + jobname = file.basename(jobname) local filebase = file.removesuffix(jobname) local deleted = { } for _, suffix in ipairs(obsolete_results) do @@ -794,7 +1035,7 @@ function scripts.context.purge_job(jobname,all) end end if #deleted > 0 then - input.report("purged files: %s", table.join(deleted,", ")) + logs.simple("purged files: %s", table.join(deleted,", ")) end end @@ -815,56 +1056,260 @@ function scripts.context.purge(all) end end if #deleted > 0 then - input.report("purged files: %s", table.join(deleted,", ")) + logs.simple("purged files: %s", table.join(deleted,", ")) end end --~ purge_for_files("test",true) --~ purge_all_files() +local function touch(name,pattern) + local name = resolvers.find_file(name) + local olddata = io.loaddata(name) + if olddata then + local oldversion, newversion = "", os.date("%Y.%m.%d %H:%M") + local newdata, ok = olddata:gsub(pattern,function(pre,mid,post) + oldversion = mid + return pre .. newversion .. post + end) + if ok > 0 then + local backup = file.replacesuffix(name,"tmp") + os.remove(backup) + os.rename(name,backup) + io.savedata(name,newdata) + return true, oldversion, newversion, name + else + return false + end + end +end + function scripts.context.touch() if environment.argument("expert") then - local function touch(name,pattern) - local name = input.find_file(name) - local olddata = io.loaddata(name) - if olddata then - local oldversion, newversion = "", os.date("%Y.%M.%d %H:%m") - local newdata, ok = olddata:gsub(pattern,function(pre,mid,post) - oldversion = mid - return pre .. newversion .. post - end) - if ok > 0 then - local backup = file.replacesuffix(name,"tmp") - os.remove(backup) - os.rename(name,backup) - io.savedata(name,newdata) - return true, oldversion, newversion, name - else - return false - end - end - end local done, oldversion, newversion, foundname = touch("context.tex", "(\\edef\\contextversion{)(.-)(})") if done then - input.report("old version : %s", oldversion) - input.report("new version : %s", newversion) - input.report("touched file: %s", foundname) + logs.simple("old version : %s", oldversion) + logs.simple("new version : %s", newversion) + logs.simple("touched file: %s", foundname) local ok, _, _, foundname = touch("cont-new.tex", "(\\newcontextversion{)(.-)(})") if ok then - input.report("touched file: %s", foundname) + logs.simple("touched file: %s", foundname) + end + local ok, _, _, foundname = touch("cont-xp.tex", "(\\edef\\contextversion{)(.-)(})") + if ok then + logs.simple("touched file: %s", foundname) + end + end + end +end + +-- extras + +function scripts.context.extras(pattern) + local found = resolvers.find_file("context.tex") + if found == "" then + logs.simple("unknown extra: %s", extra) + else + pattern = file.join(dir.expand_name(file.dirname(found)),string.format("mtx-context-%s.tex",pattern or "*")) + local list = dir.glob(pattern) + if not extra or extra == "" then + logs.extendbanner("extras") + else + logs.extendbanner(extra) + end + for k,v in ipairs(list) do + local data = io.loaddata(v) or "" + data = string.match(data,"begin help(.-)end help") + if data then + local h = { string.format("extra: %s (%s)",string.gsub(v,"^.*mtx%-context%-(.-)%.tex$","%1"),v) } + for s in string.gmatch(data,"%% *(.-)[\n\r]") do + h[#h+1] = s + end + logs.help(table.concat(h,"\n"),"nomoreinfo") + end + end + end +end + +function scripts.context.extra() + local extra = environment.argument("extra") + if type(extra) == "string" then + if environment.argument("help") then + scripts.context.extras(extra) + else + local fullextra = extra + if not string.find(fullextra,"mtx%-context%-") then + fullextra = "mtx-context-" .. extra + end + local foundextra = resolvers.find_file(fullextra) + if foundextra == "" then + scripts.context.extras() + return + else + logs.simple("processing extra: %s", foundextra) end + environment.setargument("purgeall",true) + local result = environment.setargument("result") or "" + if result == "" then + environment.setargument("result","context-extra") + end + scripts.context.run(nil,foundextra) end + else + scripts.context.extras() end end +-- todo: we need to do a dummy run + +function scripts.context.track() + environment.files = { "m-track" } + scripts.context.multipass.nofruns = 1 + scripts.context.run() + -- maybe filter from log +end + function scripts.context.timed(action) - input.starttiming(scripts.context) - action() - input.stoptiming(scripts.context) - input.report("total runtime: %s",input.elapsedtime(scripts.context)) + statistics.timed(action) end -banner = banner .. " | context tools " +local zipname = "cont-tmf.zip" +local mainzip = "http://www.pragma-ade.com/context/latest/" .. zipname +local validtrees = { "texmf-local", "texmf-context" } + +function zip.loaddata(zipfile,filename) -- should be in zip lib + local f = zipfile:open(filename) + if f then + local data = f:read("*a") + f:close() + return data + end + return nil +end + +function scripts.context.update() + local force = environment.argument("force") + local socket = require("socket") + local http = require("socket.http") + local basepath = resolvers.find_file("context.tex") or "" + if basepath == "" then + logs.simple("quiting, no 'context.tex' found") + return + end + local basetree = basepath.match(basepath,"^(.-)tex/context/base/context.tex$") or "" + if basetree == "" then + logs.simple("quiting, no proper tds structure (%s)",basepath) + return + end + local function is_okay(basetree) + for _, tree in next, validtrees do + local pattern = string.gsub(tree,"%-","%%-") + if basetree:find(pattern) then + return tree + end + end + return false + end + local okay = is_okay(basetree) + if not okay then + logs.simple("quiting, tree '%s' is protected",okay) + return + else + logs.simple("updating tree '%s'",okay) + end + if not lfs.chdir(basetree) then + logs.simple("quiting, unable to change to '%s'",okay) + return + end + logs.simple("fetching '%s'",mainzip) + local latest = http.request(mainzip) + if not latest then + logs.simple("context tree '%s' can be updated, use --force",okay) + return + end + io.savedata("cont-tmf.zip",latest) + if false then + -- variant 1 + os.execute("mtxrun --script unzip cont-tmf.zip") + else + -- variant 2 + local zipfile = zip.open(zipname) + if not zipfile then + logs.simple("quiting, unable to open '%s'",zipname) + return + end + local newfile = zip.loaddata(zipfile,"tex/context/base/context.tex") + if not newfile then + logs.simple("quiting, unable to open '%s'","context.tex") + return + end + local oldfile = io.loaddata(resolvers.find_file("context.tex")) or "" + local function versiontonumber(what,str) + local version = str:match("\\edef\\contextversion{(.-)}") or "" + local year, month, day, hour, minute = str:match("\\edef\\contextversion{(%d+)%.(%d+)%.(%d+) *(%d+)%:(%d+)}") + if year and minute then + local time = os.time { year=year,month=month,day=day,hour=hour,minute=minute} + logs.simple("%s version: %s (%s)",what,version,time) + return time + else + logs.simple("%s version: %s (unknown)",what,version) + return nil + end + end + local oldversion = versiontonumber("old",oldfile) + local newversion = versiontonumber("new",newfile) + if not oldversion or not newversion then + logs.simple("quiting, version cannot be determined") + return + elseif oldversion == newversion then + logs.simple("quiting, your current version is up-to-date") + return + elseif oldversion > newversion then + logs.simple("quiting, your current version is newer") + return + end + for k in zipfile:files() do + local filename = k.filename + if filename:find("/$") then + lfs.mkdir(filename) + else + local data = zip.loaddata(zipfile,filename) + if data then + if force then + io.savedata(filename,data) + end + logs.simple(filename) + end + end + end + for _, scriptname in next, { "luatools.lua", "mtxrun.lua" } do + local oldscript = resolvers.find_file(scriptname) or "" + if oldscript ~= "" and is_okay(oldscript) then + local newscript = "./scripts/context/lua/" .. scriptname + local data = io.loaddata(newscript) or "" + if data ~= "" then + logs.simple("replacing script '%s' by '%s'",oldscript,newscript) + if force then + io.savedata(oldscript,data) + end + end + else + logs.simple("keeping script '%s'",oldscript) + end + end + if force then + os.execute("context --generate") + os.execute("context --make") + end + end + if force then + logs.simple("context tree '%s' has been updated",okay) + else + logs.simple("context tree '%s' can been updated (use --force)",okay) + end +end + +logs.extendbanner("ConTeXt Tools 0.51",true) messages.help = [[ --run process (one or more) files (default action) @@ -877,41 +1322,82 @@ messages.help = [[ --once only one run --purge(all) purge files (--pattern=...) --result=name rename result to given name +--arrange run extra arrange pass --expert expert options --interface use specified user interface ]] messages.expert = [[ -expert options: also provide --expert +expert options: + +--touch update context version number (remake needed afterwards, also provide --expert) +--update update context from website (not to be confused with contextgarden) +--profile profile job (use: mtxrun --script profile --analyse) +--track show/set tracker variables +--timing generate timing and statistics overview +--extra=name process extra (mtx-context-<name> in distribution) +]] + +messages.private = [[ +private options: ---touch update context version (remake needed afterwards) +--dumphash dump hash table afterwards +--dumpdelta dump hash table afterwards (only new entries) ]] -input.verbose = true +messages.special = [[ +special options: + +--pdftex process file with texexec using pdftex +--xetex process file with texexec using xetex + +--pipe don't check for file and enter scroll mode (--dummyfile=whatever.tmp) +]] if environment.argument("once") then scripts.context.multipass.nofruns = 1 end +if environment.argument("profile") then + os.setenv("MTX_PROFILE_RUN","YES") +end + if environment.argument("run") then - scripts.context.timed(scripts.context.run) -elseif environment.argument("make") then - scripts.context.timed(scripts.context.make) -elseif environment.argument("generate") then - scripts.context.timed(scripts.context.generate) +-- scripts.context.timed(scripts.context.run) + scripts.context.timed(scripts.context.autoctx) +elseif environment.argument("make") or environment.argument("generate") then + scripts.context.timed(function() + if environment.argument("generate") then + scripts.context.generate() + end + if environment.argument("make") then + scripts.context.make() + end + end) elseif environment.argument("ctx") then scripts.context.timed(scripts.context.ctx) +elseif environment.argument("mp") or environment.argument("metapost") then + scripts.context.timed(scripts.context.metapost) elseif environment.argument("version") then scripts.context.version() elseif environment.argument("touch") then scripts.context.touch() +elseif environment.argument("update") then + scripts.context.update() elseif environment.argument("expert") then - input.help(banner,messages.expert) + logs.help(table.join({ messages.expert, messages.private, messages.special },"\n")) +elseif environment.argument("extra") then + scripts.context.extra() elseif environment.argument("help") then - input.help(banner,messages.help) + logs.help(messages.help) +elseif environment.argument("track") and type(environment.argument("track")) == "boolean" then + scripts.context.track() elseif environment.files[1] then - scripts.context.timed(scripts.context.run) +-- scripts.context.timed(scripts.context.run) + scripts.context.timed(scripts.context.autoctx) +elseif environment.argument("pipe") then + scripts.context.timed(scripts.context.pipe) elseif environment.argument("purge") then -- only when no filename given, supports --pattern scripts.context.purge() @@ -919,6 +1405,9 @@ elseif environment.argument("purgeall") then -- only when no filename given, supports --pattern scripts.context.purge(true) else - input.help(banner,messages.help) + logs.help(messages.help) end +if environment.argument("profile") then + os.setenv("MTX_PROFILE_RUN","NO") +end diff --git a/scripts/context/lua/mtx-convert.lua b/scripts/context/lua/mtx-convert.lua index eca050f29..cf1d640c5 100644 --- a/scripts/context/lua/mtx-convert.lua +++ b/scripts/context/lua/mtx-convert.lua @@ -51,7 +51,7 @@ do dir.mkdirs(outputpath) local tmpname = file.replacesuffix(newname,"tmp") local command = graphics.converters[suffix](oldname,tmpname) - input.report("command: %s",command) + logs.simple("command: %s",command) io.flush() os.spawn(command) os.remove(newname) @@ -88,7 +88,7 @@ function scripts.convert.convertall() end end -banner = banner .. " | graphic conversion tools " +logs.extendbanner("Graphic Conversion Tools 0.10",true) messages.help = [[ --convertall convert all graphics on path @@ -98,10 +98,8 @@ messages.help = [[ --delay time between sweeps ]] -input.verbose = true - if environment.argument("convertall") then scripts.convert.convertall() else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua index ef9e37258..befba924e 100644 --- a/scripts/context/lua/mtx-fonts.lua +++ b/scripts/context/lua/mtx-fonts.lua @@ -6,7 +6,11 @@ if not modules then modules = { } end modules ['mtx-fonts'] = { license = "see context related readme files" } -dofile(input.find_file("font-syn.lua")) +if not fontloader then fontloader = fontforge end + +dofile(resolvers.find_file("font-otp.lua","tex")) +dofile(resolvers.find_file("font-syn.lua","tex")) +dofile(resolvers.find_file("font-mis.lua","tex")) scripts = scripts or { } scripts.fonts = scripts.fonts or { } @@ -15,57 +19,55 @@ function scripts.fonts.reload(verbose) fonts.names.load(true,verbose) end +function scripts.fonts.names(name) + name = name or "luatex-fonts-names.lua" + fonts.names.identify(true) + local data = fonts.names.data + if data then + data.fallback_mapping = nil + logs.report("fontnames","saving names in '%s'",name) + io.savedata(name,table.serialize(data,true)) + elseif lfs.isfile(name) then + os.remove(name) + end +end + local function showfeatures(v,n,f,s,t) - local iv = input.verbose - input.verbose = true - input.report("fontname: %s",v) - input.report("fullname: %s",n) - input.report("filename: %s",f) - if t == "otf" or t == "ttf" then - local filename = input.find_file(f,t) or "" - if filename ~= "" then - local ff = fontforge.open(filename) - if ff then - local data = fontforge.to_table(ff) - fontforge.close(ff) - local features = { } - local function collect(what) - if data[what] then - for _, d in ipairs(data[what]) do - if d.features then - for _, df in ipairs(d.features) do - features[df.tag] = features[df.tag] or { } - for _, ds in ipairs(df.scripts) do - features[df.tag][ds.script] = features[df.tag][ds.script] or { } - for _, lang in ipairs(ds.langs) do - features[df.tag][ds.script][lang] = true - end - end - end - end + logs.simple("fontname: %s",v) + logs.simple("fullname: %s",n) + logs.simple("filename: %s",f) + local features = fonts.get_features(f,t) + if features then + for what, v in table.sortedpairs(features) do + local data = features[what] + if data and next(data) then + logs.simple() + logs.simple("%s features:",what) + logs.simple() + logs.simple("feature script languages") + logs.simple() + for f,ff in table.sortedpairs(data) do + local done = false + for s, ss in table.sortedpairs(ff) do + if s == "*" then s = "all" end + if ss ["*"] then ss["*"] = nil ss.all = true end + if done then + f = "" + else + done = true end - end - end - collect('gsub') - collect('gpos') - input.report("") - for _, f in ipairs(table.sortedkeys(features)) do - local ff = features[f] - for _, s in ipairs(table.sortedkeys(ff)) do - local ss = ff[s] - input.report("feature: %s, script: %s, language: %s",f:lower(),s:lower(),(table.concat(table.sortedkeys(ss), " ")):lower()) + logs.simple("% -8s % -8s % -8s",f,s,table.concat(table.sortedkeys(ss), " ")) end end end end end - input.report("") - input.verbose = iv + logs.reportline() end function scripts.fonts.list(pattern,reload,all,info) if reload then - input.report("fontnames, reloading font database") + logs.simple("fontnames, reloading font database") end -- make a function for this pattern = pattern:lower() @@ -81,7 +83,7 @@ function scripts.fonts.list(pattern,reload,all,info) -- local t = fonts.names.list(pattern,reload) if reload then - input.report("fontnames, done\n\n") + logs.simple("fontnames, done\n\n") end if t then local s, w = table.sortedkeys(t), { 0, 0, 0 } @@ -112,51 +114,52 @@ function scripts.fonts.save(name,sub) local function save(savename,fontblob) if fontblob then savename = savename:lower() .. ".lua" - input.report("fontsave, saving data in %s",savename) - table.tofile(savename,fontforge.to_table(fontblob),"return") - fontforge.close(fontblob) + logs.simple("fontsave, saving data in %s",savename) + table.tofile(savename,fontloader.to_table(fontblob),"return") + fontloader.close(fontblob) end end if name and name ~= "" then - local filename = input.find_file(name) -- maybe also search for opentype + local filename = resolvers.find_file(name) -- maybe also search for opentype if filename and filename ~= "" then local suffix = file.extname(filename) if suffix == 'ttf' or suffix == 'otf' or suffix == 'ttc' then - local fontinfo = fontforge.info(filename) + local fontinfo = fontloader.info(filename) if fontinfo then + logs.simple("font: %s located as %s",name,filename) if fontinfo[1] then for _, v in ipairs(fontinfo) do - save(v.fontname,fontforge.open(filename,v.fullname)) + save(v.fontname,fontloader.open(filename,v.fullname)) end else - save(fontinfo.fullname,fontforge.open(filename)) + save(fontinfo.fullname,fontloader.open(filename)) end end else - input.verbose = true - input.report("font: %s not saved",filename) + logs.simple("font: %s not saved",filename) end else - input.verbose = true - input.report("font: %s not found",name) + logs.simple("font: %s not found",name) end end end -banner = banner .. " | font tools " +logs.extendbanner("Font Tools 0.20",true) messages.help = [[ --reload generate new font database --list [--info] list installed fonts (show info) --save save open type font in raw table +--names generate 'luatex-fonts-names.lua' (not for context!) --pattern=str filter files --all provide alternatives ]] if environment.argument("reload") then - local verbose = environment.argument("verbose") - scripts.fonts.reload(verbose) + scripts.fonts.reload(true) +elseif environment.argument("names") then + scripts.fonts.names() elseif environment.argument("list") then local pattern = environment.argument("pattern") or environment.files[1] or "" local all = environment.argument("all") @@ -168,5 +171,5 @@ elseif environment.argument("save") then local sub = environment.files[2] or "" scripts.fonts.save(name,sub) else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-grep.lua b/scripts/context/lua/mtx-grep.lua index 18e36d2ea..82a80314a 100644 --- a/scripts/context/lua/mtx-grep.lua +++ b/scripts/context/lua/mtx-grep.lua @@ -9,54 +9,85 @@ if not modules then modules = { } end modules ['mtx-babel'] = { scripts = scripts or { } scripts.grep = scripts.grep or { } -banner = banner .. " | simple grepper " +logs.extendbanner("Simple Grepper 0.10",true) + +local find, format = string.find, string.format + +local cr = lpeg.P("\r") +local lf = lpeg.P("\n") +local crlf = cr * lf +local newline = crlf + cr + lf +local content = lpeg.C((1-newline)^0) * newline + +local write_nl = texio.write_nl function scripts.grep.find(pattern, files, offset) if pattern and pattern ~= "" then - local format = string.format - input.starttiming(scripts.grep) - local count, nofmatches, noffiles, nofmatchedfiles = environment.argument("count"), 0, 0, 0 - local function grep(name) - local data = io.loaddata(name) - if data then - noffiles = noffiles + 1 - local n, m = 0, 0 - for line in data:gmatch("[^\n]+") do -- faster than loop over lines + statistics.starttiming(scripts.grep) + local nofmatches, noffiles, nofmatchedfiles = 0, 0, 0 + local n, m, name, check = 0, 0, "", nil + local count, nocomment = environment.argument("count"), environment.argument("nocomment") + if nocomment then + if count then + check = function(line) n = n + 1 - if line:find(pattern) then + if find(line,"^[%%#]") then + -- skip + elseif find(line,pattern) then m = m + 1 - if not count then - input.log(format("%s %s: %s",name,n,line)) - io.flush() - end end end - if count and m > 0 then - nofmatches = nofmatches + m - nofmatchedfiles = nofmatchedfiles + 1 - input.log(format("%s: %s",name,m)) - io.flush() + else + check = function(line) + n = n + 1 + if find(line,"^[%%#]") then + -- skip + elseif find(line,pattern) then + m = m + 1 + write_nl(format("%s %s: %s",name,n,line)) + io.flush() + end + end + end + else + if count then + check = function(line) + n = n + 1 + if find(line,pattern) then + m = m + 1 + end + end + else + check = function(line) + n = n + 1 + if find(line,pattern) then + m = m + 1 + write_nl(format("%s %s: %s",name,n,line)) + io.flush() + end end end end ---~ for i=offset or 1, #files do ---~ local filename = files[i] ---~ if filename:find("%*") then ---~ for _, name in ipairs(dir.glob(filename)) do ---~ grep(name) ---~ end ---~ else ---~ grep(filename) ---~ end ---~ end + local capture = (content/check)^0 for i=offset or 1, #files do - for _, name in ipairs(dir.glob(files[i])) do - grep(name) + for _, nam in ipairs(dir.glob(files[i])) do + name = nam + local data = io.loaddata(name) + if data then + n, m, noffiles = 0, 0, noffiles + 1 + capture:match(data) + if count and m > 0 then + nofmatches = nofmatches + m + nofmatchedfiles = nofmatchedfiles + 1 + write_nl(format("%s: %s",name,m)) + io.flush() + end + end end end - input.stoptiming(scripts.grep) + statistics.stoptiming(scripts.grep) if count and nofmatches > 0 then - input.log(format("\nfiles: %s, matches: %s, matched files: %s, runtime: %0.3f seconds",noffiles,nofmatches,nofmatchedfiles,input.loadtime(scripts.grep))) + write_nl(format("\nfiles: %s, matches: %s, matched files: %s, runtime: %0.3f seconds",noffiles,nofmatches,nofmatchedfiles,statistics.elapsedtime(scripts.grep))) end end end @@ -64,9 +95,10 @@ end messages.help = [[ --pattern search for pattern (optional) --count count matches only -]] +--nocomment skip lines that start with %% or # -input.verbose = true +patterns are lua patterns and need to be escaped accordingly +]] local pattern = environment.argument("pattern") local files = environment.files and #environment.files > 0 and environment.files @@ -76,5 +108,5 @@ if pattern and files then elseif files then scripts.grep.find(files[1], files, 2) else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-interface.lua b/scripts/context/lua/mtx-interface.lua index b57617846..264a2dbe4 100644 --- a/scripts/context/lua/mtx-interface.lua +++ b/scripts/context/lua/mtx-interface.lua @@ -11,7 +11,9 @@ local format = string.format scripts = scripts or { } scripts.interface = scripts.interface or { } -local flushers = { } +local flushers = { } +local userinterfaces = { 'en','cs','de','it','nl','ro','fr','pe' } +local messageinterfaces = { 'en','cs','de','it','nl','ro','fr','pe','no' } function flushers.scite(interface,collection) local result, i = {}, 0 @@ -59,25 +61,23 @@ end function flushers.raw(interface,collection) for _, command in ipairs(collection) do - input.report(command) + logs.simple(command) end end function scripts.interface.editor(editor) local interfaces= environment.files if #interfaces == 0 then - interfaces= { 'en','cs','de','it','nl','ro','fr' } + interfaces= userinterfaces end - local xmlfile = input.find_file("cont-en.xml") or "" + local xmlfile = resolvers.find_file("cont-en.xml") or "" if xmlfile == "" then - input.verbose = true - input.report("unable to locate cont-en.xml") + logs.simple("unable to locate cont-en.xml") end for _, interface in ipairs(interfaces) do - local keyfile = input.find_file(format("keys-%s.xml",interface)) or "" + local keyfile = resolvers.find_file(format("keys-%s.xml",interface)) or "" if keyfile == "" then - input.verbose = true - input.report("unable to locate keys-*.xml") + logs.simple("unable to locate keys-*.xml") else local collection = { } local mappings = { } @@ -112,7 +112,7 @@ function scripts.interface.editor(editor) end function scripts.interface.check() - local xmlfile = input.find_file("cont-en.xml") or "" + local xmlfile = resolvers.find_file("cont-en.xml") or "" if xmlfile ~= "" then local f = io.open("cont-en-check.tex","w") if f then @@ -138,14 +138,12 @@ function scripts.interface.check() end function scripts.interface.context() - local verbose = input.verbose - input.verbose = true - local filename = input.find_file("mult-def.lua") or "" + local filename = resolvers.find_file("mult-def.lua") or "" if filename ~= "" then local interface = dofile(filename) if interface and next(interface) then local variables, constants, commands, elements = interface.variables, interface.constants, interface.commands, interface.elements - local filename = input.find_file("cont-en.xml") or "" + local filename = resolvers.find_file("cont-en.xml") or "" local xmldata = filename ~= "" and (io.loaddata(filename) or "") local function flush(texresult,xmlresult,language,what,tag) local t = interface[what] @@ -156,7 +154,7 @@ function scripts.interface.context() local v = t[key] local value = v[language] or v["en"] if not value then - input.report(format("warning, no value for key '%s' for language '%s'",key,language)) + logs.simple(format("warning, no value for key '%s' for language '%s'",key,language)) else local value = t[key][language] or t[key].en texresult[#texresult+1] = format("\\setinterface%s{%s}{%s}",tag,key,value) @@ -194,9 +192,9 @@ function scripts.interface.context() local texfilename = format("mult-%s.tex",language) local xmlfilename = format("keys-%s.xml",language) io.savedata(texfilename,table.concat(texresult,"\n")) - input.report(format("saving interface definitions '%s'",texfilename)) + logs.simple(format("saving interface definitions '%s'",texfilename)) io.savedata(xmlfilename,table.concat(xmlresult,"\n")) - input.report(format("saving interface translations '%s'",xmlfilename)) + logs.simple(format("saving interface translations '%s'",xmlfilename)) if language ~= "en" and xmldata ~= "" then local newdata = xmldata:gsub("(<cd:interface.*language=.)en(.)","%1"..language.."%2",1) newdata = replace(newdata, 'cd:string', 'value', interface.commands, interface.elements, language) @@ -207,17 +205,36 @@ function scripts.interface.context() newdata = replace(newdata, 'cd:inherit', 'name', interface.commands, interface.elements, language) local xmlfilename = format("cont-%s.xml",language) io.savedata(xmlfilename,newdata) - input.report(format("saving interface specification '%s'",xmlfilename)) + logs.simple(format("saving interface specification '%s'",xmlfilename)) end end end end - input.verbose = verbose end +function scripts.interface.messages() + local filename = resolvers.find_file("mult-mes.lua") or "" + if filename ~= "" then + local messages = dofile(filename) + for _, interface in ipairs(messageinterfaces) do + local texresult = { } + for category, data in next, messages do + for tag, message in next, data do + if tag ~= "files" then + local msg = message[interface] or message["all"] or message["en"] + if msg then + texresult[#texresult+1] = format("\\setinterfacemessage{%s}{%s}{%s}",category,tag,msg) + end + end + end + end + texresult[#texresult+1] = format("%%\n\\endinput") + io.savedata(format("mult-m%s.tex",interface),table.concat(texresult,"\n")) + end + end +end - -banner = banner .. " | interface tools " +logs.extendbanner("Interface Tools 0.11",true) messages.help = [[ --scite generate scite interface @@ -225,10 +242,13 @@ messages.help = [[ --jedit generate scite interface --check generate check file --context generate context definition files +--messages generate context message files ]] if environment.argument("context") then scripts.interface.context() +elseif environment.argument("messages") then + scripts.interface.messages() elseif environment.argument("scite") or environment.argument("bbedit") or environment.argument("jedit") then if environment.argument("scite") then scripts.interface.editor("scite") @@ -242,5 +262,5 @@ elseif environment.argument("scite") or environment.argument("bbedit") or enviro elseif environment.argument("check") then scripts.interface.check() else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-metatex.lua b/scripts/context/lua/mtx-metatex.lua new file mode 100644 index 000000000..f8c871a7b --- /dev/null +++ b/scripts/context/lua/mtx-metatex.lua @@ -0,0 +1,69 @@ +if not modules then modules = { } end modules ['mtx-metatex'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- future versions will deal with specific variants of metatex + +scripts = scripts or { } +scripts.metatex = scripts.metatex or { } + +-- metatex + +function scripts.metatex.make() + local command = "luatools --make --compile metatex" + logs.simple("running command: %s",command) + os.spawn(command) +end + +--~ function scripts.metatex.run() +--~ local name = environment.files[1] or "" +--~ if name ~= "" then +--~ local command = "luatools --fmt=metatex " .. name +--~ logs.simple("running command: %s",command) +--~ os.spawn(command) +--~ end +--~ end + +function scripts.metatex.run(ctxdata,filename) + local filename = environment.files[1] or "" + if filename ~= "" then + local formatfile, scriptfile = resolvers.locate_format("metatex") + if formatfile and scriptfile then + local command = string.format("luatex --fmt=%s --lua=%s %s", + string.quote(formatfile), string.quote(scriptfile), string.quote(filename)) + logs.simple("running command: %s",command) + os.spawn(command) + elseif formatname then + logs.simple("error, no format found with name: %s",formatname) + else + logs.simple("error, no format found (provide formatname or interface)") + end + end +end + +function scripts.metatex.timed(action) + statistics.timed(action) +end + +logs.extendbanner("MetaTeX Tools 0.10",true) + +messages.help = [[ +--run process (one or more) files (default action) +--make create metatex format(s) +]] + +if environment.argument("run") then + scripts.metatex.timed(scripts.metatex.run) +elseif environment.argument("make") then + scripts.metatex.timed(scripts.metatex.make) +elseif environment.argument("help") then + logs.help(messages.help,false) +elseif environment.files[1] then + scripts.metatex.timed(scripts.metatex.run) +else + logs.help(messages.help,false) +end diff --git a/scripts/context/lua/mtx-mptopdf.lua b/scripts/context/lua/mtx-mptopdf.lua index 0c685a249..4243625ad 100644 --- a/scripts/context/lua/mtx-mptopdf.lua +++ b/scripts/context/lua/mtx-mptopdf.lua @@ -74,7 +74,7 @@ do end end local runner = mpbin .. rest .. fn - input.report("running: %s\n", runner) + logs.simple("running: %s\n", runner) return (os.execute(runner)) end @@ -93,7 +93,7 @@ function scripts.mptopdf.convertall() if scripts.mptopdf.aux.make_mps(fn,latex,rawmp,metafun) then files = dir.glob(file.nameonly(fn) .. ".*") -- reset else - input.report("error while processing mp file '%s'", fn) + logs.simple("error while processing mp file '%s'", fn) exit(1) end local report = { } @@ -104,20 +104,20 @@ function scripts.mptopdf.convertall() end end if #report > 0 then - input.report("number of converted files: %i", #report) - input.report("") + logs.simple("number of converted files: %i", #report) + logs.simple("") for _, r in ipairs(report) do - input.report("%s => %s", r[1], r[2]) + logs.simple("%s => %s", r[1], r[2]) end else - input.report("no input files match %s", table.concat(files,' ')) + logs.simple("no input files match %s", table.concat(files,' ')) end else - input.report("no files match %s", table.concat(environment.files,' ')) + logs.simple("no files match %s", table.concat(environment.files,' ')) end end -banner = banner .. " | mptopdf converter " +logs.extendbanner("MetaPost to PDF Converter 0.51",true) messages.help = [[ --rawmp raw metapost run @@ -125,14 +125,12 @@ messages.help = [[ --latex force --tex=latex ]] -input.verbose = true - if environment.files[1] then scripts.mptopdf.convertall() else if not environment.arguments.help then - input.report("provide MP output file (or pattern)") - input.report("") + logs.simple("provide MP output file (or pattern)") + logs.simple("") end - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtx-package.lua b/scripts/context/lua/mtx-package.lua new file mode 100644 index 000000000..06c89907a --- /dev/null +++ b/scripts/context/lua/mtx-package.lua @@ -0,0 +1,68 @@ +if not modules then modules = { } end modules ['mtx-package'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, gsub, gmatch = string.format, string.gsub, string.gmatch + +scripts = scripts or { } +messages = messages or { } +scripts.package = scripts.package or { } + +function scripts.package.merge_luatex_files(name,strip) + local oldname = resolvers.find_file(name) or "" + oldname = file.replacesuffix(oldname,"lua") + if oldname == "" then + logs.simple("missing '%s'",name) + else + local newname = file.removesuffix(oldname) .. "-merged.lua" + local data = io.loaddata(oldname) or "" + if data == "" then + logs.simple("missing '%s'",newname) + else + logs.simple("loading '%s'",oldname) + local collected = { } + collected[#collected+1] = format("-- merged file : %s\n",newname) + collected[#collected+1] = format("-- parent file : %s\n",oldname) + collected[#collected+1] = format("-- merge date : %s\n",os.date()) + -- loadmodule can have extra arguments + for lib in gmatch(data,"loadmodule *%([\'\"](.-)[\'\"]") do + if file.basename(lib) ~= file.basename(newname) then + local fullname = resolvers.find_file(lib) or "" + if fullname == "" then + logs.simple("missing '%s'",lib) + else + logs.simple("fetching '%s'",fullname) + local data = io.loaddata(fullname) + if strip then + data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-%ldx%]%]%-%-[\n\r]*","") + data = gsub(data,"%-%-%~[^\n\r]*[\n\r]*","\n") + data = gsub(data,"%s+%-%-[^\n\r]*[\n\r]*","\n") + data = gsub(data,"[\n\r]+","\n") + end + collected[#collected+1] = "\ndo -- begin closure to overcome local limits and interference\n\n" + collected[#collected+1] = data + collected[#collected+1] = "\nend -- closure\n" + end + end + end + logs.simple("saving '%s'",newname) + io.savedata(newname,table.concat(collected)) + end + end +end + +logs.extendbanner("Package Tools 0.1",true) + +messages.help = [[ +--merge merge 'loadmodule' into merge file +]] + +if environment.argument("merge") then + scripts.package.merge_luatex_files(environment.files[1] or "") +else + logs.help(messages.help) +end diff --git a/scripts/context/lua/mtx-patterns.lua b/scripts/context/lua/mtx-patterns.lua index 7fda6900c..7f130465b 100644 --- a/scripts/context/lua/mtx-patterns.lua +++ b/scripts/context/lua/mtx-patterns.lua @@ -19,8 +19,8 @@ scripts.patterns.list = { { "cs", "hyph-cs.tex", "czech" }, { "??", "hyph-cy.tex", "welsh" }, { "da", "hyph-da.tex", "danish" }, - { "de", "hyph-de-1901.tex", "german, old spelling" }, - { "deo", "hyph-de-1996.tex", "german, new spelling" }, + { "deo", "hyph-de-1901.tex", "german, old spelling" }, + { "de", "hyph-de-1996.tex", "german, new spelling" }, --~ { "??", "hyph-el-monoton.tex", "" }, --~ { "??", "hyph-el-polyton.tex", "" }, --~ { "agr", "hyph-grc", "ancient greek" }, @@ -60,7 +60,7 @@ scripts.patterns.list = { { "??", "hyph-sr-cyrl.tex", "serbian" }, { "sv", "hyph-sv.tex", "swedish" }, { "tr", "hyph-tr.tex", "turkish" }, - { "??", "hyph-uk.tex", "ukrainian" }, + { "uk", "hyph-uk.tex", "ukrainian" }, { "??", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" }, } @@ -91,6 +91,7 @@ local permitted_characters = table.tohash { 0x0009, -- tab 0x0027, -- apostrofe 0x002D, -- hyphen + 0x200C, -- } function scripts.patterns.load(path,name,mnemonic,fullcheck) @@ -104,9 +105,9 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) local subdata = io.loaddata(subfull) or "" if subdata == "" then if mnemonic then - input.report("no subfile %s for language %s",subname,mnemonic) + logs.simple("no subfile %s for language %s",subname,mnemonic) else - input.report("no subfile %s",name) + logs.simple("no subfile %s",name) end end return previous .. subdata @@ -122,15 +123,15 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) line = line:gsub("%%","%%%%") if fullcheck then if mnemonic then - input.report("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line) + logs.simple("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line) else - input.report("invalid utf in file %s, line %s: %s",name,n,line) + logs.simple("invalid utf in file %s, line %s: %s",name,n,line) end else if mnemonic then - input.report("file %s for %s contains invalid utf",name,mnemonic) + logs.simple("file %s for %s contains invalid utf",name,mnemonic) else - input.report("file %s contains invalid utf",name) + logs.simple("file %s contains invalid utf",name) end break end @@ -154,17 +155,17 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) for k, v in pairs(h) do if not permitted_commands[k] then okay = false end if mnemonic then - input.report("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) + logs.simple("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) else - input.report("command \\%s found in file %s, n=%s",k,name,v) + logs.simple("command \\%s found in file %s, n=%s",k,name,v) end end if not environment.argument("fast") then for k, v in pairs(c) do if mnemonic then - input.report("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) + logs.simple("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) else - input.report("command \\%s found in comment of file %s, n=%s",k,name,v) + logs.simple("command \\%s found in comment of file %s, n=%s",k,name,v) end end end @@ -222,9 +223,9 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) local stripped = { } for k, v in pairs(p) do if mnemonic then - input.report("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) + logs.simple("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else - input.report("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v) + logs.simple("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v) end if not permitted_characters[k] then okay = false @@ -234,9 +235,9 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end for k, v in pairs(h) do if mnemonic then - input.report("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) + logs.simple("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else - input.report("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v) + logs.simple("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v) end if not permitted_characters[k] then okay = false @@ -246,15 +247,15 @@ function scripts.patterns.load(path,name,mnemonic,fullcheck) end local stripset = "" for k, v in pairs(stripped) do - input.report("entries that contain character %s will be omitted",char(k)) + logs.simple("entries that contain character %s will be omitted",char(k)) stripset = stripset .. "%" .. char(k) end return okay, pats, hyps, comment, stripset, pused, hused else if mnemonic then - input.report("no file %s for language %s",fullname,mnemonic) + logs.simple("no file %s for language %s",fullname,mnemonic) else - input.report("no file %s",fullname) + logs.simple("no file %s",fullname) end return false, { }, { }, "", "", { }, { } end @@ -265,14 +266,14 @@ function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,commen local nofhyphenations = #hyphenations local pu = table.concat(table.sortedkeys(pused), " ") local hu = table.concat(table.sortedkeys(hused), " ") - input.report("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations) + logs.simple("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations) if mnemonic ~= "??" then local rmefile = file.join(destination,"lang-"..mnemonic..".rme") local patfile = file.join(destination,"lang-"..mnemonic..".pat") local hypfile = file.join(destination,"lang-"..mnemonic..".hyp") local topline = "% generated by mtxrun --script pattern --convert" local banner = "% for comment and copyright, see " .. rmefile - input.report("saving language data for %s",mnemonic) + logs.simple("saving language data for %s",mnemonic) if not comment or comment == "" then comment = "% no comment" end if not type(destination) == "string" then destination = "." end os.remove(rmefile) @@ -285,60 +286,58 @@ function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,commen end function scripts.patterns.prepare() - dofile(input.find_file("char-def.lua")) + dofile(resolvers.find_file("char-def.lua")) end function scripts.patterns.check() local path = environment.argument("path") or "." local found = false - local verbose = input.verbose - input.verbose = true if #environment.files > 0 then for _, name in ipairs(environment.files) do - input.report("checking language file %s", name) + logs.simple("checking language file %s", name) local okay = scripts.patterns.load(path,name,nil,not environment.argument("fast")) if #environment.files > 1 then - input.report("") + logs.simple("") end end else for k, v in pairs(scripts.patterns.list) do local mnemonic, name = v[1], v[2] - input.report("checking language %s, file %s", mnemonic, name) + logs.simple("checking language %s, file %s", mnemonic, name) local okay = scripts.patterns.load(path,name,mnemonic,not environment.argument("fast")) if not okay then - input.report("there are errors that need to be fixed") + logs.simple("there are errors that need to be fixed") end - input.report("") + logs.simple("") end end - input.verbose = verbose end function scripts.patterns.convert() local path = environment.argument("path") or "." - local destination = environment.argument("destination") or "." - if path == destination then - input.report("source path and destination path should differ (use --path and/or --destination)") + if path == "" then + logs.simple("provide sourcepath using --path ") else - local verbose = input.verbose - input.verbose = true - for k, v in pairs(scripts.patterns.list) do - local mnemonic, name = v[1], v[2] - input.report("converting language %s, file %s", mnemonic, name) - local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) - if okay then - scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) - else - input.report("convertion aborted due to error(s)") + local destination = environment.argument("destination") or "." + if path == destination then + logs.simple("source path and destination path should differ (use --path and/or --destination)") + else + for k, v in pairs(scripts.patterns.list) do + local mnemonic, name = v[1], v[2] + logs.simple("converting language %s, file %s", mnemonic, name) + local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) + if okay then + scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) + else + logs.simple("convertion aborted due to error(s)") + end + logs.simple("") end - input.report("") end end - input.verbose = verbose end -banner = banner .. " | pattern tools " +logs.extendbanner("Pattern Tools 0.20",true) messages.help = [[ --convert generate context language files (mnemonic driven, if not given then all) @@ -354,10 +353,10 @@ elseif environment.argument("convert") then scripts.patterns.prepare() scripts.patterns.convert() else - input.help(banner,messages.help) + logs.help(messages.help) end -- mtxrun --script pattern --check hyph-*.tex --- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination e:/tmp/patterns +-- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns +-- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination=e:/tmp/patterns diff --git a/scripts/context/lua/mtx-profile.lua b/scripts/context/lua/mtx-profile.lua new file mode 100644 index 000000000..d99f7e926 --- /dev/null +++ b/scripts/context/lua/mtx-profile.lua @@ -0,0 +1,164 @@ +if not modules then modules = { } end modules ['mtx-profile'] = { + version = 1.000, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- todo: also line number +-- todo: sort runtime as option + +local match, format, find = string.match, string.format, string.find + +scripts = scripts or { } +scripts.profiler = scripts.profiler or { } + +local timethreshold = 0 +local callthreshold = 2500 +local countthreshold = 2500 + +local functiontemplate = "%12s %03.4f %9i %s" +local calltemplate = "%9i %s" +local totaltemplate = "%i internal calls, %i function calls taking %3.4f seconds" +local thresholdtemplate = "thresholds: %i internal calls, %i function calls, %i seconds" + +function scripts.profiler.analyse(filename) + local f = io.open(filename) + if f then + local times, counts, calls = { }, { }, { } + local totalruntime, totalcount, totalcalls = 0, 0, 0 + while true do + local line = f:read() + if line then + local stacklevel, filename, functionname, linenumber, currentline, localtime, totaltime = line:match("^(%d+)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)") + if not filename then + -- next + elseif filename == "=[C]" then + if not functionname:find("^%(") then + calls[functionname] = (calls[functionname] or 0) + 1 + end + else + local filename = filename:match("^@(.*)$") + if filename then + local fi = times[filename] + if not fi then fi = { } times[filename] = fi end + fi[functionname] = (fi[functionname] or 0) + tonumber(localtime) + counts[functionname] = (counts[functionname] or 0) + 1 + end + end + else + break + end + end + f:close() + print("") + local loaded = { } + for _, filename in ipairs(table.sortedkeys(times)) do + local functions = times[filename] + for _, functionname in ipairs(table.sortedkeys(functions)) do + local totaltime = functions[functionname] + local count = counts[functionname] + totalcount = totalcount + count + if totaltime > timethreshold or count > countthreshold then + totalruntime = totalruntime + totaltime + local functionfile, somenumber = functionname:match("^@(.+):(.-)$") + if functionfile then + local number = tonumber(somenumber) + if number then + if not loaded[functionfile] then + loaded[functionfile] = string.splitlines(io.loaddata(functionfile) or "") + end + functionname = loaded[functionfile][number] or functionname + functionname = functionname:gsub("^%s*","") + functionname = functionname:gsub("%s*%-%-.*$","") + functionname = number .. ": " .. functionname + end + end + filename = file.basename(filename) + print(functiontemplate:format(filename,totaltime,count,functionname)) + end + end + end + print("") + for _, call in ipairs(table.sortedkeys(calls)) do + local n = calls[call] + totalcalls = totalcalls + n + if n > callthreshold then + print(calltemplate:format(n,call)) + end + end + print("") + print(totaltemplate:format(totalcalls,totalcount,totalruntime)) + print("") + print(thresholdtemplate:format(callthreshold,countthreshold,timethreshold)) + end +end + +function scripts.profiler.analyse(filename) + local f = io.open(filename) + local calls = { } + local lines = 0 + if f then + while true do + local line = f:read() + if line then + lines = lines + 1 + local c = match(line,"\\([a-zA-Z%!%?@]+) *%->") + if c then + local cc = calls[c] + if not cc then + calls[c] = 1 + else + calls[c] = cc + 1 + end + end + else + break + end + end + f:close() + local noc = 0 +local criterium = 100 + for name, n in next, calls do + if n > criterium then + if find(name,"^@@[a-z][a-z]") then + -- parameter + elseif find(name,"^[cvserft]%!") then + -- variables and constants + elseif find(name,"^%?%?[a-z][a-z]$") then + -- prefix + elseif find(name,"^%!%!") then + -- reserved + elseif find(name,"^@.+@$") then + -- weird + else + noc = noc + n + print(format("%6i: %s",n,name)) + end + end + end + print("") + print(format("number of lines: %s",lines)) + print(format("number of calls: %s",noc)) + print(format("criterium calls: %s",criterium)) + end +end + +--~ scripts.profiler.analyse("t:/manuals/mk/mk-fonts-profile.lua") +--~ scripts.profiler.analyse("t:/manuals/mk/mk-introduction-profile.lua") + +logs.extendbanner("LuaTeX Profiler 1.00",true) + +messages.help = [[ +--analyse analyse lua calls +--trace analyse tex calls +]] + +if environment.argument("analyse") then + scripts.profiler.analyse(environment.files[1] or "luatex-profile.log") +elseif environment.argument("trace") then + scripts.profiler.analyse(environment.files[1] or "temp.log") +else + logs.help(messages.help) +end diff --git a/scripts/context/lua/mtx-server-ctx-fonttest.lua b/scripts/context/lua/mtx-server-ctx-fonttest.lua new file mode 100644 index 000000000..efaae66e3 --- /dev/null +++ b/scripts/context/lua/mtx-server-ctx-fonttest.lua @@ -0,0 +1,681 @@ +if not modules then modules = { } end modules ['mtx-server-ctx-fonttest'] = { + version = 1.001, + comment = "Font Feature Tester", + author = "Hans Hagen", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +dofile(resolvers.find_file("l-aux.lua","tex")) +dofile(resolvers.find_file("trac-lmx.lua","tex")) +dofile(resolvers.find_file("font-ott.lua","tex")) +dofile(resolvers.find_file("font-syn.lua","tex")) +dofile(resolvers.find_file("font-mis.lua","tex")) +dofile(resolvers.find_file("font-otp.lua","tex")) + +local format, gsub, concat, match, find = string.format, string.gsub, table.concat, string.match, string.find + +local sample_line = "This is a sample line!" +local tempname = "mtx-server-ctx-fonttest-temp" +local temppath = caches.setpath("temp","mtx-server-ctx-fonttest") +local basename = "mtx-server-ctx-fonttest-data.lua" +local basepath = temppath + +for _, suffix in ipairs { "tex", "pdf", "log" } do + os.remove(file.join(temppath,file.addsuffix(tempname,suffix))) +end + +local process_templates = { } + +process_templates.default = [[ +\starttext + \setcharactermirroring[1] + \definefontfeature[sample][%s] + \definedfont[name:%s*sample] + \startTEXpage[offset=3pt] + \detokenize{%s} + \stopTEXpage +\stoptext +]] + +process_templates.cache = [[ +\starttext + \definedfont[name:%s] + \startTEXpage[offset=3pt] + cached: \detokenize{%s} + \stopTEXpage +\stoptext +]] + +process_templates.trace = [[ +\usemodule[fnt-20] + +\definefontfeature[sample][%s] + +\setupcolors[state=start] + +\setcharactermirroring[1] + +\setvariables + [otftracker] + [title=Test Run, + font=name:%s, + direction=0, + features=sample, + sample={\detokenize{%s}}] +]] + +local javascripts = [[ +function selected_radio(name) { + var form = document.forms["main-form"] ; + var script = form.elements[name] ; + if (script) { + var n = script.length ; + if (n) { + for (var i=0; i<n; i++) { + if (script[i].checked) { + return script[i].value ; + } + } + } + } + return "" ; +} + +function reset_valid() { + var fields = document.getElementsByTagName("span") ; + for (var i=0; i<fields.length; i++) { + var e = fields[i] + if (e) { + if (e.className == "valid") { + e.className = "" ; + } + } + } +} + +function set_valid() { + var script = selected_radio("script") ; + var language = selected_radio("language") ; + if (script && language) { + var s = feature_hash[script] ; + if (s) { + for (l in s) { + var e = document.getElementById("t-l-" + l) ; + if (e) { + e.className = "valid" ; + } + } + var l = s[language] ; + if (l) { + for (i in l) { + var e = document.getElementById("t-f-" + i) ; + if (e) { + e.className = "valid" ; + } + } + } + var e = document.getElementById("t-s-" + script) ; + if (e) { + e.className = "valid" ; + } + } + } +} + +function check_form() { + reset_valid() ; + set_valid() ; +} + +function check_script() { + reset_valid() ; + set_valid() ; +} + +function check_language() { + reset_valid() ; + set_valid() ; +} + +function check_feature() { + // not needed +} +]] + +local cache = { } + +local function showfeatures(f) + if f then + local features = cache[f] + if features == nil then + features = fonts.get_features(f) + if not features then + logs.simple("building cache for '%s'",f) + io.savedata(file.join(temppath,file.addsuffix(tempname,"tex")),format(process_templates.cache,f,f)) + os.execute(format("mtxrun --path=%s --script context --once --batchmode --mode=*nofonts %s",temppath,tempname)) + features = fonts.get_features(f) + end + cache[f] = features or false + logs.simple("caching info of '%s'",f) + else + logs.simple("using cached info of '%s'",f) + end + if features then + local scr, lan, fea, rev = { }, { }, { }, { } + local function show(what) + local data = features[what] + if data and next(data) then + for f,ff in pairs(data) do + if find(f,"<") then + -- ignore aat for the moment + else + fea[f] = true + for s, ss in pairs(ff) do + if find(s,"%*") then + -- ignore * + else + scr[s] = true + local rs = rev[s] if not rs then rs = {} rev[s] = rs end + for k, l in pairs(ss) do + if find(k,"%*") then + -- ignore * + else + lan[k] = true + local rsk = rs[k] if not rsk then rsk = { } rs[k] = rsk end + rsk[f] = true + end + end + end + end + end + end + end + end + for what, v in table.sortedpairs(features) do + show(what) + end + local stupid = { } + stupid[#stupid+1] = "var feature_hash = new Array ;" + for s, sr in pairs(rev) do + stupid[#stupid+1] = format("feature_hash['%s'] = new Array ;",s) + for l, lr in pairs(sr) do + stupid[#stupid+1] = format("feature_hash['%s']['%s'] = new Array ;",s,l) + for f, fr in pairs(lr) do + stupid[#stupid+1] = format("feature_hash['%s']['%s']['%s'] = true ;",s,l,f) + end + end + end + -- gpos feature script languages + return { + scripts = scr, + languages = lan, + features = fea, + javascript = concat(stupid,"\n") + } + end + end +end + +local function select_font() + local t = fonts.names.list(".*") + if t then + local listoffonts = { } + local t = fonts.names.list(".*") + if t then + listoffonts[#listoffonts+1] = "<table>" + listoffonts[#listoffonts+1] = "<tr><th>safe name</th><th>font name</th><th>filename</th><th>sub font </th><th>type</th></tr>" + for k, id in ipairs(table.sortedkeys(t)) do + local ti = t[id] + local type, name, file, sub = ti[1], ti[2], ti[3], ti[4] + if type == "otf" or type == "ttf" or type == "ttc" then + if sub then sub = "(sub)" else sub = "" end + listoffonts[#listoffonts+1] = format("<tr><td><a href='mtx-server-ctx-fonttest.lua?selection=%s'>%s</a></td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>",id,id,name,file,sub,type) + end + end + listoffonts[#listoffonts+1] = "</table>" + return concat(listoffonts,"\n") + end + end + return "<b>no fonts</b>" +end + +local edit_template = [[ + <textarea name='sampletext' rows='5' cols='100'>%s</textarea> + <br/> <br/>name: <input type='text' name='name' size='20' value=%q/> title: <input type='text' name='title' size='40' value=%q/> + <br/> <br/>scripts: %s + <br/> <br/>languages: %s + <br/> <br/>features: %s + <br/> <br/>options: %s +]] + +local result_template = [[ + <br/> <br/> + <embed src="%s#toolbar=0&navpanes=0&scrollbar=0" width="100%%"/> + <br/> <br/> results: + <a href='%s' target="source">tex file</a> + <a href='%s' target="result">pdf file</a> + <br/> <br/> +]] + +scripts.webserver.registerpath(temppath) + +local function edit_font(currentfont,detail,tempname) + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local htmldata = showfeatures(fontfile) + if htmldata then + local features, languages, scripts, options = { }, { }, { }, { } + for k,v in ipairs(table.sortedkeys(htmldata.scripts)) do + local s = fonts.otf.tables.scripts[v] or v + if detail and v == detail.script then + scripts[#scripts+1] = format("<input title='%s' id='s-%s' type='radio' name='script' value='%s' onclick='check_script()' checked='checked'/> <span id='t-s-%s'>%s</span>",s,v,v,v,v) + else + scripts[#scripts+1] = format("<input title='%s' id='s-%s' type='radio' name='script' value='%s' onclick='check_script()' /> <span id='t-s-%s'>%s</span>",s,v,v,v,v) + end + end + for k,v in ipairs(table.sortedkeys(htmldata.languages)) do + local l = fonts.otf.tables.languages[v] or v + if detail and v == detail.language then + languages[#languages+1] = format("<input title='%s' id='l-%s' type='radio' name='language' value='%s' onclick='check_language()' checked='checked'/> <span id='t-l-%s'>%s</span>",l,v,v,v,v) + else + languages[#languages+1] = format("<input title='%s' id='l-%s' type='radio' name='language' value='%s' onclick='check_language()' /> <span id='t-l-%s'>%s</span>",l,v,v,v,v) + end + end + for k,v in ipairs(table.sortedkeys(htmldata.features)) do + local f = fonts.otf.tables.features[v] or v + if detail and detail["f-"..v] then + features[#features+1] = format("<input title='%s' id='f-%s' type='checkbox' name='f-%s' onclick='check_feature()' checked='checked'/> <span id='t-f-%s'>%s</span>",f,v,v,v,v) + else + features[#features+1] = format("<input title='%s' id='f-%s' type='checkbox' name='f-%s' onclick='check_feature()' /> <span id='t-f-%s'>%s</span>",f,v,v,v,v) + end + end + for k, v in ipairs { "trace", "basemode" } do + if detail and detail["o-"..v] then + options[#options+1] = format("<input id='o-%s' type='checkbox' name='o-%s' checked='checked'/> %s",v,v,v) + else + options[#options+1] = format("<input id='o-%s' type='checkbox' name='o-%s'/> %s",v,v,v) + end + end + local e = format(edit_template, + (detail and detail.sampletext) or sample_line,(detail and detail.name) or "no name",(detail and detail.title) or "", + concat(scripts," "),concat(languages," "),concat(features," "),concat(options," ")) + if tempname then + local pdffile, texfile = file.addsuffix(tempname,"pdf"), file.addsuffix(tempname,"tex") + local r = format(result_template,pdffile,texfile,pdffile) + return e .. r, htmldata.javascript or "" + else + return e, htmldata.javascript or "" + end + else + return "error, nothing set up yet" + end +end + +local function process_font(currentfont,detail) -- maybe just fontname + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local features = { + "mode=node", + format("language=%s",detail.language or "dflt"), + format("script=%s",detail.script or "dflt"), + } + for k,v in pairs(detail) do + local f = match(k,"^f%-(.*)$") + if f then + features[#features+1] = format("%s=yes",f) + end + end + local variant = process_templates.default + if detail["o-trace"] then + variant = process_templates.trace + end + local sample = string.strip(detail.sampletext or "") + if sample == "" then sample = sample_line end + logs.simple("sample text: %s",sample) + io.savedata(file.join(temppath,file.addsuffix(tempname,"tex")),format(variant,concat(features,","),currentfont,sample)) + os.execute(format("mtxrun --path=%s --script context --once --batchmode --mode=*nofonts %s",temppath,tempname)) + return edit_font(currentfont,detail,tempname) +end + +local tex_template = [[ +<pre><tt> +%s +</tt></pre> +]] + +local function show_source(currentfont,detail) + if tempname and tempname ~= "" then + return format(tex_template,io.loaddata(file.join(temppath,file.addsuffix(tempname,"tex"))) or "no source yet") + else + return "no source file" + end +end + +local function show_log(currentfont,detail) + if tempname and tempname ~= "" then + local data = io.loaddata(file.join(temppath,file.addsuffix(tempname,'log'))) or "no log file yet" + data = gsub(data,"[%s%%]*begin of optionfile.-end of optionfile[%s%%]*","\n") + return format(tex_template,data) + else + return "no log file" + end +end + +local function show_font(currentfont,detail) + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local features = fonts.get_features(fontfile) + local result = { } + result[#result+1] = format("<h1>names</h1>",what) + result[#result+1] = "<table>" + result[#result+1] = format("<tr><td class='tc'>fontname:</td><td>%s</td></tr>",currentfont) + result[#result+1] = format("<tr><td class='tc'>fullname:</td><td>%s</td></tr>",fontname) + result[#result+1] = format("<tr><td class='tc'>filename:</td><td>%s</td></tr>",fontfile) + result[#result+1] = "</table>" + if features then + for what, v in table.sortedpairs(features) do + local data = features[what] + if data and next(data) then + result[#result+1] = format("<h1>%s features</h1>",what) + result[#result+1] = "<table>" + result[#result+1] = "<tr><th>feature</th><th>tag </th><th>script </th><th>languages </th></tr>" + for f,ff in table.sortedpairs(data) do + local done = false + for s, ss in table.sortedpairs(ff) do + if s == "*" then s = "all" end + if ss ["*"] then ss["*"] = nil ss.all = true end + if done then + f = "" + else + done = true + end + local title = fonts.otf.tables.features[f] or "" + result[#result+1] = format("<tr><td width='50%%'>%s </td><td><tt>%s </tt></td><td><tt>%s </tt></td><td><tt>%s </tt></td></tr>",title,f,s,concat(table.sortedkeys(ss)," ")) + end + end + result[#result+1] = "</table>" + end + end + else + result[#result+1] = "<br/><br/>This font has no features." + end + return concat(result,"\n") +end + + +local info_template = [[ +<pre><tt> +version : %s +comment : %s +author : %s +copyright : %s + +maillist : ntg-context at ntg.nl +webpage : www.pragma-ade.nl +wiki : contextgarden.net +</tt></pre> +]] + +local function info_about() + local m = modules ['mtx-server-ctx-fonttest'] + return format(info_template,m.version,m.comment,m.author,m.copyright) +end + +local save_template = [[ + the current setup has been saved: + <br/> <br/> + <table> + <tr><td class='tc'>name </td><td>%s</td></tr> + <tr><td class='tc'>title </td><td>%s</td></tr> + <tr><td class='tc'>font </td><td>%s</td></tr> + <tr><td class='tc'>script </td><td>%s</td></tr> + <tr><td class='tc'>language </td><td>%s</td></tr> + <tr><td class='tc'>features </td><td>%s</td></tr> + <tr><td class='tc'>options </td><td>%s</td></tr> + <tr><td class='tc'>sampletext </td><td>%s</td></tr> + </table> +]] + +local function loadbase() + local datafile = file.join(basepath,basename) + local storage = io.loaddata(datafile) or "" + if storage == "" then + storage = { } + else + logs.simple("loading '%s'",datafile) + storage = loadstring(storage) + storage = (storage and storage()) or { } + end + return storage +end + +local function loadstored(detail,currentfont,name) + local storage = loadbase() + storage = storage and storage[name] + if storage then + currentfont = storage.font + detail.script = storage.script or detail.script + detail.language = storage.language or detail.language + detail.title = storage.title or detail.title + detail.sampletext = storage.text or detail.sampletext + detail.name = name or "no name" + for k,v in pairs(storage.features) do + detail["f-"..k] = v + end + for k,v in pairs(storage.options) do + detail["o-"..k] = v + end + end + detail.loadname = nil + return detail, currentfont +end + +local function savebase(storage,name) + local datafile = file.join(basepath,basename) + logs.simple("saving '%s' in '%s'",name or "data",datafile) + io.savedata(datafile,table.serialize(storage,true)) +end + +local function deletestored(detail,currentfont,name) + local storage = loadbase() + if storage and name and storage[name] then + logs.simple("deleting '%s' from base",name) + storage[name] = nil + savebase(storage) + end + detail.deletename = nil + return detail, "" +end + +local function save_font(currentfont,detail) + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + local name, title, script, language, features, options, text = currentfont, "", "dflt", "dflt", { }, { }, "" + if detail then + local htmldata = showfeatures(fontfile) + script = detail.script or script + language = detail.language or language + text = string.strip(detail.sampletext or text) + name = string.strip(detail.name or name) + title = string.strip(detail.title or title) + for k,v in pairs(htmldata.features) do + if detail["f-"..k] then features[k] = true end + end + for k,v in ipairs { "trace", "basemode" } do + if detail["o-"..v] then options[k] = true end + end + end + if name == "" then + name = "no name" + end + local storage = loadbase() + storage[name] = { + font = currentfont, title = title, script = script, language = language, features = features, options = options, text = text, + } + savebase(storage,name) + return format(save_template,name,title,currentfont,script,language,concat(table.sortedkeys(features)," "),concat(table.sortedkeys(options)," "),text) +end + +local function load_font(currentfont) + local datafile = file.join(basepath,basename) + local storage = loadbase(datafile) + local result = {} + result[#result+1] = format("<tr><th>del </th><th>name </th><th>font </th><th>fontname </th><th>script </th><th>language </th><th>features </th><th>title </th><th>sampletext </th></tr>") + for k,v in table.sortedpairs(storage) do + local fontname, fontfile, issub = fonts.names.specification(v.font or "") + result[#result+1] = format("<tr><td><a href='mtx-server-ctx-fonttest.lua?deletename=%s'>x</a> </td><td><a href='mtx-server-ctx-fonttest.lua?loadname=%s'>%s</a> </td><td>%s </td<td>%s </td><td>%s </td><td>%s </td><td>%s </td><td>%s </td><td>%s </td></tr>", + k,k,k,v.font,fontname,v.script,v.language,concat(table.sortedkeys(v.features)," "),v.title or "no title",v.text or "") + end + if #result == 1 then + return "nothing saved yet" + else + return format("<table>%s</table>",concat(result,"\n")) + end +end + +local function reset_font(currentfont) + return edit_font(currentfont) +end + +local extras_template = [[ + <a href='mtx-server-ctx-fonttest.lua?extra=reload'>remake font database</a> (take some time)<br/><br/> +]] + +local function do_extras(detail,currentfont,extra) + return extras_template +end + +local extras = { } + +local function do_extra(detail,currentfont,extra) + local e = extras[extra] + if e then e(detail,currentfont,extra) end + return do_extras(detail,currentfont,extra) +end + +function extras.reload() + local command = "mtxrun --script font --reload" + logs.simple("run command: %s",command) + os.execute(command) + return do_extras() +end + + +local status_template = [[ + <input type="hidden" name="currentfont" value="%s" /> +]] + +function doit(configuration,filename,hashed) + + local start = os.clock() + + local detail = url.query(hashed.query or "") + + local currentfont = detail.currentfont + local action = detail.action + local selection = detail.selection + + local loadname = detail.loadname + local deletename = detail.deletename + local extra = detail.extra + + if loadname and loadname ~= "" then + detail, currentfont = loadstored(detail,currentfont,loadname) + action = "process" + elseif deletename and deletename ~= "" then + detail, currentfont = deletestored(detail,currentfont,deletename) + action = "load" + elseif selection and selection ~= "" then + currentfont = selection + elseif extra and extra ~= "" then + do_extra(detail,currentfont,extra) + action = "extras" + end + + lmx.restore() + + local fontname, fontfile, issub = fonts.names.specification(currentfont or "") + + if fontfile then + lmx.variables['title-default'] = format('ConTeXt Font Tester: %s (%s)',fontname,fontfile) + else + lmx.variables['title-default'] = 'ConTeXt Font Tester' + end + + lmx.variables['color-background-green'] = '#4F6F6F' + lmx.variables['color-background-blue'] = '#6F6F8F' + lmx.variables['color-background-yellow'] = '#8F8F6F' + lmx.variables['color-background-purple'] = '#8F6F8F' + + lmx.variables['color-background-body'] = '#808080' + lmx.variables['color-background-main'] = '#3F3F3F' + lmx.variables['color-background-one'] = lmx.variables['color-background-green'] + lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] + + lmx.variables['title'] = lmx.variables['title-default'] + + lmx.set('title', lmx.get('title')) + lmx.set('color-background-one', lmx.get('color-background-green')) + lmx.set('color-background-two', lmx.get('color-background-blue')) + + -- lua table and adapt + + lmx.set('formaction', "mtx-server-ctx-fonttest.lua") + + local menu = { } + for k, v in ipairs { 'process', 'select', 'save', 'load', 'edit', 'reset', 'features', 'source', 'log', 'info', 'extras'} do + menu[#menu+1] = format("<button name='action' value='%s' type='submit'>%s</button>",v,v) + end + lmx.set('menu', concat(menu," ")) + + logs.simple("action: %s",action or "no action") + + lmx.set("status",format(status_template,currentfont or "")) + + local result + + if action == "select" then + lmx.set('maintext',select_font()) + elseif action == "info" then + lmx.set('maintext',info_about()) + elseif action == "extras" then + lmx.set('maintext',do_extras()) + elseif currentfont and currentfont ~= "" then + if action == "save" then + lmx.set('maintext',save_font(currentfont,detail)) + elseif action == "load" then + lmx.set('maintext',load_font(currentfont,detail)) + elseif action == "source" then + lmx.set('maintext',show_source(currentfont,detail)) + elseif action == "log" then + lmx.set('maintext',show_log(currentfont,detail)) + elseif action == "features" then + lmx.set('maintext',show_font(currentfont,detail)) + else + local e, s + if action == "process" then + e, s = process_font(currentfont,detail) + elseif action == "reset" then + e, s = reset_font(currentfont) + elseif action == "edit" then + e, s = edit_font(currentfont,detail) + else + e, s = process_font(currentfont,detail) + end + lmx.set('maintext',e) + lmx.set('javascriptdata',s) + lmx.set('javascripts',javascripts) + lmx.set('javascriptinit', "check_form()") + end + else + lmx.set('maintext',select_font()) + end + + result = { content = lmx.convert('context-fonttest.lmx') } + + logs.simple("time spent on page: %0.03f seconds",os.clock()-start) + + return result + +end + +return doit, true + +--~ make_lmx_page("test") diff --git a/scripts/context/lua/mtx-server-ctx-help.lua b/scripts/context/lua/mtx-server-ctx-help.lua new file mode 100644 index 000000000..c53d9f6e0 --- /dev/null +++ b/scripts/context/lua/mtx-server-ctx-help.lua @@ -0,0 +1,648 @@ +if not modules then modules = { } end modules ['mtx-server-ctx-help'] = { + version = 1.001, + comment = "Basic Definition Browser", + author = "Hans Hagen", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--~ dofile(resolvers.find_file("l-xml.lua","tex")) +dofile(resolvers.find_file("l-aux.lua","tex")) +dofile(resolvers.find_file("trac-lmx.lua","tex")) + +-- problem ... serialize parent stack + +local format = string.format +local concat = table.concat + +-- -- -- make this a module: cont-xx.lua + +document = document or { } +document.setups = document.setups or { } + +document.setups.div = { + pe = "<div dir='rtl' lang='arabic'>%s</div>" +} + +document.setups.span = { + pe = "<span dir='rtl' lang='arabic'>%s</span>" +} + +document.setups.translations = document.setups.translations or { + + nl = { + ["title"] = "setup", + ["formula"] = "formule", + ["number"] = "getal", + ["list"] = "lijst", + ["dimension"] = "maat", + ["mark"] = "markering", + ["reference"] = "verwijzing", + ["command"] = "commando", + ["file"] = "file", + ["name"] = "naam", + ["identifier"] = "naam", + ["text"] = "tekst", + ["section"] = "sectie", + ["singular"] = "naam enkelvoud", + ["plural"] = "naam meervoud", + ["matrix"] = "n*m", + ["see"] = "zie", + ["inherits"] = "erft van", + ["optional"] = "optioneel", + ["displaymath"] = "formule", + ["index"] = "ingang", + ["math"] = "formule", + ["nothing"] = "leeg", + ["file"] = "file", + ["position"] = "positie", + ["reference"] = "verwijzing", + ["csname"] = "naam", + ["destination"] = "bestemming", + ["triplet"] = "triplet", + ["word"] = "woord", + ["content"] = "tekst", + }, + + en = { + ["title"] = "setup", + ["formula"] = "formula", + ["number"] = "number", + ["list"] = "list", + ["dimension"] = "dimension", + ["mark"] = "mark", + ["reference"] = "reference", + ["command"] = "command", + ["file"] = "file", + ["name"] = "name", + ["identifier"] = "identifier", + ["text"] = "text", + ["section"] = "section", + ["singular"] = "singular name", + ["plural"] = "plural name", + ["matrix"] = "n*m", + ["see"] = "see", + ["inherits"] = "inherits from", + ["optional"] = "optional", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + + ["noargument"] = "\\cs", + ["oneargument"] = "\\cs#1{..}", + ["twoarguments"] = "\\cs#1#2{..}{..}", + ["threearguments"] = "\\cs#1#2#3{..}{..}{..}", + + }, + + de = { + ["title"] = "Setup", + ["formula"] = "Formel", + ["number"] = "Nummer", + ["list"] = "Liste", + ["dimension"] = "Dimension", + ["mark"] = "Beschriftung", + ["reference"] = "Referenz", + ["command"] = "Befehl", + ["file"] = "Datei", + ["name"] = "Name", + ["identifier"] = "Name", + ["text"] = "Text", + ["section"] = "Abschnitt", + ["singular"] = "singular", + ["plural"] = "plural", + ["matrix"] = "n*m", + ["see"] = "siehe", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + cz = { + ["title"] = "setup", + ["formula"] = "rovnice", + ["number"] = "cislo", + ["list"] = "seznam", + ["dimension"] = "dimenze", + ["mark"] = "znacka", + ["reference"] = "reference", + ["command"] = "prikaz", + ["file"] = "soubor", + ["name"] = "jmeno", + ["identifier"] = "jmeno", + ["text"] = "text", + ["section"] = "sekce", + ["singular"] = "jmeno v singularu", + ["plural"] = "jmeno v pluralu", + ["matrix"] = "n*m", + ["see"] = "viz", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + it = { + ["title"] = "setup", + ["formula"] = "formula", + ["number"] = "number", + ["list"] = "list", + ["dimension"] = "dimension", + ["mark"] = "mark", + ["reference"] = "reference", + ["command"] = "command", + ["file"] = "file", + ["name"] = "name", + ["identifier"] = "name", + ["text"] = "text", + ["section"] = "section", + ["singular"] = "singular name", + ["plural"] = "plural name", + ["matrix"] = "n*m", + ["see"] = "see", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + ro = { + ["title"] = "setari", + ["formula"] = "formula", + ["number"] = "numar", + ["list"] = "lista", + ["dimension"] = "dimensiune", + ["mark"] = "marcaj", + ["reference"] = "referinta", + ["command"] = "comanda", + ["file"] = "fisier", + ["name"] = "nume", + ["identifier"] = "nume", + ["text"] = "text", + ["section"] = "sectiune", + ["singular"] = "nume singular", + ["plural"] = "nume pluram", + ["matrix"] = "n*m", + ["see"] = "vezi", + ["inherits"] = "inherits from", + ["optional"] = "optioneel", + ["displaymath"] = "formula", + ["index"] = "entry", + ["math"] = "formula", + ["nothing"] = "empty", + ["file"] = "file", + ["position"] = "position", + ["reference"] = "reference", + ["csname"] = "name", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "word", + ["content"] = "text", + }, + + fr = { + ["title"] = "réglage", + ["formula"] = "formule", + ["number"] = "numéro", + ["list"] = "liste", + ["dimension"] = "dimension", + ["mark"] = "marquage", + ["reference"] = "reference", + ["command"] = "commande", + ["file"] = "fichier", + ["name"] = "nom", + ["identifier"] = "identificateur", + ["text"] = "texte", + ["section"] = "section", + ["singular"] = "nom singulier", + ["plural"] = "nom pluriel", + ["matrix"] = "n*m", + ["see"] = "vois", + ["inherits"] = "herite de", + ["optional"] = "optionel", + ["displaymath"] = "formule", + ["index"] = "entrée", + ["math"] = "formule", + ["nothing"] = "vide", + ["file"] = "fichier", + ["position"] = "position", + ["reference"] = "réference", + ["csname"] = "nom", + ["destination"] = "destination", + ["triplet"] = "triplet", + ["word"] = "mot", + ["content"] = "texte", + } + +} + +document.setups.formats = { + interface = [[<a href='mtx-server-ctx-help.lua?interface=%s'>%s</a>]], + href = [[<a href='mtx-server-ctx-help.lua?command=%s'>%s</a>]], + source = [[<a href='mtx-server-ctx-help.lua?source=%s'>%s</a>]], + optional_single = "[optional string %s]", + optional_list = "[optional list %s]", + mandate_single = "[mandate string %s]", + mandate_list = "[mandate list %s]", + parameter = [[<tr><td width='15%%'>%s</td><td width='15%%'>%s</td><td width='70%%'>%s</td></tr>]], + parameters = [[<table width='100%%'>%s</table>]], + listing = [[<pre><t>%s</t></listing>]], + special = "<i>%s</i>", + default = "<u>%s</u>", +} + +local function translate(tag,int,noformat) + local t = document.setups.translations + local te = t["en"] + local ti = t[int] or te + if noformat then + return ti[tag] or te[tag] or tag + else + return document.setups.formats.special:format(ti[tag] or te[tag] or tag) + end +end + +local function translated(e,int) + local attributes = e.at + local s = attributes.type or "?" + local tag = s:match("^cd:(.*)$") + if attributes.default == "yes" then + return document.setups.formats.default:format(tag) + elseif tag then + return translate(tag,int) + else + return s + end +end + +document.setups.loaded = document.setups.loaded or { } + +document.setups.current = { } +document.setups.showsources = false + +function document.setups.load(filename) + filename = resolvers.find_file(filename) or "" + if filename ~= "" then + local current = document.setups.loaded[filename] + if not current then + local loaded = xml.load(filename) + if loaded then + -- xml.inject(document.setups.root,"/",loaded) + current = { + file = filename, + root = loaded, + names = { }, + used = { }, + } + document.setups.loaded[filename] = current + end + end + document.setups.current = current or { } + end +end + +function document.setups.name(ek) + local at = ek.at + local name = at.name + if at.type == 'environment' then + name = "start" .. name + end + if at.variant then + name = name .. ":" .. at.variant + end + if at.generated == "yes" then + name = name .. "*" + end + return name:lower() +end + +function document.setups.csname(ek,int) + local cs = "" + local at = ek.at + if at.type == 'environment' then + cs = translate("start",int,true) .. cs + end + for r, d, k in xml.elements(ek,'cd:sequence/(cd:string|variable)') do + local dk = d[k] + if dk.tg == "string" then + cs = cs .. dk.at.value + else + cs = cs .. dk.at.value -- to be translated + end + end + return cs +end + +function document.setups.names() + local current = document.setups.current + local names = current.names + if not names or #names == 0 then + names = { } + local name = document.setups.name + local csname = document.setups.csname + for r, d, k in xml.elements(current.root,'cd:command') do + local dk = d[k] + names[#names+1] = { dk.at.name, csname(dk,int) } + end + table.sort(names, function(a,b) return a[2]:lower() < b[2]:lower() end) + current.names = names + end + return names +end + +function document.setups.show(name) + local current = document.setups.current + if current.root then + local name = name:gsub("[<>]","") + local setup = xml.first(current.root,"cd:command[@name='" .. name .. "']") + current.used[#current.used+1] = setup + xml.sprint(setup) + end +end + +function document.setups.showused() + local current = document.setups.current + if current.root and next(current.used) then + for k,v in ipairs(table.sortedkeys(current.used)) do + xml.sprint(current.used[v]) + end + end +end +function document.setups.showall() + local current = document.setups.current + if current.root then + local list = { } + xml.each_element(current.root,"cd:command", function(r,d,t) + local ek = d[t] + list[document.setups.name(ek)] = ek + end ) + for k,v in ipairs(table.sortedkeys(list)) do + xml.sprint(list[v]) + end + end +end +function document.setups.resolve(name) + local current = document.setups.current + if current.root then + local e = xml.filter(current.root,format("cd:define[@name='%s']/text()",name)) + if e then + xml.sprint(e) + end + end +end + +function document.setups.collect(name,int) + local current = document.setups.current + local formats = document.setups.formats + local command = xml.filter(current.root,format("cd:command[@name='%s']",name)) + if command then + local attributes = command.at + local data = { + command = command, + category = attributes.category or "", + } + if document.setups.showsources then + data.source = (attributes.file and formats.source:format(attributes.file,attributes.file)) or "" + else + data.source = attributes.file or "" + end + local sequence, n = { "\\" .. document.setups.csname(command,int) }, 0 + local arguments = { } + for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do + n = n + 1 + local attributes = d[k].at + if attributes.optional == 'yes' then + if attributes.list == 'yes' then + sequence[#sequence+1] = formats.optional_list:format(n) + else + sequence[#sequence+1] = formats.optional_single:format(n) + end + else + if attributes.list == 'yes' then + sequence[#sequence+1] = formats.mandate_list:format(n) + else + sequence[#sequence+1] = formats.mandate_single:format(n) + end + end + end + data.sequence = concat(sequence, " ") + local parameters, n = { }, 0 + for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do + n = n + 1 + if d[k].tg == "keywords" then + local left = sequence[n+1] + local right = { } + for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do + local tag = d[k].tg + if tag == "resolve" then + local name = d[k].at.name or "" + if name ~= "" then + local resolved = xml.filter(current.root,format("cd:define[@name='%s']",name)) + for r, d, k in xml.elements(resolved,"cd:constant") do + right[#right+1] = translated(d[k],int) + end + end + else + right[#right+1] = translated(d[k],int) + end + end + parameters[#parameters+1] = formats.parameter:format(left,"",concat(right, ", ")) + else + local what = sequence[n+1] + for r, d, k in xml.elements(d[k],"(cd:parameter|cd:inherit)") do + local tag = d[k].tg + local left, right = d[k].at.name or "?", { } + if tag == "inherit" then + local name = d[k].at.name or "?" + local goto = document.setups.formats.href:format(name,"\\"..name) + if #parameters > 0 and not parameters[#parameters]:find("<br/>") then + parameters[#parameters+1] = formats.parameter:format("<br/>","","") + end + parameters[#parameters+1] = formats.parameter:format(what,formats.special:format(translate("inherits",int)),goto) + else + for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do + local tag = d[k].tg + if tag == "resolve" then + local name = d[k].at.name or "" + if name ~= "" then + local resolved = xml.filter(current.root,format("cd:define[@name='%s']",name)) + for r, d, k in xml.elements(resolved,"cd:constant") do + right[#right+1] = translated(d[k],int) + end + end + else + right[#right+1] = translated(d[k],int) + end + end + parameters[#parameters+1] = formats.parameter:format(what,left,concat(right, ", ")) + end + what = "" + end + end + parameters[#parameters+1] = formats.parameter:format("<br/>","","") + end + data.parameters = parameters + return data + else + return nil + end +end + +-- -- -- + +tex = tex or { } + +lmx.variables['color-background-green'] = '#4F6F6F' +lmx.variables['color-background-blue'] = '#6F6F8F' +lmx.variables['color-background-yellow'] = '#8F8F6F' +lmx.variables['color-background-purple'] = '#8F6F8F' + +lmx.variables['color-background-body'] = '#808080' +lmx.variables['color-background-main'] = '#3F3F3F' +lmx.variables['color-background-main-left'] = '#3F3F3F' +lmx.variables['color-background-main-right'] = '#5F5F5F' +lmx.variables['color-background-one'] = lmx.variables['color-background-green'] +lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] + +lmx.variables['title-default'] = 'ConTeXt Help Information' +lmx.variables['title'] = lmx.variables['title-default'] + +function lmx.loadedfile(filename) + return io.loaddata(resolvers.find_file(filename)) -- return resolvers.texdatablob(filename) +end + +-- -- -- + +local interfaces = { + czech = 'cz', + dutch = 'nl', + english = 'en', + french = 'fr', + german = 'de', + italian = 'it', + persian = 'pe', + romanian = 'ro', +} + +local lastinterface, lastcommand, lastsource = "en", "", "" + +local function doit(configuration,filename,hashed) + + local formats = document.setups.formats + + local start = os.clock() + + local detail = aux.settings_to_hash(hashed.query or "") + + lastinterface, lastcommand, lastsource = detail.interface or lastinterface, detail.command or lastcommand, detail.source or lastsource + + if lastinterface then + logs.simple("checking interface: %s",lastinterface) + document.setups.load(format("cont-%s.xml",lastinterface)) + end + + local div = document.setups.div[lastinterface] + local span = document.setups.span[lastinterface] + + local result = { content = "error" } + + local names, refs, ints = document.setups.names(lastinterface), { }, { } + for k,v in ipairs(names) do + refs[k] = document.setups.formats.href:format(v[1],v[2]) + end + for k,v in ipairs(table.sortedkeys(interfaces)) do + ints[k] = document.setups.formats.interface:format(interfaces[v],v) + end + + lmx.restore() + lmx.set('title', 'ConTeXt Help Information') + lmx.set('color-background-one', lmx.get('color-background-green')) + lmx.set('color-background-two', lmx.get('color-background-blue')) + + local n = concat(refs,"<br/>") + local i = concat(ints,"<br/><br/>") + + if div then + lmx.set('names',div:format(n)) + lmx.set('interfaces',div:format(i)) + else + lmx.set('names', n) + lmx.set('interfaces', i) + end + + -- first we need to add information about mkii/mkiv + + if document.setups.showsources and lastsource and lastsource ~= "" then + -- todo: mkii, mkiv, tex (can be different) + local data = io.loaddata(resolvers.find_file(lastsource)) + lmx.set('maintitle', lastsource) + lmx.set('maintext', formats.listing:format(data)) + lastsource = "" + elseif lastcommand and lastcommand ~= "" then + local data = document.setups.collect(lastcommand,lastinterface) + if data then + lmx.set('maintitle', data.sequence) + local extra = { } + for k, v in ipairs { "environment", "category", "source" } do + if data[v] and data[v] ~= "" then + lmx.set(v, data[v]) + extra[#extra+1] = v .. ": " .. data[v] + end + end + lmx.set('extra', concat(extra,", ")) + lmx.set('maintext', formats.parameters:format(concat(data.parameters))) + else + lmx.set('maintext', "select command") + end + else + lmx.set('maintext', "no definition") + end + + local content = lmx.convert('context-help.lmx') + + logs.simple("time spent on page: %0.03f seconds",os.clock()-start) + + return { content = content } +end + +return doit, true diff --git a/scripts/context/lua/mtx-server-ctx-startup.lua b/scripts/context/lua/mtx-server-ctx-startup.lua new file mode 100644 index 000000000..fcb757b3e --- /dev/null +++ b/scripts/context/lua/mtx-server-ctx-startup.lua @@ -0,0 +1,53 @@ +if not modules then modules = { } end modules ['mtx-server-ctx-startup'] = { + version = 1.001, + comment = "Overview Of Goodies", + author = "Hans Hagen", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +dofile(resolvers.find_file("trac-lmx.lua","tex")) + +function doit(configuration,filename,hashed) + + lmx.restore() + + lmx.variables['color-background-green'] = '#4F6F6F' + lmx.variables['color-background-blue'] = '#6F6F8F' + lmx.variables['color-background-yellow'] = '#8F8F6F' + lmx.variables['color-background-purple'] = '#8F6F8F' + + lmx.variables['color-background-body'] = '#808080' + lmx.variables['color-background-main'] = '#3F3F3F' + lmx.variables['color-background-one'] = lmx.variables['color-background-green'] + lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] + + lmx.variables['title'] = "Overview Of Goodies" + + lmx.set('title', lmx.get('title')) + lmx.set('color-background-one', lmx.get('color-background-green')) + lmx.set('color-background-two', lmx.get('color-background-blue')) + + + local list = { } + local root = file.dirname(resolvers.find_file("mtx-server.lua") or ".") + if root == "" then root = "." end + local pattern = root .. "/mtx-server-ctx-*.lua" + local files = dir.glob(pattern) + for i=1,#files do + local filename = file.basename(files[i]) + local name = string.match(filename,"mtx%-server%-ctx%-(.-)%.lua$") + if name and name ~= "startup" then + list[#list+1] = string.format("<a href='%s' target='ctx-%s'>%s</a><br/><br/>",filename,name,name) + end + end + + lmx.set('maintext',table.concat(list,"\n")) + + result = { content = lmx.convert('context-base.lmx') } + + return result + +end + +return doit, true diff --git a/scripts/context/lua/mtx-server.lua b/scripts/context/lua/mtx-server.lua index d9eb355f6..74f0ed924 100644 --- a/scripts/context/lua/mtx-server.lua +++ b/scripts/context/lua/mtx-server.lua @@ -9,9 +9,10 @@ if not modules then modules = { } end modules ['mtx-server'] = { scripts = scripts or { } scripts.webserver = scripts.webserver or { } -dofile(input.find_file("l-url.lua")) +dofile(resolvers.find_file("l-url.lua","tex")) +dofile(resolvers.find_file("luat-soc.lua","tex")) -local socket = require("socket") +local socket = socket or require("socket") -- redundant in future version local format = string.format -- The following two lists are taken from webrick (ruby) and @@ -126,13 +127,39 @@ local handlers = { } local function errormessage(client,configuration,n) local data = format("<head><title>%s %s</title></head><html><h2>%s %s</h2></html>",n,messages[n],n,messages[n]) - input.report("handling error %s: %s",n,messages[n]) + logs.simple("handling error %s: %s",n,messages[n]) handlers.generic(client,configuration,data,nil,true) end +local validpaths, registered = { }, { } + +function scripts.webserver.registerpath(name) + if not registered[name] then + local cleanname = string.gsub(name,"%.%.","deleted-parent") + logs.simple("registering path '%s'",cleanname) + validpaths[#validpaths+1] = cleanname + registered[name] = true + end +end + function handlers.generic(client,configuration,data,suffix,iscontent) if not iscontent then - data = io.loaddata(file.join(configuration.root,data)) + local name = data + logs.simple("requested file '%s'",name) + local fullname = file.join(configuration.root,name) + data = io.loaddata(fullname) or "" + if data == "" then + for n=1,#validpaths do + local fullname = file.join(validpaths[n],name) + data = io.loaddata(fullname) or "" + if data ~= "" then + logs.simple("sending generic file '%s'",fullname) + break + end + end + else + logs.simple("sending generic file '%s'",fullname) + end end if data and data ~= "" then client:send("HTTP/1.1 200 OK\r\n") @@ -155,21 +182,37 @@ end --~ return { content = filename } --~ end +local loaded = { } + function handlers.lua(client,configuration,filename,suffix,iscontent,hashed) -- filename will disappear, and become hashed.filename local filename = file.join(configuration.scripts,filename) - if not input.aux.qualified_path(filename) then + if not file.is_qualified_path(filename) then filename = file.join(configuration.root,filename) end -- todo: split url in components, see l-url; rather trivial - input.report("locating script: %s",filename) - if lfs.isfile(filename) then - local result = loadfile(filename) - input.report("return type: %s",type(result)) - if result and type(result) == "function" then - -- result() should return a table { [type=,] [length=,] content= }, function or string - result = result() + local result, keep = loaded[filename], false + if result then + logs.simple("reusing script: %s",filename) + else + logs.simple("locating script: %s",filename) + if lfs.isfile(filename) then + logs.simple("loading script: %s",filename) + result = loadfile(filename) + logs.simple("return type: %s",type(result)) + if result and type(result) == "function" then + -- result() should return a table { [type=,] [length=,] content= }, function or string + result, keep = result() + if keep then + logs.simple("saving script: %s",type(result)) + loaded[filename] = result + end + end + else + errormessage(client,configuration,404) end - if result and type(result) == "function" then + end + if result then + if type(result) == "function" then result = result(configuration,filename,hashed) -- second argument will become query end if result and type(result) == "string" then @@ -181,9 +224,9 @@ function handlers.lua(client,configuration,filename,suffix,iscontent,hashed) -- local action = handlers[suffix] or handlers.generic action(client,configuration,result.content,suffix,true) -- content elseif result.filename then - local suffix = file.extname(filename) or "text/html" + local suffix = file.extname(result.filename) or "text/html" local action = handlers[suffix] or handlers.generic - action(client,configuration,filename,suffix,false) -- filename + action(client,configuration,result.filename,suffix,false) -- filename else errormessage(client,configuration,404) end @@ -198,18 +241,19 @@ end handlers.luc = handlers.lua handlers.html = handlers.htm -local indices = { "index.htm", "index.html" } +local indices = { "index.htm", "index.html" } +local portnumber = 31415 -- pi suits tex function scripts.webserver.run(configuration) -- check configuration - configuration.port = tonumber(configuration.port or os.getenv("MTX_SERVER_PORT") or 8080) or 8080 + configuration.port = tonumber(configuration.port or os.getenv("MTX_SERVER_PORT") or portnumber) or portnumber if not configuration.root or not lfs.isdir(configuration.root) then configuration.root = os.getenv("MTX_SERVER_ROOT") or "." end -- locate root and index file in tex tree if not lfs.isdir(configuration.root) then for _, name in ipairs(indices) do - local root = input.resolve("path:" .. name) or "" + local root = resolvers.resolve("path:" .. name) or "" if root ~= "" then configuration.root = root configuration.index = configuration.index or name @@ -217,6 +261,7 @@ function scripts.webserver.run(configuration) end end end + configuration.root = dir.expand_name(configuration.root) if not configuration.index then for _, name in ipairs(indices) do if lfs.isfile(file.join(configuration.root,name)) then @@ -226,14 +271,17 @@ function scripts.webserver.run(configuration) end configuration.index = configuration.index or "unknown" end - configuration.scripts = configuration.scripts or "cgi" + if not configuration.scripts or configuration.scripts == "" then + configuration.scripts = dir.expand_name(file.join(configuration.root or ".",configuration.scripts or ".")) + end -- so far for checks - input.report("running at port: %s",configuration.port) - input.report("document root: %s",configuration.root) - input.report("main index file: %s",configuration.index) - input.report("scripts subpath: %s",configuration.scripts) + logs.simple("running at port: %s",configuration.port) + logs.simple("document root: %s",configuration.root or resolvers.ownpath) + logs.simple("main index file: %s",configuration.index) + logs.simple("scripts subpath: %s",configuration.scripts) local server = assert(socket.bind("*", configuration.port)) while true do -- no multiple clients + local start = os.clock() local client = server:accept() client:settimeout(configuration.timeout or 60) local request, e = client:receive() @@ -241,26 +289,26 @@ function scripts.webserver.run(configuration) errormessage(client,configuration,404) else local from = client:getpeername() - input.report("request from: %s",tostring(from)) + logs.simple("request from: %s",tostring(from)) local fullurl = request:match("GET (.+) HTTP/.*$") -- todo: more clever -fullurl = socket.url.unescape(fullurl) -local hashed = url.hashed(fullurl) -local query = url.query(hashed.query) -filename = hashed.path + fullurl = socket.url.unescape(fullurl) + local hashed = url.hashed(fullurl) + local query = url.query(hashed.query) + local filename = hashed.path if filename then filename = socket.url.unescape(filename) - input.report("requested action: %s",filename) + logs.simple("requested action: %s",filename) if filename:find("%.%.") then filename = nil -- invalid path end if filename == nil or filename == "" or filename == "/" then filename = configuration.index - input.report("invalid filename, forcing: %s",filename) + logs.simple("invalid filename, forcing: %s",filename) end local suffix = file.extname(filename) local action = handlers[suffix] or handlers.generic if action then - input.report("performing action: %s",filename) + logs.simple("performing action: %s",filename) action(client,configuration,filename,suffix,false,hashed) -- filename and no content else errormessage(client,configuration,404) @@ -270,10 +318,11 @@ filename = hashed.path end end client:close() + logs.simple("time spent with client: %0.03f seconds",os.clock()-start) end end -banner = banner .. " | webserver " +logs.extendbanner("Simple Webserver 0.10") messages.help = [[ --start start server @@ -281,15 +330,26 @@ messages.help = [[ --root server root --scripts scripts sub path --index index file +--auto start on own path ]] -if environment.argument("start") then +if environment.argument("auto") then + local path = resolvers.find_file("mtx-server.lua") or "." scripts.webserver.run { port = environment.argument("port"), - root = environment.argument("root"), -- "e:/websites/www.pragma-ade.com", + root = environment.argument("root") or file.dirname(path) or ".", + scripts = environment.argument("scripts") or file.dirname(path) or ".", + } +elseif environment.argument("start") then + scripts.webserver.run { + port = environment.argument("port"), + root = environment.argument("root") or ".", -- "e:/websites/www.pragma-ade.com", index = environment.argument("index"), - scripts = environment.argument("scripts") or "cgi", + scripts = environment.argument("scripts"), } else - input.help(banner,messages.help) + logs.help(messages.help) end + + +-- mtxrun --script server --start => http://localhost:8080/mtx-server-ctx-help.lua diff --git a/scripts/context/lua/mtx-timing.lua b/scripts/context/lua/mtx-timing.lua new file mode 100644 index 000000000..1dcb9aa0e --- /dev/null +++ b/scripts/context/lua/mtx-timing.lua @@ -0,0 +1,201 @@ +if not modules then modules = { } end modules ['mtx-timing'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, gsub, concat = string.format, string.gsub, table.concat + +dofile(resolvers.find_file("trac-tim.lua","tex")) +dofile(resolvers.find_file("trac-lmx.lua","tex")) + +local meta = [[ + beginfig(%s) ; + begingroup ; + save p, q, b, h, w ; + path p, q, b ; numeric h, w ; + linecap := butt ; + h := 100 ; + w := 800pt ; + p := %s ; + q := %s ; + p := p shifted -llcorner p ; + q := q shifted -llcorner q ; + q := q xstretched w ; + p := p xstretched w ; + b := boundingbox (llcorner p -- llcorner p shifted (w,h)) ; + draw b withcolor white withpen pencircle scaled 4pt ; + draw p withcolor red withpen pencircle scaled 4pt ; + draw q withcolor blue withpen pencircle scaled 2pt ; + endgroup ; + endfig ; +]] + +local html_graphic = [[ + <h1><a name='graphic-%s'>%s (red) %s (blue)</a></h1> + <table> + <tr> + <td>%s</td> + <td valign='top'> + min: %s<br/> + max: %s<br/> + pages: %s<br/> + average: %s<br/> + </td> + </tr> + </table> + <br/> +]] + +local html_menu = [[ + <a href='#graphic-%s'>%s</a> +]] + +local directrun = true + +function goodies.progress.make_svg(filename,other) + local metadata, menudata, c = { }, { }, 0 + metadata[#metadata+1] = 'outputformat := "svg" ;' + for _, kind in pairs { "parameters", "nodes" } do + local mdk = { } + menudata[kind] = mdk + for n, name in pairs(goodies.progress[kind](filename)) do + local first = goodies.progress.path(filename,name) + local second = goodies.progress.path(filename,other) + c = c + 1 + metadata[#metadata+1] = format(meta,c,first,second) + mdk[#mdk+1] = { name, c } + end + end + metadata[#metadata+1] = "end ." + metadata = concat(metadata,"\n\n") + if directrun then + dofile(resolvers.find_file("mlib-run.lua","tex")) + commands = commands or { } + commands.writestatus = logs.report + local result = metapost.directrun("metafun","timing data","svg",true,metadata) + return menudata, result + else + local mpname = file.replacesuffix(filename,"mp") + io.savedata(mpname,metadata) + os.execute(format("mpost --progname=context --mem=metafun.mem %s",mpname)) + os.remove(mpname) + os.remove(file.removesuffix(filename).."-mpgraph.mpo") -- brr + os.remove(file.removesuffix(filename)..".log") -- brr + return menudata + end +end + +function goodies.progress.makehtml(filename,other,menudata,metadata) + local graphics = { } + local result = { graphics = graphics } + for _, kind in pairs { "parameters", "nodes" } do + local md = menudata[kind] + local menu = { } + result[kind] = menu + for k, v in ipairs(md) do + local name, number = v[1], v[2] + local min = goodies.progress.bot(filename,name) + local max = goodies.progress.top(filename,name) + local pages = goodies.progress.pages(filename) + local average = math.round(max/pages) + if directrun then + local data = metadata[number] + menu[#menu+1] = format(html_menu,name,name) + graphics[#graphics+1] = format(html_graphic,name,name,other,data,min,max,pages,average) + else + local mpname = file.replacesuffix(filename,number) + local data = io.loaddata(mpname) or "" + -- data = gsub(data,"<!%-%-(.-)%-%->[\n\r]*","") + data = gsub(data,"<%?xml.->","") + menu[#menu+1] = format(html_menu,name,name) + graphics[#graphics+1] = format(html_graphic,name,name,other,data,min,max,pages,average) + os.remove(mpname) + end + end + end + return result +end + +function goodies.progress.valid_file(name) + return name and name ~= "" and lfs.isfile(name .. "-luatex-progress.lut") +end + +function goodies.progress.make_lmx_page(name,launch,remove) + local filename = name .. "-luatex-progress" + local other = "elapsed_time" + local template = 'context-timing.lmx' + + lmx.variables['color-background-green'] = '#4F6F6F' + lmx.variables['color-background-blue'] = '#6F6F8F' + lmx.variables['color-background-yellow'] = '#8F8F6F' + lmx.variables['color-background-purple'] = '#8F6F8F' + + lmx.variables['color-background-body'] = '#808080' + lmx.variables['color-background-main'] = '#3F3F3F' + lmx.variables['color-background-one'] = lmx.variables['color-background-green'] + lmx.variables['color-background-two'] = lmx.variables['color-background-blue'] + + lmx.variables['title-default'] = 'ConTeXt Timing Information' + lmx.variables['title'] = lmx.variables['title-default'] + + lmx.htmfile = function(name) return name .. "-timing.xhtml" end + lmx.lmxfile = function(name) return resolvers.find_file(name,'tex') end + + lmx.set('title', format('ConTeXt Timing Information: %s',file.basename(name))) + lmx.set('color-background-one', lmx.get('color-background-green')) + lmx.set('color-background-two', lmx.get('color-background-blue')) + + goodies.progress.convert(filename) + + local menudata, metadata = goodies.progress.make_svg(filename,other) + local htmldata = goodies.progress.makehtml(filename,other,menudata,metadata) + + lmx.set('parametersmenu', concat(htmldata.parameters, " ")) + lmx.set('nodesmenu', concat(htmldata.nodes, " ")) + lmx.set('graphics', concat(htmldata.graphics, "\n\n")) + + if launch then + local htmfile = lmx.show(template) + if remove then + os.sleep(1) -- give time to launch + os.remove(htmfile) + end + else + lmx.make(template) + end + + lmx.restore() +end + +scripts = scripts or { } +scripts.timings = scripts.timings or { } + +function scripts.timings.xhtml(filename) + if filename == "" then + logs.simple("provide filename") + elseif not goodies.progress.valid_file(filename) then + logs.simple("first run context again with the --timing option") + else + local basename = file.removesuffix(filename) + local launch = environment.argument("launch") + local remove = environment.argument("remove") + goodies.progress.make_lmx_page(basename,launch,remove) + end +end + +logs.extendbanner("ConTeXt Timing Tools 0.1",true) + +messages.help = [[ +--xhtml make xhtml file +--launch launch after conversion +--remove remove after launching +]] + +if environment.argument("xhtml") then + scripts.timings.xhtml(environment.files[1] or "") +else + logs.help(messages.help) +end diff --git a/scripts/context/lua/mtx-unzip.lua b/scripts/context/lua/mtx-unzip.lua new file mode 100644 index 000000000..f990f4210 --- /dev/null +++ b/scripts/context/lua/mtx-unzip.lua @@ -0,0 +1,101 @@ +-- maybe --pattern + +logs.extendbanner("Simple Unzipper 0.10") + +messages.help = [[ +--list list files in archive +--junk flatten unzipped directory structure +--extract extract files +]] + +scripts = scripts or { } +scripts.unzipper = scripts.unzipper or { } + +function scripts.unzipper.help() + logs.help(messages.help) +end + +function scripts.unzipper.opened() + local filename = environment.files[1] + if filename and filename ~= "" then + filename = file.addsuffix(filename,'zip') + local zipfile = zip.open(filename) + if zipfile then + return zipfile + end + end + logs.report("unzip", "no zip file: " .. filename) + return false +end + +function scripts.unzipper.list() + local zipfile = scripts.unzipper.opened() + if zipfile then + local n = 0 + for k in zipfile:files() do + if #k.filename > n then n = #k.filename end + end + local files, paths, compressed, uncompressed = 0, 0, 0, 0 + for k in zipfile:files() do + if k.filename:find("/$") then + paths = paths + 1 + print(string.format("%s", k.filename:rpadd(n," "))) + else + files = files + 1 + local cs, us = k.compressed_size, k.uncompressed_size + if cs > compressed then + compressed = cs + end + if us > uncompressed then + uncompressed = us + end + print(string.format("%s % 9i % 9i", k.filename:rpadd(n," "),cs,us)) + end + end + print(string.format("\n%s % 9i % 9i", (files .. " files, " .. paths .. " directories"):rpadd(n," "),compressed,uncompressed)) + end +end + +function zip.loaddata(zipfile,filename) + local f = zipfile:open(filename) + if f then + local data = f:read("*a") + f:close() + return data + end + return nil +end + +function scripts.unzipper.extract() + local zipfile = scripts.unzipper.opened() + if zipfile then + local junk = environment.arguments["j"] or environment.arguments["junk"] + for k in zipfile:files() do + local filename = k.filename + if filename:find("/$") then + if not junk then + lfs.mkdir(filename) + end + else + local data = zip.loaddata(zipfile,filename) + if data then + if junk then + filename = file.basename(filename) + end + io.savedata(filename,data) + print(filename) + end + end + end + end +end + +if environment.arguments["h"] or environment.arguments["help"] then + scripts.unzipper.help() +elseif environment.arguments["l"] or environment.arguments["list"] then + scripts.unzipper.list(zipfile) +elseif environment.files[1] then -- implicit --extract + scripts.unzipper.extract(zipfile) +else + scripts.unzipper.help() +end diff --git a/scripts/context/lua/mtx-update.lua b/scripts/context/lua/mtx-update.lua index b56780c68..66f6898d3 100644 --- a/scripts/context/lua/mtx-update.lua +++ b/scripts/context/lua/mtx-update.lua @@ -1,5 +1,5 @@ if not modules then modules = { } end modules ['mtx-update'] = { - version = 1.001, + version = 1.002, comment = "companion to mtxrun.lua", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", @@ -11,15 +11,20 @@ if not modules then modules = { } end modules ['mtx-update'] = { -- Together with Arthur Reutenauer she made sure that it worked well on all -- platforms that matter. +local format, concat, gmatch = string.format, table.concat, string.gmatch + scripts = scripts or { } scripts.update = scripts.update or { } minimals = minimals or { } minimals.config = minimals.config or { } +-- this is needed under windows +-- else rsync fails to set the right chmod flags to files + os.setenv("CYGWIN","nontsec") -scripts.update.allformats = { +scripts.update.texformats = { "cont-en", "cont-nl", "cont-cz", @@ -28,125 +33,105 @@ scripts.update.allformats = { "cont-it", "cont-ro", "cont-uk", - "metafun", + "cont-pe", + "cont-xp", "mptopdf", "plain" } -scripts.update.fewformats = { - "cont-en", - "cont-nl", +scripts.update.mpformats = { "metafun", - "mptopdf", - "plain" + "mpost", } +-- experimental is not functional at the moment + scripts.update.repositories = { "current", "experimental" } +-- more options than just these two are available (no idea why this is here) + scripts.update.versions = { "current", "latest" } +-- list of basic folders that are needed to make a functional distribution + +scripts.update.base = { + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, -- not *really* needed, but helpful + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" }, +} + +-- binaries and font-related files +-- for pdftex we don't need OpenType fonts, for LuaTeX/XeTeX we don't need TFM files + scripts.update.engines = { ["luatex"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, { "fonts/new/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/<version>/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "context/config/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/<platform>/", "texmf-<platform>" }, - { "bin/context/<platform>/", "texmf-<platform>" }, - { "bin/metapost/<platform>/", "texmf-<platform>" }, { "bin/luatex/<platform>/", "texmf-<platform>" }, - { "bin/man/", "texmf-<platform>" } }, ["xetex"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, { "base/xetex/", "texmf" }, { "fonts/new/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/<version>/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "context/config/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/<platform>/", "texmf-<platform>" }, - { "bin/context/<platform>/", "texmf-<platform>" }, - { "bin/metapost/<platform>/", "texmf-<platform>" }, { "bin/xetex/<platform>/", "texmf-<platform>" }, - { "bin/man/", "texmf-<platform>" } }, ["pdftex"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, { "fonts/old/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/<version>/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "context/config/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/<platform>/", "texmf-<platform>" }, - { "bin/context/<platform>/", "texmf-<platform>" }, - { "bin/metapost/<platform>/", "texmf-<platform>" }, { "bin/pdftex/<platform>/", "texmf-<platform>" }, - { "bin/man/", "texmf-<platform>" } }, ["all"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, - { "base/xetex/", "texmf" }, - { "fonts/old/", "texmf" }, { "fonts/new/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/<version>/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "context/config/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/<platform>/", "texmf-<platform>" }, - { "bin/context/<platform>/", "texmf-<platform>" }, - { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "fonts/old/", "texmf" }, + { "base/xetex/", "texmf" }, { "bin/luatex/<platform>/", "texmf-<platform>" }, { "bin/xetex/<platform>/", "texmf-<platform>" }, { "bin/pdftex/<platform>/", "texmf-<platform>" }, - { "bin/man/", "texmf-<platform>" } }, } scripts.update.platforms = { - ["mswin"] = "mswin", - ["windows"] = "mswin", - ["win32"] = "mswin", - ["win"] = "mswin", - ["linux"] = "linux", - ["freebsd"] = "freebsd", - ["linux-32"] = "linux", - ["linux-64"] = "linux-64", - ["linux32"] = "linux", - ["linux64"] = "linux-64", - ["linux-ppc"] = "linux-ppc", - ["ppc"] = "linux-ppc", - ["osx"] = "osx-intel", - ["osx-intel"] = "osx-intel", - ["osx-ppc"] = "osx-ppc", - ["osx-powerpc"] = "osx-ppc", - ["osxintel"] = "osx-intel", - ["osxppc"] = "osx-ppc", - ["osxpowerpc"] = "osx-ppc", + ["mswin"] = "mswin", + ["windows"] = "mswin", + ["win32"] = "mswin", + ["win"] = "mswin", + ["linux"] = "linux", + ["freebsd"] = "freebsd", + ["freebsd-amd64"] = "freebsd-amd64", + ["linux-32"] = "linux", + ["linux-64"] = "linux-64", + ["linux32"] = "linux", + ["linux64"] = "linux-64", + ["linux-ppc"] = "linux-ppc", + ["ppc"] = "linux-ppc", + ["osx"] = "osx-intel", + ["macosx"] = "osx-intel", + ["osx-intel"] = "osx-intel", + ["osx-ppc"] = "osx-ppc", + ["osx-powerpc"] = "osx-ppc", + ["osxintel"] = "osx-intel", + ["osxppc"] = "osx-ppc", + ["osxpowerpc"] = "osx-ppc", + ["solaris-intel"] = "solaris-intel", + ["solaris-sparc"] = "solaris-sparc", + ["solaris"] = "solaris-sparc", +} + +-- the list is filled up later (when we know what modules to download) + +scripts.update.modules = { } function scripts.update.run(str) @@ -160,7 +145,7 @@ function scripts.update.run(str) end function scripts.update.fullpath(path) - if input.aux.rootbased_path(path) then + if file.is_rootbased_path(path) then return path else return lfs.currentdir() .. "/" .. path @@ -168,47 +153,138 @@ function scripts.update.fullpath(path) end function scripts.update.synchronize() + logs.report("update","start") + local texroot = scripts.update.fullpath(states.get("paths.root")) - local engines = states.get('engines') - local platforms = states.get('platforms') - local repositories = states.get('repositories') - local bin = states.get("rsync.program") - local url = states.get("rsync.server") - local version = states.get("context.version") + local engines = states.get('engines') or { } + local platforms = states.get('platforms') or { } + local repositories = states.get('repositories') -- minimals + local bin = states.get("rsync.program") -- rsync + local url = states.get("rsync.server") -- contextgarden.net + local version = states.get("context.version") -- current (or beta) + local extras = states.get("extras") -- extra goodies (like modules) local force = environment.argument("force") + + bin = string.gsub(bin,"\\","/") + if not url:find("::$") then url = url .. "::" end local ok = lfs.attributes(texroot,"mode") == "directory" if not ok and force then dir.mkdirs(texroot) ok = lfs.attributes(texroot,"mode") == "directory" end + + if force then + dir.mkdirs(format("%s/%s", texroot, "texmf-cache")) + dir.mkdirs(format("%s/%s", texroot, "texmf-local")) + end + if ok or not force then - if force then - dir.mkdirs(string.format("%s/%s", texroot, "texmf-cache")) + + local fetched, individual, osplatform = { }, { }, os.currentplatform() + + -- takes a collection as argument and returns a list of folders + + local function collection_to_list_of_folders(collection, platform) + local archives = {} + for _, c in ipairs(collection) do + local archive = c[1] + archive = archive:gsub("<platform>", platform) + archive = archive:gsub("<version>", version) + archives[#archives+1] = archive + end + return archives end - local fetched, individual = { }, { } - for engine, _ in pairs(engines) do - local collections = scripts.update.engines[engine] - if collections then - for _, collection in ipairs(collections) do - for platform, _ in pairs(platforms) do - platform = scripts.update.platforms[platform] - if platform then - local archive = collection[1]:gsub("<platform>", platform) - local destination = string.format("%s/%s", texroot, collection[2]:gsub("<platform>", platform)) - destination = destination:gsub("\\","/") - archive = archive:gsub("<version>",version) ---~ if platform == "windows" or platform == "mswin" then - if os.currentplatform() == "windows" or os.currentplatform() == "mswin" then - destination = destination:gsub("([a-zA-Z]):/", "/cygdrive/%1/") - end - individual[#individual+1] = { archive, destination } + + -- takes a list of folders as argument and returns a string for rsync + -- sample input: + -- {'bin/common', 'bin/context'} + -- output: + -- 'minimals/current/bin/common minimals/current/bin/context' + + local function list_of_folders_to_rsync_string(list_of_folders) + local repository = 'current' + local prefix = format("%s/%s/", states.get('rsync.module'), repository) -- minimals/current/ + + return prefix .. concat(list_of_folders, format(" %s", prefix)) + end + + -- example of usage: print(list_of_folders_to_rsync_string(collection_to_list_of_folders(scripts.update.base, os.currentplatform))) + + -- rename function and add some more functionality: + -- * recursive/non-recursive (default: non-recursive) + -- * filter folders or regular files only (default: no filter) + -- * grep for size of included files (with --stats switch) + + local function get_list_of_files_from_rsync(list_of_folders) + -- temporary file to store the output of rsync (could be a more random name; watch for overwrites) + local temp_file = "rsync.tmp.txt" + -- a set of folders + local folders = {} + local command = format("%s %s'%s' > %s", bin, url, list_of_folders_to_rsync_string(list_of_folders), temp_file) + os.execute(command) + -- read output of rsync + local data = io.loaddata(temp_file) or "" + -- for every line extract the filename + for chmod, s in data:gmatch("([d%-][rwx%-]+).-(%S+)[\n\r]") do + -- skip "current" folder + if s ~= '.' and chmod:len() == 10 then + folders[#folders+1] = s + end + end + -- delete the file to which we have put output of rsync + os.remove(temp_file) + return folders + end + + -- rsync://contextgarden.net/minimals/current/modules/ + + if extras and type(extras) == "table" then + -- fetch the list of available modules from rsync server + local available_modules = get_list_of_files_from_rsync({"modules/"}) + -- hash of requested modules + -- local h = table.tohash(extras:split(",")) + for _, s in ipairs(available_modules) do + -- if extras == "all" or h[s] then + if extras.all or extras[s] then + scripts.update.modules[#scripts.update.modules+1] = { format("modules/%s/",s), "texmf-context" } + end + end + -- TODO: check if every module from the list has been added and issue warning otherwise + -- one idea to do it: remove every value from h once added and then check if anything is left in h + end + + local function add_collection(collection,platform) + if collection and platform then + platform = scripts.update.platforms[platform] + if platform then + for _, c in ipairs(collection) do + local archive = c[1]:gsub("<platform>", platform) + local destination = format("%s/%s", texroot, c[2]:gsub("<platform>", platform)) + destination = destination:gsub("\\","/") + archive = archive:gsub("<version>",version) + if osplatform == "windows" or osplatform == "mswin" then + destination = destination:gsub("([a-zA-Z]):/", "/cygdrive/%1/") end + individual[#individual+1] = { archive, destination } end end end end + + for platform, _ in pairs(platforms) do + add_collection(scripts.update.base,platform) + end + for platform, _ in pairs(platforms) do + add_collection(scripts.update.modules,platform) + end + for engine, _ in pairs(engines) do + for platform, _ in pairs(platforms) do + add_collection(scripts.update.engines[engine],platform) + end + end + local combined = { } for _, repository in ipairs(scripts.update.repositories) do if repositories[repository] then @@ -219,11 +295,11 @@ function scripts.update.synchronize() cd = { } combined[destination] = cd end - cd[#cd+1] = string.format("%s/%s/%s",states.get('rsync.module'),repository,archive) + cd[#cd+1] = format("%s/%s/%s",states.get('rsync.module'),repository,archive) end end end - if input.verbose then + if logs.verbose then for k, v in pairs(combined) do logs.report("update", k) for k,v in ipairs(v) do @@ -232,24 +308,53 @@ function scripts.update.synchronize() end end for destination, archive in pairs(combined) do - local archives, command = table.concat(archive," "), "" - local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete") - if true then -- environment.argument("keep") or destination:find("%.$") then - command = string.format("%s %s %s'%s' '%s'", bin, normalflags, url, archives, destination) - else - command = string.format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination) + local archives, command = concat(archive," "), "" + -- local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete") + -- if environment.argument("keep") or destination:find("%.$") then + -- command = format("%s %s %s'%s' '%s'", bin, normalflags, url, archives, destination) + -- else + -- command = format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination) + -- end + local normalflags, deleteflags = states.get("rsync.flags.normal"), "" + if (destination:find("texmf$") or destination:find("texmf%-context$")) and (not environment.argument("keep")) then + deleteflags = states.get("rsync.flags.delete") end - logs.report("mtx update", string.format("running command: %s",command)) + command = format("%s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, url, archives, destination) + logs.report("mtx update", format("running command: %s",command)) if not fetched[command] then scripts.update.run(command) fetched[command] = command end end + + local function update_script(script, platform) + local bin = bin:gsub("\\","/") + local texroot = texroot:gsub("\\","/") + platform = scripts.update.platforms[platform] + if platform then + local command + if platform == 'mswin' then + bin = bin:gsub("([a-zA-Z]):/", "/cygdrive/%1/") + texroot = texroot:gsub("([a-zA-Z]):/", "/cygdrive/%1/") + command = string.format("%s -t %s/texmf-context/scripts/context/lua/%s.lua %s/texmf-mswin/bin/", bin, texroot, script, texroot) + else + command = string.format("%s -tgo --chmod=a+x %s/texmf-context/scripts/context/lua/%s.lua %s/texmf-%s/bin/%s", bin, texroot, script, texroot, platform, script) + end + logs.report("mtx update", format("updating %s for %s: %s", script, platform, command)) + scripts.update.run(command) + end + end + + for platform, _ in pairs(platforms) do + update_script('luatools',platform) + update_script('mtxrun',platform) + end + else - logs.report("mtx update", string.format("no valid texroot: %s",texroot)) + logs.report("mtx update", format("no valid texroot: %s",texroot)) end if not force then - logs.report("update", "use --force to really update") + logs.report("update", "use --force to really update files") end logs.report("update","done") end @@ -262,32 +367,59 @@ function table.fromhash(t) return h end - +-- make the ConTeXt formats function scripts.update.make() + logs.report("make","start") + local force = environment.argument("force") local texroot = scripts.update.fullpath(states.get("paths.root")) - local engines = states.get('engines') + local engines= states.get('engines') local platforms = states.get('platforms') local formats = states.get('formats') - input.load_tree(texroot) + + resolvers.load_tree(texroot) + -- update filename database for pdftex/xetex scripts.update.run("mktexlsr") + -- update filename database for luatex scripts.update.run("luatools --generate") - local formatlist = table.concat(table.fromhash(formats), " ") + local askedformats = formats + local texformats = table.tohash(scripts.update.texformats) + local mpformats = table.tohash(scripts.update.mpformats) + for k,v in pairs(texformats) do + if not askedformats[k] then + texformats[k] = nil + end + end + for k,v in pairs(mpformats) do + if not askedformats[k] then + mpformats[k] = nil + end + end + local formatlist = concat(table.fromhash(texformats), " ") if formatlist ~= "" then for engine in pairs(engines) do - -- todo: just handle make here or in mtxrun --script context --make ---~ os.execute("set") - scripts.update.run(string.format("texexec --make --all --fast --%s %s",engine,formatlist)) + if engine == "luatex" then + scripts.update.run(format("context --make")) -- maybe also formatlist + else + -- todo: just handle make here or in mtxrun --script context --make + scripts.update.run(format("texexec --make --all --fast --%s %s",engine,formatlist)) + end end end + local formatlist = concat(table.fromhash(mpformats), " ") + if formatlist ~= "" then + scripts.update.run(format("texexec --make --all --fast %s",formatlist)) + end if not force then - logs.report("make", "use --force to really make") + logs.report("make", "use --force to really make formats") end + scripts.update.run("mktexlsr") + scripts.update.run("luatools --generate") logs.report("make","done") end -banner = banner .. " | download tools " +logs.extendbanner("Download Tools 0.20",true) messages.help = [[ --platform=string platform (windows, linux, linux-64, osx-intel, osx-ppc, linux-ppc) @@ -296,8 +428,9 @@ messages.help = [[ --repository=string specify version (current, experimental) --context=string specify version (current, latest, yyyy.mm.dd) --rsync=string rsync binary (rsync) ---texroot installation directory (not guessed for the moment) ---engine tex engine (luatex, pdftex, xetex) +--texroot=string installation directory (not guessed for the moment) +--engine=string tex engine (luatex, pdftex, xetex) +--extras=string extra modules (can be list or 'all') --force instead of a dryrun, do the real thing --update update minimal tree --make also make formats and generate file databases @@ -305,8 +438,6 @@ messages.help = [[ --state update tree using saved state ]] -input.verbose = true - scripts.savestate = true if scripts.savestate then @@ -315,7 +446,7 @@ if scripts.savestate then -- tag, value, default, persistent - input.starttiming(states) + statistics.starttiming(states) states.set("info.version",0.1) -- ok states.set("info.count",(states.get("info.count") or 0) + 1,1,false) -- ok @@ -333,11 +464,11 @@ if scripts.savestate then states.set("context.version", environment.argument("context"), "current", true) -- ok local valid = table.tohash(scripts.update.repositories) - for r in string.gmatch(environment.argument("repository") or "current","([^, ]+)") do + for r in gmatch(environment.argument("repository") or "current","([^, ]+)") do if valid[r] then states.set("repositories." .. r, true) end end local valid = scripts.update.engines - for r in string.gmatch(environment.argument("engine") or "all","([^, ]+)") do + for r in gmatch(environment.argument("engine") or "all","([^, ]+)") do if r == "all" then for k, v in pairs(valid) do if k ~= "all" then @@ -349,12 +480,16 @@ if scripts.savestate then end end local valid = scripts.update.platforms - for r in string.gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do + for r in gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do if valid[r] then states.set("platforms." .. r, true) end end - local valid = table.tohash(scripts.update.allformats) - for r in string.gmatch(environment.argument("formats") or "","([^, ]+)") do + local valid = table.tohash(scripts.update.texformats) + for r in gmatch(environment.argument("formats") or "","([^, ]+)") do + if valid[r] then states.set("formats." .. r, true) end + end + local valid = table.tohash(scripts.update.mpformats) + for r in gmatch(environment.argument("formats") or "","([^, ]+)") do if valid[r] then states.set("formats." .. r, true) end end @@ -362,7 +497,9 @@ if scripts.savestate then states.set("formats.cont-nl", true) states.set("formats.metafun", true) - -- modules + for r in gmatch(environment.argument("extras") or "","([^, ]+)") do + states.set("extras." .. r, true) + end logs.report("state","loaded") @@ -382,12 +519,12 @@ if environment.argument("update") then elseif environment.argument("make") then scripts.update.make() else - input.help(banner,messages.help) + logs.help(messages.help) end if scripts.savestate then - input.stoptiming(states) - states.set("info.runtime",tonumber(input.elapsedtime(states))) + statistics.stoptiming(states) + states.set("info.runtime",tonumber(statistics.elapsedtime(states))) if environment.argument("force") then states.save() logs.report("state","saved") diff --git a/scripts/context/lua/mtx-watch.lua b/scripts/context/lua/mtx-watch.lua index b7b6fb77b..2e4dcf6ef 100644 --- a/scripts/context/lua/mtx-watch.lua +++ b/scripts/context/lua/mtx-watch.lua @@ -58,6 +58,20 @@ do end end end + local function toset(t) + if type(t) == "table" then + return table.concat(t,",") + else + return t + end + end + local function noset(t) + if type(t) == "table" then + return t[1] + else + return t + end + end local function process() local done = false for _, path in ipairs(environment.files) do @@ -77,8 +91,8 @@ do local command = joblog.command if command then local replacements = { - inputpath = (joblog.paths and joblog.paths.input ) or ".", - outputpath = (joblog.paths and joblog.paths.output) or ".", + inputpath = toset((joblog.paths and joblog.paths.input ) or "."), + outputpath = noset((joblog.paths and joblog.paths.output) or "."), filename = joblog.filename or "", } command = command:gsub("%%(.-)%%", replacements) @@ -225,7 +239,7 @@ function scripts.watch.show_logs(path) -- removes duplicates end end -banner = banner .. " | watchdog" +logs.extendbanner("Watchdog 1.00",true) messages.help = [[ --logpath optional path for log files @@ -236,8 +250,6 @@ messages.help = [[ --showlog show log data ]] -input.verbose = true - if environment.argument("watch") then scripts.watch.watch() elseif environment.argument("collect") then @@ -245,5 +257,5 @@ elseif environment.argument("collect") then elseif environment.argument("showlog") then scripts.watch.show_logs() else - input.help(banner,messages.help) + logs.help(messages.help) end diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 6c68ec51a..0af429bf1 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -8,6 +8,7 @@ if not modules then modules = { } end modules ['mtxrun'] = { license = "see context related readme files" } + -- one can make a stub: -- -- #!/bin/sh @@ -26,7 +27,7 @@ if not modules then modules = { } end modules ['mtxrun'] = { -- one. Interesting is that using a scripting language instead of c does -- not have a speed penalty. Actually the lua variant is more efficient, -- especially when multiple calls to kpsewhich are involved. The lua --- library also gives way more ocntrol. +-- library also gives way more control. -- to be done / considered -- @@ -37,144 +38,42 @@ if not modules then modules = { } end modules ['mtxrun'] = { -- remember for subruns: _CTX_K_S_#{original}_ -- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] -banner = "version 1.1.2 - 2007+ - PRAGMA ADE / CONTEXT" -- not local texlua = true -- begin library merge --- filename : l-string.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-string'] = 1.001 ---~ function string.split(str, pat) -- taken from the lua wiki ---~ local t = {n = 0} -- so this table has a length field, traverse with ipairs then! ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = string.find(str, fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ table.insert(t,cap) ---~ end ---~ last_end = e+1 ---~ s, e, cap = string.find(str, fpat, last_end) ---~ end ---~ if last_end<=string.len(str) then ---~ table.insert(t,(string.sub(str,last_end))) ---~ end ---~ return t ---~ end ---~ function string:split(pat) -- taken from the lua wiki but adapted ---~ local t = { } -- self and colon usage (faster) ---~ local fpat = "(.-)"..pat ---~ local last_end = 1 ---~ local s, e, cap = self:find(fpat, 1) ---~ while s ~= nil do ---~ if s~=1 or cap~="" then ---~ t[#t+1] = cap ---~ end ---~ last_end = e+1 ---~ s, e, cap = self:find(fpat, last_end) ---~ end ---~ if last_end <= #self then ---~ t[#t+1] = self:sub(last_end) ---~ end ---~ return t ---~ end +do -- create closure to overcome 200 locals limit ---~ a piece of brilliant code by Rici Lake (posted on lua list) -- only names changed ---~ ---~ function string:splitter(pat) ---~ local st, g = 1, self:gmatch("()"..pat.."()") ---~ local function splitter(self) ---~ if st then ---~ local s, f = g() ---~ local rv = self:sub(st, (s or 0)-1) ---~ st = f ---~ return rv ---~ end ---~ end ---~ return splitter, self ---~ end +if not modules then modules = { } end modules ['l-string'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -function string:splitter(pat) - -- by Rici Lake (posted on lua list) -- only names changed - -- p 79 ref man: () returns position of match - local st, g = 1, self:gmatch("()("..pat..")") - local function strgetter(self, segs, seps, sep, cap1, ...) - st = sep and seps + #sep - return self:sub(segs, (seps or 0) - 1), cap1 or sep, ... - end - local function strsplitter(self) - if st then return strgetter(self, st, g()) end - end - return strsplitter, self -end +local sub, gsub, find, match, gmatch, format, char, byte, rep = string.sub, string.gsub, string.find, string.match, string.gmatch, string.format, string.char, string.byte, string.rep -function string:split(separator) - local t = {} - for k in self:splitter(separator) do t[#t+1] = k end - return t -end +if not string.split then --- faster than a string:split: + -- this will be overloaded by a faster lpeg variant -function string:splitchr(chr) - if #self > 0 then - local t = { } - for s in (self..chr):gmatch("(.-)"..chr) do - t[#t+1] = s + function string:split(pattern) + if #self > 0 then + local t = { } + for s in gmatch(self..pattern,"(.-)"..pattern) do + t[#t+1] = s + end + return t + else + return { } end - return t - else - return { } end -end -function string.piecewise(str, pat, fnc) -- variant of split - for k in string.splitter(str,pat) do fnc(k) end end ---~ function string.piecewise(str, pat, fnc) -- variant of split ---~ for k in str:splitter(pat) do fnc(k) end ---~ end - ---~ do if lpeg then - ---~ -- this alternative is 30% faster esp when we cache them ---~ -- problem: no expressions - ---~ splitters = { } - ---~ function string:split(separator) ---~ if #self > 0 then ---~ local split = splitters[separator] ---~ if not split then ---~ -- based on code by Roberto ---~ local p = lpeg.P(separator) ---~ local c = lpeg.C((1-p)^0) ---~ split = lpeg.Ct(c*(p*c)^0) ---~ splitters[separator] = split ---~ end ---~ return split:match(self) ---~ else ---~ return { } ---~ end ---~ end - ---~ string.splitchr = string.split - ---~ function string:piecewise(separator,fnc) ---~ for _,v in pairs(self:split(separator)) do ---~ fnc(v) ---~ end ---~ end - ---~ end end - local chr_to_esc = { ["%"] = "%%", ["."] = "%.", @@ -188,20 +87,20 @@ local chr_to_esc = { string.chr_to_esc = chr_to_esc function string:esc() -- variant 2 - return (self:gsub("(.)",chr_to_esc)) + return (gsub(self,"(.)",chr_to_esc)) end function string:unquote() - return (self:gsub("^([\"\'])(.*)%1$","%2")) + return (gsub(self,"^([\"\'])(.*)%1$","%2")) end -function string:quote() +function string:quote() -- we could use format("%q") return '"' .. self:unquote() .. '"' end function string:count(pattern) -- variant 3 local n = 0 - for _ in self:gmatch(pattern) do + for _ in gmatch(self,pattern) do n = n + 1 end return n @@ -210,29 +109,25 @@ end function string:limit(n,sentinel) if #self > n then sentinel = sentinel or " ..." - return self:sub(1,(n-#sentinel)) .. sentinel + return sub(self,1,(n-#sentinel)) .. sentinel else return self end end function string:strip() - return (self:gsub("^%s*(.-)%s*$", "%1")) + return (gsub(self,"^%s*(.-)%s*$", "%1")) end ---~ function string.strip(str) -- slightly different ---~ return (string.gsub(string.gsub(str,"^%s*(.-)%s*$","%1"),"%s+"," ")) ---~ end - function string:is_empty() - return not self:find("%S") + return not find(find,"%S") end function string:enhance(pattern,action) local ok, n = true, 0 while ok do ok = false - self = self:gsub(pattern, function(...) + self = gsub(self,pattern, function(...) ok, n = true, n + 1 return action(...) end) @@ -240,59 +135,19 @@ function string:enhance(pattern,action) return self, n end ---~ function string:enhance(pattern,action) ---~ local ok, n = 0, 0 ---~ repeat ---~ self, ok = self:gsub(pattern, function(...) ---~ n = n + 1 ---~ return action(...) ---~ end) ---~ until ok == 0 ---~ return self, n ---~ end - ---~ function string:to_hex() ---~ if self then ---~ return (self:gsub("(.)",function(c) ---~ return string.format("%02X",c:byte()) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - ---~ function string:from_hex() ---~ if self then ---~ return (self:gsub("(..)",function(c) ---~ return string.char(tonumber(c,16)) ---~ end)) ---~ else ---~ return "" ---~ end ---~ end - -string.chr_to_hex = { } -string.hex_to_chr = { } +local chr_to_hex, hex_to_chr = { }, { } for i=0,255 do - local c, h = string.char(i), string.format("%02X",i) - string.chr_to_hex[c], string.hex_to_chr[h] = h, c + local c, h = char(i), format("%02X",i) + chr_to_hex[c], hex_to_chr[h] = h, c end ---~ function string:to_hex() ---~ if self then return (self:gsub("(.)",string.chr_to_hex)) else return "" end ---~ end - ---~ function string:from_hex() ---~ if self then return (self:gsub("(..)",string.hex_to_chr)) else return "" end ---~ end - function string:to_hex() - return ((self or ""):gsub("(.)",string.chr_to_hex)) + return (gsub(self or "","(.)",chr_to_hex)) end function string:from_hex() - return ((self or ""):gsub("(..)",string.hex_to_chr)) + return (gsub(self or "","(..)",hex_to_chr)) end if not string.characters then @@ -306,7 +161,7 @@ if not string.characters then end local function nextbyte(str, index) index = index + 1 - return (index <= #str) and index or nil, string.byte(str:sub(index,index)) + return (index <= #str) and index or nil, byte(str:sub(index,index)) end function string:bytes() return nextbyte, self, 0 @@ -314,9 +169,7 @@ if not string.characters then end ---~ function string:padd(n,chr) ---~ return self .. self.rep(chr or " ",n-#self) ---~ end +-- we can use format for this (neg n) function string:rpadd(n,chr) local m = n-#self @@ -338,8 +191,8 @@ end string.padd = string.rpadd -function is_number(str) - return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 +function is_number(str) -- tonumber + return find(str,"^[%-%+]?[%d]-%.?[%d+]$") == 1 end --~ print(is_number("1")) @@ -351,9 +204,9 @@ end --~ print(is_number("+.1")) function string:split_settings() -- no {} handling, see l-aux for lpeg variant - if self:find("=") then + if find(self,"=") then local t = { } - for k,v in self:gmatch("(%a+)=([^%,]*)") do + for k,v in gmatch(self,"(%a+)=([^%,]*)") do t[k] = v end return t @@ -375,24 +228,67 @@ local patterns_escapes = { } function string:pattesc() - return (self:gsub(".",patterns_escapes)) + return (gsub(self,".",patterns_escapes)) end function string:tohash() local t = { } - for s in self:gmatch("([^, ]+)") do -- lpeg + for s in gmatch(self,"([^, ]+)") do -- lpeg t[s] = true end return t end +local pattern = lpeg.Ct(lpeg.C(1)^0) --- filename : l-lpeg.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +function string:totable() + return pattern:match(self) +end -if not versions then versions = { } end versions['l-lpeg'] = 1.001 +--~ for _, str in ipairs { +--~ "1234567123456712345671234567", +--~ "a\tb\tc", +--~ "aa\tbb\tcc", +--~ "aaa\tbbb\tccc", +--~ "aaaa\tbbbb\tcccc", +--~ "aaaaa\tbbbbb\tccccc", +--~ "aaaaaa\tbbbbbb\tcccccc", +--~ } do print(string.tabtospace(str)) end + +function string.tabtospace(str,tab) + -- we don't handle embedded newlines + while true do + local s = find(str,"\t") + if s then + if not tab then tab = 7 end -- only when found + local d = tab-(s-1)%tab + if d > 0 then + str = gsub(str,"\t",rep(" ",d),1) + else + str = gsub(str,"\t","",1) + end + else + break + end + end + return str +end + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-lpeg'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local P, S, Ct, C, Cs, Cc = lpeg.P, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc --~ l-lpeg.lua : @@ -416,36 +312,40 @@ if not versions then versions = { } end versions['l-lpeg'] = 1.001 local hash = { } function lpeg.anywhere(pattern) --slightly adapted from website - return lpeg.P { lpeg.P(pattern) + 1 * lpeg.V(1) } + return P { P(pattern) + 1 * lpeg.V(1) } end function lpeg.startswith(pattern) --slightly adapted - return lpeg.P(pattern) + return P(pattern) end ---~ g = lpeg.splitter(" ",function(s) ... end) -- gmatch:lpeg = 3:2 - function lpeg.splitter(pattern, action) - return (((1-lpeg.P(pattern))^1)/action+1)^0 + return (((1-P(pattern))^1)/action+1)^0 end -local crlf = lpeg.P("\r\n") -local cr = lpeg.P("\r") -local lf = lpeg.P("\n") -local space = lpeg.S(" \t\f\v") +-- variant: + +--~ local parser = lpeg.Ct(lpeg.splitat(newline)) + +local crlf = P("\r\n") +local cr = P("\r") +local lf = P("\n") +local space = S(" \t\f\v") -- + string.char(0xc2, 0xa0) if we want utf (cf mail roberto) local newline = crlf + cr + lf local spacing = space^0 * newline -local empty = spacing * lpeg.Cc("") -local nonempty = lpeg.Cs((1-spacing)^1) * spacing^-1 +local empty = spacing * Cc("") +local nonempty = Cs((1-spacing)^1) * spacing^-1 local content = (empty + nonempty)^1 -local capture = lpeg.Ct(content^0) +local capture = Ct(content^0) function string:splitlines() return capture:match(self) end +lpeg.linebyline = content -- better make a sublibrary + --~ local p = lpeg.splitat("->",false) print(p:match("oeps->what->more")) -- oeps what more --~ local p = lpeg.splitat("->",true) print(p:match("oeps->what->more")) -- oeps what->more --~ local p = lpeg.splitat("->",false) print(p:match("oeps")) -- oeps @@ -453,16 +353,16 @@ end local splitters_s, splitters_m = { }, { } -function lpeg.splitat(separator,single) +local function splitat(separator,single) local splitter = (single and splitters_s[separator]) or splitters_m[separator] if not splitter then - separator = lpeg.P(separator) + separator = P(separator) if single then - local other, any = lpeg.C((1 - separator)^0), lpeg.P(1) - splitter = other * (separator * lpeg.C(any^0) + "") + local other, any = C((1 - separator)^0), P(1) + splitter = other * (separator * C(any^0) + "") splitters_s[separator] = splitter else - local other = lpeg.C((1 - separator)^0) + local other = C((1 - separator)^0) splitter = other * (separator * other)^0 splitters_m[separator] = splitter end @@ -470,26 +370,43 @@ function lpeg.splitat(separator,single) return splitter end +lpeg.splitat = splitat --- filename : l-table.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local cache = { } + +function string:split(separator) + local c = cache[separator] + if not c then + c = Ct(splitat(separator)) + cache[separator] = c + end + return c:match(self) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit -if not versions then versions = { } end versions['l-table'] = 1.001 +if not modules then modules = { } end modules ['l-table'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} table.join = table.concat local concat, sort, insert, remove = table.concat, table.sort, table.insert, table.remove -local format = string.format +local format, find, gsub, lower, dump = string.format, string.find, string.gsub, string.lower, string.dump local getmetatable, setmetatable = getmetatable, setmetatable -local pairs, ipairs, type, next, tostring = pairs, ipairs, type, next, tostring +local type, next, tostring, ipairs = type, next, tostring, ipairs function table.strip(tab) local lst = { } for i=1,#tab do - local s = tab[i]:gsub("^%s*(.-)%s*$","%1") + local s = gsub(tab[i],"^%s*(.-)%s*$","%1") if s == "" then -- skip this one else @@ -501,7 +418,7 @@ end local function sortedkeys(tab) local srt, kind = { }, 0 -- 0=unknown 1=string, 2=number 3=mixed - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key if kind == 3 then -- no further check @@ -528,7 +445,7 @@ end local function sortedhashkeys(tab) -- fast one local srt = { } - for key,_ in pairs(tab) do + for key,_ in next, tab do srt[#srt+1] = key end sort(srt) @@ -538,14 +455,25 @@ end table.sortedkeys = sortedkeys table.sortedhashkeys = sortedhashkeys +function table.sortedpairs(t) + local s = sortedhashkeys(t) -- maybe just sortedkeys + local n = 0 + local function kv(s) + n = n + 1 + local k = s[n] + return k, t[k] + end + return kv, s +end + function table.append(t, list) - for _,v in pairs(list) do + for _,v in next, list do insert(t,v) end end function table.prepend(t, list) - for k,v in pairs(list) do + for k,v in next, list do insert(t,k,v) end end @@ -554,7 +482,7 @@ function table.merge(t, ...) -- first one is target t = t or {} local lst = {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do t[k] = v end end @@ -564,7 +492,7 @@ end function table.merged(...) local tmp, lst = { }, {...} for i=1,#lst do - for k, v in pairs(lst[i]) do + for k, v in next, lst[i] do tmp[k] = v end end @@ -596,13 +524,14 @@ end local function fastcopy(old) -- fast one if old then local new = { } - for k,v in pairs(old) do + for k,v in next, old do if type(v) == "table" then new[k] = fastcopy(v) -- was just table.copy else new[k] = v end end + -- optional second arg local mt = getmetatable(old) if mt then setmetatable(new,mt) @@ -619,7 +548,7 @@ local function copy(t, tables) -- taken from lua wiki, slightly adapted if not tables[t] then tables[t] = tcopy end - for i,v in pairs(t) do -- brrr, what happens with sparse indexed + for i,v in next, t do -- brrr, what happens with sparse indexed if type(i) == "table" then if tables[i] then i = tables[i] @@ -652,7 +581,7 @@ function table.sub(t,i,j) end function table.replace(a,b) - for k,v in pairs(b) do + for k,v in next, b do a[k] = v end end @@ -674,16 +603,18 @@ end function table.tohash(t,value) local h = { } - if value == nil then value = true end - for _, v in pairs(t) do -- no ipairs here - h[v] = value + if t then + if value == nil then value = true end + for _, v in next, t do -- no ipairs here + h[v] = value + end end return h end function table.fromhash(t) local h = { } - for k, v in pairs(t) do -- no ipairs here + for k, v in next, t do -- no ipairs here if v then h[#h+1] = k end end return h @@ -707,24 +638,10 @@ local reserved = table.tohash { -- intercept a language flaw, no reserved words 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true', 'until', 'while', } -local function key(k) - if type(k) == "number" then -- or k:find("^%d+$") then - if hexify then - return ("[0x%04X]"):format(k) - else - return "["..k.."]" - end - elseif noquotes and not reserved[k] and k:find("^%a[%a%d%_]*$") then - return k - else - return '["'..k..'"]' - end -end - local function simple_table(t) if #t > 0 then local n = 0 - for _,v in pairs(t) do + for _,v in next, t do n = n + 1 end if n == #t then @@ -734,14 +651,14 @@ local function simple_table(t) local tv = type(v) if tv == "number" then if hexify then - tt[#tt+1] = ("0x%04X"):format(v) + tt[#tt+1] = format("0x%04X",v) else - tt[#tt+1] = tostring(v) + tt[#tt+1] = tostring(v) -- tostring not needed end elseif tv == "boolean" then tt[#tt+1] = tostring(v) elseif tv == "string" then - tt[#tt+1] = ("%q"):format(v) + tt[#tt+1] = format("%q",v) else tt = nil break @@ -753,51 +670,72 @@ local function simple_table(t) return nil end +-- Because this is a core function of mkiv I moved some function calls +-- inline. +-- +-- twice as fast in a test: +-- +-- local propername = lpeg.P(lpeg.R("AZ","az","__") * lpeg.R("09","AZ","az", "__")^0 * lpeg.P(-1) ) + local function do_serialize(root,name,depth,level,indexed) if level > 0 then depth = depth .. " " if indexed then - handle(("%s{"):format(depth)) + handle(format("%s{",depth)) elseif name then - handle(("%s%s={"):format(depth,key(name))) + --~ handle(format("%s%s={",depth,key(name))) + if type(name) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s[0x%04X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif noquotes and not reserved[name] and find(name,"^%a[%w%_]*$") then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end else - handle(("%s{"):format(depth)) + handle(format("%s{",depth)) end end if root and next(root) then local first, last = nil, 0 -- #root cannot be trusted here if compact then - for k,v in ipairs(root) do -- NOT: for k=1,#root do (we need to quit at nil) + -- NOT: for k=1,#root do (we need to quit at nil) + for k,v in ipairs(root) do -- can we use next? if not first then first = k end last = last + 1 end end - --~ for _,k in pairs(sortedkeys(root)) do -- 1% faster: local sk = sortedkeys(root) for i=1,#sk do local k = sk[i] local v = root[k] + --~ if v == root then + -- circular + --~ else local t = type(v) if compact and first and type(k) == "number" and k >= first and k <= last then if t == "number" then if hexify then - handle(("%s 0x%04X,"):format(depth,v)) + handle(format("%s 0x%04X,",depth,v)) else - handle(("%s %s,"):format(depth,v)) + handle(format("%s %s,",depth,v)) end elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s,"):format(depth,v)) + if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + handle(format("%s %s,",depth,v)) else - handle(("%s %q,"):format(depth,v)) + handle(format("%s %q,",depth,v)) end elseif t == "table" then if not next(v) then - handle(("%s {},"):format(depth)) - elseif inline then + handle(format("%s {},",depth)) + elseif inline then -- and #t > 0 local st = simple_table(v) if st then - handle(("%s { %s },"):format(depth,concat(st,", "))) + handle(format("%s { %s },",depth,concat(st,", "))) else do_serialize(v,k,depth,level+1,true) end @@ -805,39 +743,102 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1,true) end elseif t == "boolean" then - handle(("%s %s,"):format(depth,tostring(v))) + handle(format("%s %s,",depth,tostring(v))) elseif t == "function" then if functions then - handle(('%s loadstring(%q),'):format(depth,v:dump())) + handle(format('%s loadstring(%q),',depth,dump(v))) else - handle(('%s "function",'):format(depth)) + handle(format('%s "function",',depth)) end else - handle(("%s %q,"):format(depth,tostring(v))) + handle(format("%s %q,",depth,tostring(v))) end elseif k == "__p__" then -- parent if false then - handle(("%s __p__=nil,"):format(depth)) + handle(format("%s __p__=nil,",depth)) end elseif t == "number" then - if hexify then - handle(("%s %s=0x%04X,"):format(depth,key(k),v)) + --~ if hexify then + --~ handle(format("%s %s=0x%04X,",depth,key(k),v)) + --~ else + --~ handle(format("%s %s=%s,",depth,key(k),v)) + --~ end + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + if hexify then + handle(format("%s %s=0x%04X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end else - handle(("%s %s=%s,"):format(depth,key(k),v)) + if hexify then + handle(format("%s [%q]=0x%04X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end end elseif t == "string" then - if reduce and (v:find("^[%-%+]?[%d]-%.?[%d+]$") == 1) then - handle(("%s %s=%s,"):format(depth,key(k),v)) + if reduce and (find(v,"^[%-%+]?[%d]-%.?[%d+]$") == 1) then + --~ handle(format("%s %s=%s,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end else - handle(("%s %s=%q,"):format(depth,key(k),v)) + --~ handle(format("%s %s=%q,",depth,key(k),v)) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,v)) + else + handle(format("%s [%s]=%q,",depth,k,v)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end end elseif t == "table" then if not next(v) then - handle(("%s %s={},"):format(depth,key(k))) + --~ handle(format("%s %s={},",depth,key(k))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end elseif inline then local st = simple_table(v) if st then - handle(("%s %s={ %s },"):format(depth,key(k),concat(st,", "))) + --~ handle(format("%s %s={ %s },",depth,key(k),concat(st,", "))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) + end else do_serialize(v,k,depth,level+1) end @@ -845,24 +846,58 @@ local function do_serialize(root,name,depth,level,indexed) do_serialize(v,k,depth,level+1) end elseif t == "boolean" then - handle(("%s %s=%s,"):format(depth,key(k),tostring(v))) + --~ handle(format("%s %s=%s,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%s,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%s,",depth,k,tostring(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%s,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%s,",depth,k,tostring(v))) + end elseif t == "function" then if functions then - handle(('%s %s=loadstring(%q),'):format(depth,key(k),v:dump())) - else - handle(('%s %s="function",'):format(depth,key(k))) + --~ handle(format('%s %s=loadstring(%q),',depth,key(k),dump(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=loadstring(%q),",depth,k,dump(v))) + else + handle(format("%s [%s]=loadstring(%q),",depth,k,dump(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=loadstring(%q),",depth,k,dump(v))) + else + handle(format("%s [%q]=loadstring(%q),",depth,k,dump(v))) + end end else - handle(("%s %s=%q,"):format(depth,key(k),tostring(v))) - -- handle(('%s %s=loadstring(%q),'):format(depth,key(k),string.dump(function() return v end))) + --~ handle(format("%s %s=%q,",depth,key(k),tostring(v))) + if type(k) == "number" then -- or find(k,"^%d+$") then + if hexify then + handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif noquotes and not reserved[k] and find(k,"^%a[%w%_]*$") then + handle(format("%s %s=%q,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%q,",depth,k,tostring(v))) + end end + --~ end end end if level > 0 then - handle(("%s},"):format(depth)) + handle(format("%s},",depth)) end end +-- replacing handle by a direct t[#t+1] = ... (plus test) is not much +-- faster (0.03 on 1.00 for zapfino.tma) + local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) noquotes = _noquotes hexify = _hexify @@ -880,7 +915,7 @@ local function serialize(root,name,_handle,_reduce,_noquotes,_hexify) end elseif tname == "number" then if hexify then - handle(("[0x%04X]={"):format(name)) + handle(format("[0x%04X]={",name)) else handle("[" .. name .. "]={") end @@ -1031,14 +1066,18 @@ function table.insert_after_value(t,value,str) end end -function table.are_equal(a,b,n,m) +local function are_equal(a,b,n,m) -- indexed if #a == #b then n = n or 1 m = m or #a for i=n,m do local ai, bi = a[i], b[i] - if (ai==bi) or (type(ai)=="table" and type(bi)=="table" and table.are_equal(ai,bi)) then - -- continue + if ai==bi then + -- same + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end else return false end @@ -1049,9 +1088,30 @@ function table.are_equal(a,b,n,m) end end +local function identical(a,b) -- assumes same structure + for ka, va in next, a do + local vb = b[k] + if va == vb then + -- same + elseif type(va) == "table" and type(vb) == "table" then + if not identical(va,vb) then + return false + end + else + return false + end + end + return true +end + +table.are_equal = are_equal +table.identical = identical + +-- maybe also make a combined one + function table.compact(t) if t then - for k,v in pairs(t) do + for k,v in next, t do if not next(v) then t[k] = nil end @@ -1080,7 +1140,7 @@ end function table.swapped(t) local s = { } - for k, v in pairs(t) do + for k, v in next, t do s[v] = k end return s @@ -1102,14 +1162,14 @@ end function table.hexed(t,seperator) local tt = { } - for i=1,#t do tt[i] = ("0x%04X"):format(t[i]) end + for i=1,#t do tt[i] = format("0x%04X",t[i]) end return concat(tt,seperator or " ") end function table.reverse_hash(h) local r = { } - for k,v in pairs(h) do - r[v] = (k:gsub(" ","")):lower() + for k,v in next, h do + r[v] = lower(gsub(k," ","")) end return r end @@ -1124,14 +1184,36 @@ function table.reverse(t) return tt end +--~ function table.keys(t) +--~ local k = { } +--~ for k,_ in next, t do +--~ k[#k+1] = k +--~ end +--~ return k +--~ end --- filename : l-io.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +--~ function table.keys_as_string(t) +--~ local k = { } +--~ for k,_ in next, t do +--~ k[#k+1] = k +--~ end +--~ return concat(k,"") +--~ end -if not versions then versions = { } end versions['l-io'] = 1.001 + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-io'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local byte = string.byte if string.find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator = "\\", ";" @@ -1139,8 +1221,8 @@ else io.fileseparator, io.pathseparator = "/" , ":" end -function io.loaddata(filename) - local f = io.open(filename,'rb') +function io.loaddata(filename,textmode) + local f = io.open(filename,(textmode and 'r') or 'rb') if f then local data = f:read('*all') -- garbagecollector.check(data) @@ -1198,146 +1280,83 @@ function io.noflines(f) return n end -do - - local sb = string.byte - - local nextchar = { - [ 4] = function(f) - return f:read(1,1,1,1) - end, - [ 2] = function(f) - return f:read(1,1) - end, - [ 1] = function(f) - return f:read(1) - end, - [-2] = function(f) - local a, b = f:read(1,1) - return b, a - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - return d, c, b, a - end - } - - function io.characters(f,n) - if f then - return nextchar[n or 1], f - else - return nil, nil - end +local nextchar = { + [ 4] = function(f) + return f:read(1,1,1,1) + end, + [ 2] = function(f) + return f:read(1,1) + end, + [ 1] = function(f) + return f:read(1) + end, + [-2] = function(f) + local a, b = f:read(1,1) + return b, a + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + return d, c, b, a end +} +function io.characters(f,n) + if f then + return nextchar[n or 1], f + else + return nil, nil + end end -do - - local sb = string.byte - ---~ local nextbyte = { ---~ [4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(a), sb(b), sb(c), sb(d) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end, ---~ [2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(a), sb(b) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [1] = function (f) ---~ local a = f:read(1) ---~ if a then ---~ return sb(a) ---~ else ---~ return nil ---~ end ---~ end, ---~ [-2] = function (f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ if b then ---~ return sb(b), sb(a) ---~ else ---~ return nil, nil ---~ end ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ if d then ---~ return sb(d), sb(c), sb(b), sb(a) ---~ else ---~ return nil, nil, nil, nil ---~ end ---~ end ---~ } - - local nextbyte = { - [4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(a), sb(b), sb(c), sb(d) - else - return nil, nil, nil, nil - end - end, - [2] = function(f) - local a, b = f:read(1,1) - if b then - return sb(a), sb(b) - else - return nil, nil - end - end, - [1] = function (f) - local a = f:read(1) - if a then - return sb(a) - else - return nil - end - end, - [-2] = function (f) - local a, b = f:read(1,1) - if b then - return sb(b), sb(a) - else - return nil, nil - end - end, - [-4] = function(f) - local a, b, c, d = f:read(1,1,1,1) - if d then - return sb(d), sb(c), sb(b), sb(a) - else - return nil, nil, nil, nil - end +local nextbyte = { + [4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(a), byte(b), byte(c), byte(d) + else + return nil, nil, nil, nil end - } - - function io.bytes(f,n) - if f then - return nextbyte[n or 1], f + end, + [2] = function(f) + local a, b = f:read(1,1) + if b then + return byte(a), byte(b) + else + return nil, nil + end + end, + [1] = function (f) + local a = f:read(1) + if a then + return byte(a) + else + return nil + end + end, + [-2] = function (f) + local a, b = f:read(1,1) + if b then + return byte(b), byte(a) else return nil, nil end + end, + [-4] = function(f) + local a, b, c, d = f:read(1,1,1,1) + if d then + return byte(d), byte(c), byte(b), byte(a) + else + return nil, nil, nil, nil + end end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1], f + else + return nil, nil + end end function io.ask(question,default,options) @@ -1373,35 +1392,21 @@ function io.ask(question,default,options) end --- filename : l-md5.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files - -if not versions then versions = { } end versions['l-md5'] = 1.001 - -if md5 then do - - local function convert(str,fmt) - return (string.gsub(md5.sum(str),".",function(chr) return string.format(fmt,string.byte(chr)) end)) - end - - if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end - if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end - if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end - -end end +end -- of closure +do -- create closure to overcome 200 locals limit --- filename : l-number.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +if not modules then modules = { } end modules ['l-number'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-number'] = 1.001 +local format = string.format -if not number then number = { } end +number = number or { } -- a,b,c,d,e,f = number.toset(100101) @@ -1409,8 +1414,6 @@ function number.toset(n) return (tostring(n)):match("(.?)(.?)(.?)(.?)(.?)(.?)(.?)(.?)") end -local format = string.format - function number.toevenhex(n) local s = format("%X",n) if #s % 2 == 0 then @@ -1431,72 +1434,72 @@ end -- -- of course dedicated "(.)(.)(.)(.)" matches are even faster -do - local one = lpeg.C(1-lpeg.S(''))^1 +local one = lpeg.C(1-lpeg.S(''))^1 - function number.toset(n) - return one:match(tostring(n)) - end +function number.toset(n) + return one:match(tostring(n)) end --- filename : l-set.lua --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-set'] = 1.001 +end -- of closure + +do -- create closure to overcome 200 locals limit -if not set then set = { } end +if not modules then modules = { } end modules ['l-set'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -do +set = set or { } - local nums = { } - local tabs = { } - local concat = table.concat +local nums = { } +local tabs = { } +local concat = table.concat - set.create = table.tohash +set.create = table.tohash - function set.tonumber(t) - if next(t) then - local s = "" - -- we could save mem by sorting, but it slows down - for k, v in pairs(t) do - if v then - -- why bother about the leading space - s = s .. " " .. k - end - end - if not nums[s] then - tabs[#tabs+1] = t - nums[s] = #tabs +function set.tonumber(t) + if next(t) then + local s = "" + -- we could save mem by sorting, but it slows down + for k, v in pairs(t) do + if v then + -- why bother about the leading space + s = s .. " " .. k end - return nums[s] - else - return 0 end - end - - function set.totable(n) - if n == 0 then - return { } - else - return tabs[n] or { } + if not nums[s] then + tabs[#tabs+1] = t + nums[s] = #tabs end + return nums[s] + else + return 0 end +end - function set.contains(n,s) - if type(n) == "table" then - return n[s] - elseif n == 0 then - return false - else - local t = tabs[n] - return t and t[s] - end +function set.totable(n) + if n == 0 then + return { } + else + return tabs[n] or { } end +end +function set.contains(n,s) + if type(n) == "table" then + return n[s] + elseif n == 0 then + return false + else + local t = tabs[n] + return t and t[s] + end end --~ local c = set.create{'aap','noot','mies'} @@ -1513,16 +1516,19 @@ end --- filename : l-os.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure +do -- create closure to overcome 200 locals limit ---~ print(table.serialize(os.uname())) +if not modules then modules = { } end modules ['l-os'] = { + version = 1.001, + comment = "companion to luat-lub.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-os'] = 1.001 +local find = string.find function os.resultof(command) return io.popen(command,"r"):read("*all") @@ -1535,7 +1541,7 @@ if not os.spawn then os.spawn = os.execute end --~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) if not io.fileseparator then - if string.find(os.getenv("PATH"),";") then + if find(os.getenv("PATH"),";") then io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" else io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" @@ -1573,11 +1579,10 @@ end os.gettimeofday = os.gettimeofday or os.clock -do - local startuptime = os.gettimeofday() - function os.runtime() - return os.gettimeofday() - startuptime - end +local startuptime = os.gettimeofday() + +function os.runtime() + return os.gettimeofday() - startuptime end --~ print(os.gettimeofday()-os.time()) @@ -1586,27 +1591,92 @@ end --~ print(os.date("%H:%M:%S",os.gettimeofday())) --~ print(os.date("%H:%M:%S",os.time())) +os.arch = os.arch or function() + local a = os.resultof("uname -m") or "linux" + os.arch = function() + return a + end + return a +end + +local platform + +function os.currentplatform(name,default) + if not platform then + local name = os.name or os.platform or name -- os.name is built in, os.platform is mine + if not name then + platform = default or "linux" + elseif name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then + if os.getenv("PROCESSOR_ARCHITECTURE") == "AMD64" then + platform = "mswin-64" + else + platform = "mswin" + end + else + local architecture = os.arch() + if name == "linux" then + if find(architecture,"x86_64") then + platform = "linux-64" + elseif find(architecture,"ppc") then + platform = "linux-ppc" + else + platform = "linux" + end + elseif name == "macosx" then + if find(architecture,"i386") then + platform = "osx-intel" + else + platform = "osx-ppc" + end + elseif name == "sunos" then + if find(architecture,"sparc") then + platform = "solaris-sparc" + else -- if architecture == 'i86pc' + platform = "solaris-intel" + end + elseif name == "freebsd" then + if find(architecture,"amd64") then + platform = "freebsd-amd64" + else + platform = "freebsd" + end + else + platform = default or name + end + end + function os.currentplatform() + return platform + end + end + return platform +end --- filename : l-file.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files -if not versions then versions = { } end versions['l-file'] = 1.001 +end -- of closure -if not file then file = { } end +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-file'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- needs a cleanup + +file = file or { } local concat = table.concat +local find, gmatch, match, gsub = string.find, string.gmatch, string.match, string.gsub function file.removesuffix(filename) - return (filename:gsub("%.[%a%d]+$","")) + return (gsub(filename,"%.[%a%d]+$","")) end -file.stripsuffix = file.removesuffix - function file.addsuffix(filename, suffix) - if not filename:find("%.[%a%d]+$") then + if not find(filename,"%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1614,23 +1684,23 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - return (filename:gsub("%.[%a%d]+$","")) .. "." .. suffix + return (gsub(filename,"%.[%a%d]+$","")) .. "." .. suffix end -function file.dirname(name) - return name:match("^(.+)[/\\].-$") or "" +function file.dirname(name,default) + return match(name,"^(.+)[/\\].-$") or (default or "") end function file.basename(name) - return name:match("^.+[/\\](.-)$") or name + return match(name,"^.+[/\\](.-)$") or name end function file.nameonly(name) - return ((name:match("^.+[/\\](.-)$") or name):gsub("%..*$","")) + return (gsub(match(name,"^.+[/\\](.-)$") or name,"%..*$","")) end function file.extname(name) - return name:match("^.+%.([^/\\]-)$") or "" + return match(name,"^.+%.([^/\\]-)$") or "" end file.suffix = file.extname @@ -1643,40 +1713,20 @@ file.suffix = file.extname function file.join(...) local pth = concat({...},"/") - pth = pth:gsub("\\","/") - local a, b = pth:match("^(.*://)(.*)$") + pth = gsub(pth,"\\","/") + local a, b = match(pth,"^(.*://)(.*)$") if a and b then - return a .. b:gsub("//+","/") + return a .. gsub(b,"//+","/") end - a, b = pth:match("^(//)(.*)$") + a, b = match(pth,"^(//)(.*)$") if a and b then - return a .. b:gsub("//+","/") - end - return (pth:gsub("//+","/")) -end - -function file.is_writable(name) - local f = io.open(name, 'w') - if f then - f:close() - return true - else - return false - end -end - -function file.is_readable(name) - local f = io.open(name,'r') - if f then - f:close() - return true - else - return false + return a .. gsub(b,"//+","/") end + return (gsub(pth,"//+","/")) end function file.iswritable(name) - local a = lfs.attributes(name) + local a = lfs.attributes(name) or lfs.attributes(file.dirname(name,".")) return a and a.permissions:sub(2,2) == "w" end @@ -1685,24 +1735,18 @@ function file.isreadable(name) return a and a.permissions:sub(1,1) == "r" end ---~ function file.split_path(str) ---~ if str:find(';') then ---~ return str:splitchr(";") ---~ else ---~ return str:splitchr(io.pathseparator) ---~ end ---~ end +file.is_readable = file.isreadable +file.is_writable = file.iswritable -- todo: lpeg function file.split_path(str) local t = { } - str = str:gsub("\\", "/") - str = str:gsub("(%a):([;/])", "%1\001%2") - for name in str:gmatch("([^;:]+)") do + str = gsub(str,"\\", "/") + str = gsub(str,"(%a):([;/])", "%1\001%2") + for name in gmatch(str,"([^;:]+)") do if name ~= "" then - name = name:gsub("\001",":") - t[#t+1] = name + t[#t+1] = gsub(name,"\001",":") end end return t @@ -1713,15 +1757,15 @@ function file.join_path(tab) end function file.collapse_path(str) - str = str:gsub("/%./","/") + str = gsub(str,"/%./","/") local n, m = 1, 1 while n > 0 or m > 0 do - str, n = str:gsub("[^/%.]+/%.%.$","") - str, m = str:gsub("[^/%.]+/%.%./","") + str, n = gsub(str,"[^/%.]+/%.%.$","") + str, m = gsub(str,"[^/%.]+/%.%./","") end - str = str:gsub("([^/])/$","%1") - str = str:gsub("^%./","") - str = str:gsub("/%.$","") + str = gsub(str,"([^/])/$","%1") + str = gsub(str,"^%./","") + str = gsub(str,"/%.$","") if str == "" then str = "." end return str end @@ -1734,7 +1778,7 @@ end --~ print(file.collapse_path("a/b/c/../..")) function file.robustname(str) - return (str:gsub("[^%a%d%/%-%.\\]+","-")) + return (gsub(str,"[^%a%d%/%-%.\\]+","-")) end file.readdata = io.loaddata @@ -1764,8 +1808,6 @@ end --~ return pattern:match(name) --~ end ---~ file.stripsuffix = file.removesuffix - --~ local pattern = (noslashes^0 * slashes)^1 * lpeg.C(noslashes^1) * -1 --~ function file.basename(name) @@ -1819,7 +1861,6 @@ end --~ end --~ local test = file.extname ---~ local test = file.stripsuffix --~ local test = file.basename --~ local test = file.dirname --~ local test = file.addsuffix @@ -1836,206 +1877,307 @@ end --~ local tim = os.clock() for i=1,250000 do local ext = test("abd.def.xxx","!!!") end print(os.clock()-tim) +-- also rewrite previous --- filename : l-dir.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +local letter = lpeg.R("az","AZ") + lpeg.S("_-+") +local separator = lpeg.P("://") -if not versions then versions = { } end versions['l-dir'] = 1.001 +local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + letter^1 * lpeg.P("/") +local rootbased = lpeg.P("/") + letter*lpeg.P(":") -dir = { } +-- ./name ../name /name c: :// name/name --- optimizing for no string.find (*) does not save time +function file.is_qualified_path(filename) + return qualified:match(filename) +end + +function file.is_rootbased_path(filename) + return rootbased:match(filename) +end -if lfs then do - local attributes = lfs.attributes - local walkdir = lfs.dir +end -- of closure - local function glob_pattern(path,patt,recurse,action) - local ok, scanner - if path == "/" then - ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe - else - ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-md5'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This also provides file checksums and checkers. + +local gsub, format, byte = string.gsub, string.format, string.byte + +local function convert(str,fmt) + return (gsub(md5.sum(str),".",function(chr) return format(fmt,byte(chr)) end)) +end + +if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end +if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end +if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end + +--~ if not md5.HEX then +--~ local function remap(chr) return format("%02X",byte(chr)) end +--~ function md5.HEX(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.hex then +--~ local function remap(chr) return format("%02x",byte(chr)) end +--~ function md5.hex(str) return (gsub(md5.sum(str),".",remap)) end +--~ end +--~ if not md5.dec then +--~ local function remap(chr) return format("%03i",byte(chr)) end +--~ function md5.dec(str) return (gsub(md5.sum(str),".",remap)) end +--~ end + +file.needs_updating_threshold = 1 + +function file.needs_updating(oldname,newname) -- size modification access change + local oldtime = lfs.attributes(oldname, modification) + local newtime = lfs.attributes(newname, modification) + if newtime >= oldtime then + return false + elseif oldtime - newtime < file.needs_updating_threshold then + return false + else + return true + end +end + +function file.checksum(name) + if md5 then + local data = io.loaddata(name) + if data then + return md5.HEX(data) end - if ok and type(scanner) == "function" then - if not path:find("/$") then path = path .. '/' end - for name in scanner do - local full = path .. name - local mode = attributes(full,'mode') - if mode == 'file' then - if full:find(patt) then - action(full) - end - elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then - glob_pattern(full,patt,recurse,action) + end + return nil +end + +function file.loadchecksum(name) + if md5 then + local data = io.loaddata(name .. ".md5") + return data and data:gsub("%s","") + end + return nil +end + +function file.savechecksum(name, checksum) + if not checksum then checksum = file.checksum(name) end + if checksum then + io.savedata(name .. ".md5",checksum) + return checksum + end + return nil +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-dir'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local type = type +local find, gmatch = string.find, string.gmatch + +dir = dir or { } + +-- optimizing for no string.find (*) does not save time + +local attributes = lfs.attributes +local walkdir = lfs.dir + +local function glob_pattern(path,patt,recurse,action) + local ok, scanner + if path == "/" then + ok, scanner = xpcall(function() return walkdir(path..".") end, function() end) -- kepler safe + else + ok, scanner = xpcall(function() return walkdir(path) end, function() end) -- kepler safe + end + if ok and type(scanner) == "function" then + if not find(path,"/$") then path = path .. '/' end + for name in scanner do + local full = path .. name + local mode = attributes(full,'mode') + if mode == 'file' then + if find(full,patt) then + action(full) end + elseif recurse and (mode == "directory") and (name ~= '.') and (name ~= "..") then + glob_pattern(full,patt,recurse,action) end end end +end - dir.glob_pattern = glob_pattern +dir.glob_pattern = glob_pattern - local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V +local P, S, R, C, Cc, Cs, Ct, Cv, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Cv, lpeg.V - local pattern = Ct { - [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), - [2] = C(((1-S("*?/"))^0 * P("/"))^0), - [3] = C(P(1)^0) - } +local pattern = Ct { + [1] = (C(P(".") + P("/")^1) + C(R("az","AZ") * P(":") * P("/")^0) + Cc("./")) * V(2) * V(3), + [2] = C(((1-S("*?/"))^0 * P("/"))^0), + [3] = C(P(1)^0) +} - local filter = Cs ( ( - P("**") / ".*" + - P("*") / "[^/]*" + - P("?") / "[^/]" + - P(".") / "%%." + - P("+") / "%%+" + - P("-") / "%%-" + - P(1) - )^0 ) - - local function glob(str,t) - if type(str) == "table" then - local t = t or { } - for _, s in ipairs(str) do - glob(s,t) - end - return t - elseif lfs.isfile(str) then +local filter = Cs ( ( + P("**") / ".*" + + P("*") / "[^/]*" + + P("?") / "[^/]" + + P(".") / "%%." + + P("+") / "%%+" + + P("-") / "%%-" + + P(1) +)^0 ) + +local function glob(str,t) + if type(str) == "table" then + local t = t or { } + for s=1,#str do + glob(str[s],t) + end + return t + elseif lfs.isfile(str) then + local t = t or { } + t[#t+1] = str + return t + else + local split = pattern:match(str) + if split then local t = t or { } - t[#t+1] = str + local action = action or function(name) t[#t+1] = name end + local root, path, base = split[1], split[2], split[3] + local recurse = find(base,"%*%*") + local start = root .. path + local result = filter:match(start .. base) + glob_pattern(start,result,recurse,action) return t else - local split = pattern:match(str) - if split then - local t = t or { } - local action = action or function(name) t[#t+1] = name end - local root, path, base = split[1], split[2], split[3] - local recurse = base:find("%*%*") - local start = root .. path - local result = filter:match(start .. base) - glob_pattern(start,result,recurse,action) - return t - else - return { } - end + return { } end end +end - dir.glob = glob +dir.glob = glob - --~ list = dir.glob("**/*.tif") - --~ list = dir.glob("/**/*.tif") - --~ list = dir.glob("./**/*.tif") - --~ list = dir.glob("oeps/**/*.tif") - --~ list = dir.glob("/oeps/**/*.tif") +--~ list = dir.glob("**/*.tif") +--~ list = dir.glob("/**/*.tif") +--~ list = dir.glob("./**/*.tif") +--~ list = dir.glob("oeps/**/*.tif") +--~ list = dir.glob("/oeps/**/*.tif") - local function globfiles(path,recurse,func,files) -- func == pattern or function - if type(func) == "string" then - local s = func -- alas, we need this indirect way - func = function(name) return name:find(s) end - end - files = files or { } - for name in walkdir(path) do - if name:find("^%.") then - --- skip - else - local mode = attributes(name,'mode') - if mode == "directory" then - if recurse then - globfiles(path .. "/" .. name,recurse,func,files) - end - elseif mode == "file" then - if func then - if func(name) then - files[#files+1] = path .. "/" .. name - end - else +local function globfiles(path,recurse,func,files) -- func == pattern or function + if type(func) == "string" then + local s = func -- alas, we need this indirect way + func = function(name) return find(name,s) end + end + files = files or { } + for name in walkdir(path) do + if find(name,"^%.") then + --- skip + else + local mode = attributes(name,'mode') + if mode == "directory" then + if recurse then + globfiles(path .. "/" .. name,recurse,func,files) + end + elseif mode == "file" then + if func then + if func(name) then files[#files+1] = path .. "/" .. name end + else + files[#files+1] = path .. "/" .. name end end end - return files end + return files +end - dir.globfiles = globfiles +dir.globfiles = globfiles - -- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") - -- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") - -- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") - -- t = dir.glob("f:/minimal/tex/**/*") - -- print(dir.ls("f:/minimal/tex/**/*")) - -- print(dir.ls("*.tex")) +-- t = dir.glob("c:/data/develop/context/sources/**/????-*.tex") +-- t = dir.glob("c:/data/develop/tex/texmf/**/*.tex") +-- t = dir.glob("c:/data/develop/context/texmf/**/*.tex") +-- t = dir.glob("f:/minimal/tex/**/*") +-- print(dir.ls("f:/minimal/tex/**/*")) +-- print(dir.ls("*.tex")) - function dir.ls(pattern) - return table.concat(glob(pattern),"\n") - end +function dir.ls(pattern) + return table.concat(glob(pattern),"\n") +end - --~ mkdirs("temp") - --~ mkdirs("a/b/c") - --~ mkdirs(".","/a/b/c") - --~ mkdirs("a","b","c") +--~ mkdirs("temp") +--~ mkdirs("a/b/c") +--~ mkdirs(".","/a/b/c") +--~ mkdirs("a","b","c") - local make_indeed = true -- false +local make_indeed = true -- false - if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";") then - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - local first, middle, last - local drive = false - first, middle, last = str:match("^(//)(//*)(.*)$") + end + local first, middle, last + local drive = false + first, middle, last = str:match("^(//)(//*)(.*)$") + if first then + -- empty network path == local path + else + first, last = str:match("^(//)/*(.-)$") if first then - -- empty network path == local path + middle, last = str:match("([^/]+)/+(.-)$") + if middle then + pth = "//" .. middle + else + pth = "//" .. last + last = "" + end else - first, last = str:match("^(//)/*(.-)$") + first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") if first then - middle, last = str:match("([^/]+)/+(.-)$") - if middle then - pth = "//" .. middle - else - pth = "//" .. last - last = "" - end + pth, drive = first .. middle, true else - first, middle, last = str:match("^([a-zA-Z]:)(/*)(.-)$") - if first then - pth, drive = first .. middle, true - else - middle, last = str:match("^(/*)(.-)$") - if not middle then - last = str - end + middle, last = str:match("^(/*)(.-)$") + if not middle then + last = str end end end - for s in last:gmatch("[^/]+") do - if pth == "" then - pth = s - elseif drive then - pth, drive = pth .. s, false - else - pth = pth .. "/" .. s - end - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + for s in gmatch(last,"[^/]+") do + if pth == "" then + pth = s + elseif drive then + pth, drive = pth .. s, false + else + pth = pth .. "/" .. s + end + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2049,79 +2191,79 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - local first, nothing, last = str:match("^(//)(//*)(.*)$") - if first then - first = lfs.currentdir() .. "/" - first = first:gsub("\\","/") - end - if not first then - first, last = str:match("^(//)/*(.*)$") - end - if not first then - first, last = str:match("^([a-zA-Z]:)(.*)$") - if first and not last:find("^/") then - local d = lfs.currentdir() - if lfs.chdir(first) then - first = lfs.currentdir() - first = first:gsub("\\","/") - end - lfs.chdir(d) + function dir.expand_name(str) + local first, nothing, last = str:match("^(//)(//*)(.*)$") + if first then + first = lfs.currentdir() .. "/" + first = first:gsub("\\","/") + end + if not first then + first, last = str:match("^(//)/*(.*)$") + end + if not first then + first, last = str:match("^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = lfs.currentdir() + if lfs.chdir(first) then + first = lfs.currentdir() + first = first:gsub("\\","/") end + lfs.chdir(d) end - if not first then - first, last = lfs.currentdir(), str - first = first:gsub("\\","/") - end - last = last:gsub("//","/") - last = last:gsub("/%./","/") - last = last:gsub("^/*","") - first = first:gsub("/*$","") - if last == "" then - return first - else - return first .. "/" .. last - end end + if not first then + first, last = lfs.currentdir(), str + first = first:gsub("\\","/") + end + last = last:gsub("//","/") + last = last:gsub("/%./","/") + last = last:gsub("^/*","") + first = first:gsub("/*$","") + if last == "" then + return first + else + return first .. "/" .. last + end + end - else +else - function dir.mkdirs(...) - local str, pth = "", "" - for _, s in ipairs({...}) do - if s ~= "" then - if str ~= "" then - str = str .. "/" .. s - else - str = s - end + function dir.mkdirs(...) + local str, pth = "", "" + for _, s in ipairs({...}) do + if s ~= "" then + if str ~= "" then + str = str .. "/" .. s + else + str = s end end - str = str:gsub("/+","/") - if str:find("^/") then - pth = "/" - for s in str:gmatch("[^/]+") do - local first = (pth == "/") - if first then - pth = pth .. s - else - pth = pth .. "/" .. s - end - if make_indeed and not first and not lfs.isdir(pth) then - lfs.mkdir(pth) - end - end - else - pth = "." - for s in str:gmatch("[^/]+") do + end + str = str:gsub("/+","/") + if find(str,"^/") then + pth = "/" + for s in gmatch(str,"[^/]+") do + local first = (pth == "/") + if first then + pth = pth .. s + else pth = pth .. "/" .. s - if make_indeed and not lfs.isdir(pth) then - lfs.mkdir(pth) - end + end + if make_indeed and not first and not lfs.isdir(pth) then + lfs.mkdir(pth) + end + end + else + pth = "." + for s in gmatch(str,"[^/]+") do + pth = pth .. "/" .. s + if make_indeed and not lfs.isdir(pth) then + lfs.mkdir(pth) end end - return pth, (lfs.isdir(pth) == true) end + return pth, (lfs.isdir(pth) == true) + end --~ print(dir.mkdirs("","","a","c")) --~ print(dir.mkdirs("a")) @@ -2131,30 +2273,35 @@ if lfs then do --~ print(dir.mkdirs("///a/b/c")) --~ print(dir.mkdirs("a/bbb//ccc/")) - function dir.expand_name(str) - if not str:find("^/") then - str = lfs.currentdir() .. "/" .. str - end - str = str:gsub("//","/") - str = str:gsub("/%./","/") - return str + function dir.expand_name(str) + if not find(str,"^/") then + str = lfs.currentdir() .. "/" .. str end - + str = str:gsub("//","/") + str = str:gsub("/%./","/") + return str end - dir.makedirs = dir.mkdirs +end -end end +dir.makedirs = dir.mkdirs --- filename : l-boolean.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-boolean'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -if not versions then versions = { } end versions['l-boolean'] = 1.001 -if not boolean then boolean = { } end +boolean = boolean or { } + +local type, tonumber = type, tonumber function boolean.tonumber(b) if b then return 1 else return 0 end @@ -2201,15 +2348,19 @@ function boolean.falsetrue() end --- filename : l-math.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-math'] = 1.001 +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-math'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -local floor = math.floor +local floor, sin, cos, tan = math.floor, math.sin, math.cos, math.tan if not math.round then function math.round(x) @@ -2229,23 +2380,213 @@ if not math.mod then end end +local pipi = 2*math.pi/360 -if not modules then modules = { } end modules ['l-xml'] = { +function math.sind(d) + return sin(d*pipi) +end + +function math.cosd(d) + return cos(d*pipi) +end + +function math.tand(d) + return tan(d*pipi) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['l-utils'] = { version = 1.001, - comment = "this module is the basis for the lxml-* ones", + comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } --- RJ: key=value ... lpeg.Ca(lpeg.Cc({}) * (pattern-producing-key-and-value / rawset)^0) +-- hm, quite unreadable + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +utils.merger.strip_comment = true --- some code may move to l-xmlext +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + utils.report("reading merge from %s",name) + data = f:read("*all") + f:close() + else + utils.report("unknown file to merge %s",name) + end + if data and utils.merger.strip_comment then + -- saves some 20K + data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + utils.report("saving merge from %s",name) + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (data:gsub(utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +--~ stripper: +--~ +--~ data = string.gsub(data,"%-%-~[^\n]*\n","") +--~ data = string.gsub(data,"\n\n+","\n") + +function utils.merger._self_libs_(libs,list) + local result, f, frozen = { }, nil, false + result[#result+1] = "\n" + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + local foundpath = nil + for _, lib in ipairs(libs) do + for _, pth in ipairs(list) do + pth = string.gsub(pth,"\\","/") -- file.clean_path + utils.report("checking library path %s",pth) + local name = pth .. "/" .. lib + if lfs.isfile(name) then + foundpath = pth + end + end + if foundpath then break end + end + if foundpath then + utils.report("using library path %s",foundpath) + local right, wrong = { }, { } + for _, lib in ipairs(libs) do + local fullname = foundpath .. "/" .. lib + if lfs.isfile(fullname) then + -- right[#right+1] = lib + utils.report("merging library %s",fullname) + result[#result+1] = "do -- create closure to overcome 200 locals limit" + result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = "end -- of closure" + else + -- wrong[#wrong+1] = lib + utils.report("no library %s",fullname) + end + end + if #right > 0 then + utils.report("merged libraries: %s",table.concat(right," ")) + end + if #wrong > 0 then + utils.report("skipped libraries: %s",table.concat(wrong," ")) + end + else + utils.report("no valid library path found") + end + return table.concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if strip ~= false then + command = "-s " .. command + end + local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + -- utils.report("removing",luafile) + os.remove(luafile) + end + return done +end + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-tab'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides -parent access; a first version used different tricky but was less optimized to we +parent access; a first version used different trickery but was less optimized to we went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> @@ -2267,17 +2608,30 @@ optimize the code.</p> --ldx]]-- xml = xml or { } -tex = tex or { } -xml.trace_lpath = false -xml.trace_print = false -xml.trace_remap = false +--~ local xml = xml + +local concat, remove, insert = table.concat, table.remove, table.insert +local type, next, setmetatable = type, next, setmetatable +local format, lower, find = string.format, string.lower, string.find -local format, concat, remove, insert, type, next = string.format, table.concat, table.remove, table.insert, type, next +--[[ldx-- +<p>This module can be used stand alone but also inside <l n='mkiv'/> in +which case it hooks into the tracker code. Therefore we provide a few +functions that set the tracers.</p> +--ldx]]-- ---~ local pairs, next, type = pairs, next, type +local trace_remap = false --- todo: some things per xml file, like namespace remapping +if trackers then + trackers.register("xml.remap", function(v) trace_remap = v end) +end + +function xml.settrace(str,value) + if str == "remap" then + trace_remap = value or false + end +end --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -2289,65 +2643,61 @@ much cleaner.</p> xml.xmlns = xml.xmlns or { } -do - - local check = lpeg.P(false) - local parse = check +local check = lpeg.P(false) +local parse = check - --[[ldx-- - <p>The next function associates a namespace prefix with an <l n='url'/>. This - normally happens independent of parsing.</p> +--[[ldx-- +<p>The next function associates a namespace prefix with an <l n='url'/>. This +normally happens independent of parsing.</p> - <typing> - xml.registerns("mml","mathml") - </typing> - --ldx]]-- +<typing> +xml.registerns("mml","mathml") +</typing> +--ldx]]-- - function xml.registerns(namespace, pattern) -- pattern can be an lpeg - check = check + lpeg.C(lpeg.P(pattern:lower())) / namespace - parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } - end +function xml.registerns(namespace, pattern) -- pattern can be an lpeg + check = check + lpeg.C(lpeg.P(lower(pattern))) / namespace + parse = lpeg.P { lpeg.P(check) + 1 * lpeg.V(1) } +end - --[[ldx-- - <p>The next function also registers a namespace, but this time we map a - given namespace prefix onto a registered one, using the given - <l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> +--[[ldx-- +<p>The next function also registers a namespace, but this time we map a +given namespace prefix onto a registered one, using the given +<l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> - <typing> - xml.checkns("m","http://www.w3.org/mathml") - </typing> - --ldx]]-- +<typing> +xml.checkns("m","http://www.w3.org/mathml") +</typing> +--ldx]]-- - function xml.checkns(namespace,url) - local ns = parse:match(url:lower()) - if ns and namespace ~= ns then - xml.xmlns[namespace] = ns - end +function xml.checkns(namespace,url) + local ns = parse:match(lower(url)) + if ns and namespace ~= ns then + xml.xmlns[namespace] = ns end +end - --[[ldx-- - <p>Next we provide a way to turn an <l n='url'/> into a registered - namespace. This used for the <t>xmlns</t> attribute.</p> - - <typing> - resolvedns = xml.resolvens("http://www.w3.org/mathml") - </typing> - - This returns <t>mml</t>. - --ldx]]-- +--[[ldx-- +<p>Next we provide a way to turn an <l n='url'/> into a registered +namespace. This used for the <t>xmlns</t> attribute.</p> - function xml.resolvens(url) - return parse:match(url:lower()) or "" - end +<typing> +resolvedns = xml.resolvens("http://www.w3.org/mathml") +</typing> - --[[ldx-- - <p>A namespace in an element can be remapped onto the registered - one efficiently by using the <t>xml.xmlns</t> table.</p> - --ldx]]-- +This returns <t>mml</t>. +--ldx]]-- +function xml.resolvens(url) + return parse:match(lower(url)) or "" end --[[ldx-- +<p>A namespace in an element can be remapped onto the registered +one efficiently by using the <t>xml.xmlns</t> table.</p> +--ldx]]-- + +--[[ldx-- <p>This version uses <l n='lpeg'/>. We follow the same approach as before, stack and top and such. This version is about twice as fast which is mostly due to the fact that we don't have to prepare the stream for cdata, doctype etc etc. This variant is @@ -2382,247 +2732,253 @@ element.</p> xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes -do +-- not just one big nested table capture (lpeg overflow) - -- not just one big nested table capture (lpeg overflow) +local nsremap, resolvens = xml.xmlns, xml.resolvens - local nsremap, resolvens = xml.xmlns, xml.resolvens +local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} - local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local mt = { __tostring = xml.text } - local mt = { __tostring = xml.text } +function xml.check_error(top,toclose) + return "" +end - function xml.check_error(top,toclose) - return "" - end +local strip = false +local cleanup = false - local strip = false - local cleanup = false +function xml.set_text_cleanup(fnc) + cleanup = fnc +end - function xml.set_text_cleanup(fnc) - cleanup = fnc +local function add_attribute(namespace,tag,value) + if cleanup and #value > 0 then + value = cleanup(value) -- new + end + if tag == "xmlns" then + xmlns[#xmlns+1] = resolvens(value) + at[tag] = value + elseif namespace == "xmlns" then + xml.checkns(tag,value) + at["xmlns:" .. tag] = value + else + at[tag] = value end +end - local function add_attribute(namespace,tag,value) - if tag == "xmlns" then - xmlns[#xmlns+1] = resolvens(value) - at[tag] = value - elseif namespace == "xmlns" then - xml.checkns(tag,value) - at["xmlns:" .. tag] = value - else - at[tag] = value - end - end - local function add_begin(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - at = { } - end - local function add_end(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local toclose = remove(stack) - top = stack[#stack] - if #stack < 1 then - errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") - elseif toclose.tg ~= tag then -- no namespace check - errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") - end - dt = top.dt - dt[#dt+1] = toclose - if toclose.at.xmlns then - remove(xmlns) - end - end - local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } - end - local function add_text(text) - if cleanup and #text > 0 then - dt[#dt+1] = cleanup(text) - else - dt[#dt+1] = text - end +local function add_begin(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing end - local function add_special(what, spacing, text) - if #spacing > 0 then - dt[#dt+1] = spacing - end - if strip and (what == "@cm@" or what == "@dt@") then - -- forget it - else - dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } - end - end - local function set_message(txt) - errorstr = "garbage at the end of the file: " .. txt:gsub("([ \n\r\t]*)","") - end - - local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V - - local space = S(' \r\n\t') - local open = P('<') - local close = P('>') - local squote = S("'") - local dquote = S('"') - local equal = P('=') - local slash = P('/') - local colon = P(':') - local valid = R('az', 'AZ', '09') + S('_-.') - local name_yes = C(valid^1) * colon * C(valid^1) - local name_nop = C(P(true)) * C(valid^1) - local name = name_yes + name_nop - - local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + - P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture - - local spacing = C(space^0) - local justtext = C((1-open)^1) - local somespace = space^1 - local optionalspace = space^0 - - local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) - local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute - local attributes = attribute^0 - - local text = justtext / add_text - local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example - - local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty - local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin - local endelement = (spacing * open * slash * name * optionalspace * close) / add_end - - local begincomment = open * P("!--") - local endcomment = P("--") * close - local begininstruction = open * P("?") - local endinstruction = P("?") * close - local begincdata = open * P("![CDATA[") - local endcdata = P("]]") * close - - local someinstruction = C((1 - endinstruction)^0) - local somecomment = C((1 - endcomment )^0) - local somecdata = C((1 - endcdata )^0) - - function entity(k,v) entities[k] = v end - - local begindoctype = open * P("!DOCTYPE") - local enddoctype = close - local beginset = P("[") - local endset = P("]") - local doctypename = C((1-somespace)^0) - local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close - local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close - local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace - local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace - local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset - local simpledoctype = (1-close)^1 -- * balanced^0 - local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) - - local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end - local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end - local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end - local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dt@",...) end - - -- nicer but slower: - -- - -- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special - -- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special - -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special - -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special - - local trailer = space^0 * (justtext/set_message)^0 - - -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file - -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 - -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 - - local grammar = P { "preamble", - preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, - parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, - } + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } + setmetatable(top, mt) + dt = top.dt + stack[#stack+1] = top + at = { } +end - -- todo: xml.new + properties like entities and strip and such (store in root) - - function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} - stack[#stack+1] = top - top.dt = { } - dt = top.dt - if not data or data == "" then - errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" - else - errorstr = "" - end - if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } - setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end - else - result = stack[1] - end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } - setmetatable(result, mt) - local rdt = result.dt - for k=1,#rdt do - local v = rdt[k] - if type(v) == "table" and not v.special then -- always table -) - result.ri = k -- rootindex - break - end - end - end - return result +local function add_end(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local toclose = remove(stack) + top = stack[#stack] + if #stack < 1 then + errorstr = format("nothing to close with %s %s", tag, xml.check_error(top,toclose) or "") + elseif toclose.tg ~= tag then -- no namespace check + errorstr = format("unable to close %s with %s %s", toclose.tg, tag, xml.check_error(top,toclose) or "") end + dt = top.dt + dt[#dt+1] = toclose + dt[0] = top + if toclose.at.xmlns then + remove(xmlns) + end +end - --[[ldx-- - <p>Packaging data in an xml like table is done with the following - function. Maybe it will go away (when not used).</p> - --ldx]]-- +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) + end + at = { } +end - function xml.is_valid(root) - return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er +local function add_text(text) + if cleanup and #text > 0 then + dt[#dt+1] = cleanup(text) + else + dt[#dt+1] = text end +end - function xml.package(tag,attributes,data) - local ns, tg = tag:match("^(.-):?([^:]+)$") - local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } - setmetatable(t, mt) - return t +local function add_special(what, spacing, text) + if #spacing > 0 then + dt[#dt+1] = spacing end + if strip and (what == "@cm@" or what == "@dt@") then + -- forget it + else + dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + end +end + +local function set_message(txt) + errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") +end + +local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V + +local space = S(' \r\n\t') +local open = P('<') +local close = P('>') +local squote = S("'") +local dquote = S('"') +local equal = P('=') +local slash = P('/') +local colon = P(':') +local valid = R('az', 'AZ', '09') + S('_-.') +local name_yes = C(valid^1) * colon * C(valid^1) +local name_nop = C(P(true)) * C(valid^1) +local name = name_yes + name_nop + +local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + + P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture + +local spacing = C(space^0) +local justtext = C((1-open)^1) +local somespace = space^1 +local optionalspace = space^0 + +local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) +local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute +local attributes = attribute^0 + +local text = justtext / add_text +local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example + +local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty +local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin +local endelement = (spacing * open * slash * name * optionalspace * close) / add_end + +local begincomment = open * P("!--") +local endcomment = P("--") * close +local begininstruction = open * P("?") +local endinstruction = P("?") * close +local begincdata = open * P("![CDATA[") +local endcdata = P("]]") * close + +local someinstruction = C((1 - endinstruction)^0) +local somecomment = C((1 - endcomment )^0) +local somecdata = C((1 - endcdata )^0) + +local function entity(k,v) entities[k] = v end + +local begindoctype = open * P("!DOCTYPE") +local enddoctype = close +local beginset = P("[") +local endset = P("]") +local doctypename = C((1-somespace)^0) +local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close +local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close +local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace +local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace +local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset +local simpledoctype = (1-close)^1 -- * balanced^0 +local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) + +local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end +local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end +local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end +local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dt@",...) end + +-- nicer but slower: +-- +-- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special +-- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special +-- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special +-- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special + +local trailer = space^0 * (justtext/set_message)^0 + +-- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file +-- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 +-- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 + +local grammar = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = text + V("parent") + emptyelement + comment + cdata + instruction, +} - function xml.is_valid(root) - return root and not root.error +-- todo: xml.new + properties like entities and strip and such (store in root) + +function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) + strip = strip_cm_and_dt or xml.strip_cm_and_dt + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} + stack[#stack+1] = top + top.dt = { } + dt = top.dt + if not data or data == "" then + errorstr = "empty xml file" + elseif not grammar:match(data) then + errorstr = "invalid xml file" + else + errorstr = "" end + if errorstr and errorstr ~= "" then + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + setmetatable(stack, mt) + if xml.error_handler then xml.error_handler("load",errorstr) end + else + result = stack[1] + end + if not no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + setmetatable(result, mt) + local rdt = result.dt + for k=1,#rdt do + local v = rdt[k] + if type(v) == "table" and not v.special then -- always table -) + result.ri = k -- rootindex + break + end + end + end + return result +end + +--[[ldx-- +<p>Packaging data in an xml like table is done with the following +function. Maybe it will go away (when not used).</p> +--ldx]]-- - xml.error_handler = (logs and logs.report) or (input and input.report) or print +function xml.is_valid(root) + return root and root.dt and root.dt[1] and type(root.dt[1]) == "table" and not root.dt[1].er +end + +function xml.package(tag,attributes,data) + local ns, tg = tag:match("^(.-):?([^:]+)$") + local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } + setmetatable(t, mt) + return t +end +function xml.is_valid(root) + return root and not root.error end +xml.error_handler = (logs and logs.report) or (input and logs.report) or print + --[[ldx-- <p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load the whole file first. The function accepts a string representing @@ -2666,32 +3022,28 @@ generic table copier. Since we know what we're dealing with we can speed up things a bit. The second argument is not to be used!</p> --ldx]]-- -do - - function copy(old,tables) - if old then - tables = tables or { } - local new = { } - if not tables[old] then - tables[old] = new - end - for k,v in pairs(old) do - new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } +function copy(old,tables) + if old then + tables = tables or { } + local new = { } + if not tables[old] then + tables[old] = new end + for k,v in pairs(old) do + new[k] = (type(v) == "table" and (tables[v] or copy(v, tables))) or v + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return { } end - - xml.copy = copy - end +xml.copy = copy + --[[ldx-- <p>In <l n='context'/> serializing the tree or parts of the tree is a major actitivity which is why the following function is pretty optimized resulting @@ -2700,207 +3052,204 @@ function for all components is about 15% slower than the concatinating alternative.</p> --ldx]]-- -do - - -- todo: add <?xml version='1.0' standalone='yes'?> when not present - - local fallbackhandle = (tex and tex.sprint) or io.write - - local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return - end - end - end - handle = handle or fallbackhandle - local etg = e.tg - if etg then +-- todo: add <?xml version='1.0' standalone='yes'?> when not present + +local fallbackhandle = (tex and tex.sprint) or io.write + +local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) + if not e then + return + elseif not nocommands then + local ec = e.command + if ec ~= nil then -- we can have all kind of types if e.special then - local edt = e.dt + local etg, edt = e.tg, e.dt local spc = specialconverter and specialconverter[etg] if spc then local result = spc(edt[1]) if result then handle(result) + return else -- no need to handle any further end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) end - else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster - if ats then - if attributeconverter then - for k,v in pairs(eat) do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in pairs(eat) do - ats[#ats+1] = format('%s=%q',k,v) - end - end + end + local xc = xml.command + if xc then + xc(e,ec) + return + end + end + end + handle = handle or fallbackhandle + local etg = e.tg + if etg then + if e.special then + local edt = e.dt + local spc = specialconverter and specialconverter[etg] + if spc then + local result = spc(edt[1]) + if result then + handle(result) + else + -- no need to handle any further end - if ern and xml.trace_remap and ern ~= ens then - ens = ern + elseif etg == "@pi@" then + -- handle(format("<?%s?>",edt[1])) + handle("<?" .. edt[1] .. "?>") + elseif etg == "@cm@" then + -- handle(format("<!--%s-->",edt[1])) + handle("<!--" .. edt[1] .. "-->") + elseif etg == "@cd@" then + -- handle(format("<![CDATA[%s]]>",edt[1])) + handle("<![CDATA[" .. edt[1] .. "]]>") + elseif etg == "@dt@" then + -- handle(format("<!DOCTYPE %s>",edt[1])) + handle("<!DOCTYPE " .. edt[1] .. ">") + elseif etg == "@rt@" then + serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) + end + else + local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } -- type test maybe faster + if ats then + if attributeconverter then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) + end + else + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") + else + -- handle(format("<%s:%s>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. ">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + if textconverter then + handle(textconverter(e)) else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) + handle(e) end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") - else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") + serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) end end + -- handle(format("</%s:%s>",ens,etg)) + handle("</" .. ens .. ":" .. etg .. ">") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end + if ats then + -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") + else + -- handle(format("<%s:%s/>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. "/>") + end + end + else + if edt and #edt > 0 then + if ats then + -- handle(format("<%s %s>",etg,concat(ats," "))) + handle("<" .. etg .. " " .. concat(ats," ") .. ">") + else + -- handle(format("<%s>",etg)) + handle("<" .. etg .. ">") + end + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + if textconverter then + handle(textconverter(ei)) else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) + handle(ei) end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") + serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) end end + -- handle(format("</%s>",etg)) + handle("</" .. etg .. ">") + else + if ats then + -- handle(format("<%s %s/>",etg,concat(ats," "))) + handle("<" .. etg .. " " .. concat(ats," ") .. "/>") + else + -- handle(format("<%s/>",etg)) + handle("<" .. etg .. "/>") + end end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end + end + elseif type(e) == "string" then + if textconverter then + handle(textconverter(e)) else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end + handle(e) + end + else + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + if textconverter then + handle(textconverter(ei)) else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) + handle(ei) end + else + serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) end end end +end - xml.serialize = serialize +xml.serialize = serialize - function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k,v in ipairs(dt) do - if type(v) == "table" and v.special and v.tg == "@pi" and v.dt:find("xml.*version=") then - found = true - break - end - end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end +end - --[[ldx-- - <p>At the cost of some 25% runtime overhead you can first convert the tree to a string - and then handle the lot.</p> - --ldx]]-- +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- - function xml.tostring(root) -- 25% overhead due to collecting - if root then - if type(root) == 'string' then - return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) - return concat(result,"") - end +function xml.tostring(root) -- 25% overhead due to collecting + if root then + if type(root) == 'string' then + return root + elseif next(root) then -- next is faster than type (and >0 test) + local result = { } + serialize(root,function(s) result[#result+1] = s end) + return concat(result,"") end - return "" end - + return "" end --[[ldx-- @@ -3010,355 +3359,394 @@ function xml.assign(dt,k,root) end end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-pth'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local concat, remove, insert = table.concat, table.remove, table.insert +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, lower, gmatch, gsub, find = string.format, string.lower, string.gmatch, string.gsub, string.find + +--[[ldx-- +<p>This module can be used stand alone but also inside <l n='mkiv'/> in +which case it hooks into the tracker code. Therefore we provide a few +functions that set the tracers. Here we overload a previously defined +function.</p> +--ldx]]-- + +local trace_lpath = false + +if trackers then + trackers.register("xml.lpath", function(v) trace_lpath = v end) +end + +local settrace = xml.settrace -- lxml-tab + +function xml.settrace(str,value) + if str == "lpath" then + trace_lpath = value or false + else + settrace(str,value) -- lxml-tab + end +end + --[[ldx-- <p>We've now arrived at an intersting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statisctics -local lpathcached = 0 -- statisctics - -do - - xml.functions = xml.functions or { } - xml.expressions = xml.expressions or { } - - local functions = xml.functions - local expressions = xml.expressions - - local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", - } +local lpathcalls = 0 -- statistics +local lpathcached = 0 -- statistics + +xml.functions = xml.functions or { } +xml.expressions = xml.expressions or { } + +local functions = xml.functions +local expressions = xml.expressions + +local actions = { + [10] = "stay", + [11] = "parent", + [12] = "subtree root", + [13] = "document root", + [14] = "any", + [15] = "many", + [16] = "initial", + [20] = "match", + [21] = "match one of", + [22] = "match and attribute eq", + [23] = "match and attribute ne", + [24] = "match one of and attribute eq", + [25] = "match one of and attribute ne", + [27] = "has attribute", + [28] = "has value", + [29] = "fast match", + [30] = "select", + [31] = "expression", + [40] = "processing instruction", +} - -- a rather dumb lpeg +-- a rather dumb lpeg - local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc - -- instead of using functions we just parse a few names which saves a call - -- later on +-- instead of using functions we just parse a few names which saves a call +-- later on - local lp_position = P("position()") / "ps" - local lp_index = P("index()") / "id" - local lp_text = P("text()") / "tx" - local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" - local lp_tag = P("tag()") / "tg" -- (rt.tg or '') - local lp_ns = P("ns()") / "ns" -- (rt.ns or '') - local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") - local lp_doequal = P("=") / "==" - local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +local lp_position = P("position()") / "ps" +local lp_index = P("index()") / "id" +local lp_text = P("text()") / "tx" +local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" +local lp_tag = P("tag()") / "tg" -- (rt.tg or '') +local lp_ns = P("ns()") / "ns" -- (rt.ns or '') +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") - local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling - return t .. "(" +local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling + return t .. "(" +end + +local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling + if expressions[t] then + return "expressions." .. t .. "(" + else + return "expressions.error(" end +end - local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling - if expressions[t] then - return "expressions." .. t .. "(" +local lparent = lpeg.P("(") +local rparent = lpeg.P(")") +local noparent = 1 - (lparent+rparent) +local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} +local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} + +-- if we use a dedicated namespace then we don't need to pass rt and k + +local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) + if expressions[t] then + if s then + return "expressions." .. t .. "(r,k," .. s ..")" else - return "expressions.error(" + return "expressions." .. t .. "(r,k)" end + else + return "expressions.error(" .. t .. ")" end +end - local lparent = lpeg.P("(") - local rparent = lpeg.P(")") - local noparent = 1 - (lparent+rparent) - local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} - local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} +local converter = lpeg.Cs ( ( + lp_position + + lp_index + + lp_text + lp_name + -- fast one + lp_special + + lp_noequal + lp_doequal + + lp_attribute + + lp_lua_function + + lp_function + +1 )^1 ) - -- if we use a dedicated namespace then we don't need to pass rt and k +-- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 - local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) - if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" - else - return "expressions." .. t .. "(r,k)" - end - else - return "expressions.error(" .. t .. ")" - end - end - - local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one - lp_special + - lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + - 1 )^1 ) - - -- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 - - local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" - return %s - end - ]] - - local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() - end - - local map = { } - - local space = S(' \r\n\t') - local squote = S("'") - local dquote = S('"') - local lparent = P('(') - local rparent = P(')') - local atsign = P('@') - local lbracket = P('[') - local rbracket = P(']') - local exclam = P('!') - local period = P('.') - local eq = P('==') + P('=') - local ne = P('<>') + P('!=') - local star = P('*') - local slash = P('/') - local colon = P(':') - local bar = P('|') - local hat = P('^') - local valid = R('az', 'AZ', '09') + S('_-') ---~ local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* ---~ local name_nop = C(P(true)) * C(valid^1) - local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* - local name_nop = Cc("*") * C(valid^1) - local name = name_yes + name_nop - local number = C((S('+-')^0 * R('09')^1)) / tonumber - local names = (bar^0 * name)^1 - local morenames = name * (bar^0 * name)^1 - local instructiontag = P('pi::') - local spacing = C(space^0) - local somespace = space^1 - local optionalspace = space^0 - local text = C(valid^0) - local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) - local empty = 1-slash - - local is_eq = lbracket * atsign * name * eq * value * rbracket - local is_ne = lbracket * atsign * name * ne * value * rbracket - local is_attribute = lbracket * atsign * name * rbracket - local is_value = lbracket * value * rbracket - local is_number = lbracket * number * rbracket - - local nobracket = 1-(lbracket+rbracket) -- must be improved - local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - - local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - - local is_one = name - local is_none = exclam * name - local is_one_of = ((lparent * names * rparent) + morenames) - local is_none_of = exclam * ((lparent * names * rparent) + morenames) - - local stay = (period ) - local parent = (period * period ) / function( ) map[#map+1] = { 11 } end - local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end - local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end - local any = (star ) / function( ) map[#map+1] = { 14 } end - local many = (star * star ) / function( ) map[#map+1] = { 15 } end - local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - - local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end - local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end - local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end - local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - - local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end - local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end - local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end - local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - - local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end - local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end - local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end - local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - - local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end - local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end - local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end - local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end - local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end - local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - - local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end - local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - - local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end - local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - - local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end - local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? - local crap = (1-slash)^1 - - -- a few ugly goodies: - - local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end - local subroottag = P('^') / function( ) map[#map+1] = { 13 } end - local roottag = P('root::') / function( ) map[#map+1] = { 12 } end - local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end - local childtag = P('child::') - local selftag = P('self::') - - -- there will be more and order will be optimized - - local selector = ( - instruction + ---~ many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty - ) +local template = [[ + return function(expressions,r,d,k,e,dt,ns,tg,id,ps) + local at, tx = e.at or { }, dt[1] or "" + return %s + end +]] - local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, - } +local function make_expression(str) + str = converter:match(str) + return str, loadstring(format(template,str))() +end + +local map = { } + +local space = S(' \r\n\t') +local squote = S("'") +local dquote = S('"') +local lparent = P('(') +local rparent = P(')') +local atsign = P('@') +local lbracket = P('[') +local rbracket = P(']') +local exclam = P('!') +local period = P('.') +local eq = P('==') + P('=') +local ne = P('<>') + P('!=') +local star = P('*') +local slash = P('/') +local colon = P(':') +local bar = P('|') +local hat = P('^') +local valid = R('az', 'AZ', '09') + S('_-') +local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* +local name_nop = Cc("*") * C(valid^1) +local name = name_yes + name_nop +local number = C((S('+-')^0 * R('09')^1)) / tonumber +local names = (bar^0 * name)^1 +local morenames = name * (bar^0 * name)^1 +local instructiontag = P('pi::') +local spacing = C(space^0) +local somespace = space^1 +local optionalspace = space^0 +local text = C(valid^0) +local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) +local empty = 1-slash + +local is_eq = lbracket * atsign * name * eq * value * rbracket +local is_ne = lbracket * atsign * name * ne * value * rbracket +local is_attribute = lbracket * atsign * name * rbracket +local is_value = lbracket * value * rbracket +local is_number = lbracket * number * rbracket + +local nobracket = 1-(lbracket+rbracket) -- must be improved +local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket + +local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket + +local is_one = name +local is_none = exclam * name +local is_one_of = ((lparent * names * rparent) + morenames) +local is_none_of = exclam * ((lparent * names * rparent) + morenames) + +local stay = (period ) +local parent = (period * period ) / function( ) map[#map+1] = { 11 } end +local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end +local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end +local any = (star ) / function( ) map[#map+1] = { 14 } end +local many = (star * star ) / function( ) map[#map+1] = { 15 } end +local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end + +local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end +local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end +local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end +local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end + +local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end +local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end +local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end +local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end + +local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end +local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end +local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end +local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end + +local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end +local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end +local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end +local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end +local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end +local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end + +local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end +local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end + +local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end + map[#map+1] = { 31, true, "*", "*", ... } end +local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end + map[#map+1] = { 31, false, "*", "*", ... } end + +local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end +local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? +local crap = (1-slash)^1 + +-- a few ugly goodies: + +local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end +local subroottag = P('^') / function( ) map[#map+1] = { 13 } end +local roottag = P('root::') / function( ) map[#map+1] = { 12 } end +local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end +local childtag = P('child::') +local selftag = P('self::') + +-- there will be more and order will be optimized + +local selector = ( + instruction + +-- many + any + -- brrr, not here ! + parent + stay + + dont_position + position + + dont_match_one_of_and_eq + dont_match_one_of_and_ne + + match_one_of_and_eq + match_one_of_and_ne + + dont_match_and_eq + dont_match_and_ne + + match_and_eq + match_and_ne + + dont_expression + expression + + dont_self_expression + self_expression + + has_attribute + has_value + + dont_match_one_of + match_one_of + + dont_match + match + + many + any + + crap + empty +) + +local grammar = P { "startup", + startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), + followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, +} - local function compose(str) - if not str or str == "" then - -- wildcard +local function compose(str) + if not str or str == "" then + -- wildcard + return true + elseif str == '/' then + -- root + return false + else + map = { } + grammar:match(str) + if #map == 0 then return true - elseif str == '/' then - -- root - return false else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) + local m = map[1][1] + if #map == 1 then + if m == 14 or m == 15 then + -- wildcard + return true + elseif m == 12 then + -- root + return false end - -- print((table.serialize(map)):gsub("[ \n]+"," ")) - return map + elseif #map == 2 and m == 12 and map[2][1] == 20 then + -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } + map[2][1] = 29 + return { map[2] } + end + if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then + insert(map, 1, { 16 }) end + -- print(gsub(table.serialize(map),"[ \n]+"," ")) + return map end end +end - local cache = { } +local cache = { } - function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 - end - if trace or xml.trace_lpath then - xml.lshow(result) - end - return result - else - return pattern +function xml.lpath(pattern,trace) + lpathcalls = lpathcalls + 1 + if type(pattern) == "string" then + local result = cache[pattern] + if result == nil then -- can be false which is valid -) + result = compose(pattern) + cache[pattern] = result + lpathcached = lpathcached + 1 end + if trace or trace_lpath then + xml.lshow(result) + end + return result + else + return pattern end +end - function lpath_cached_patterns() - return cache - end - - local fallbackreport = (texio and texio.write) or io.write +function xml.cached_patterns() + return cache +end - function xml.lshow(pattern,report) - report = report or fallbackreport - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") - else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k,v in ipairs(lp) do - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" - end +-- we run out of locals (limited to 200) +-- +-- local fallbackreport = (texio and texio.write) or io.write + +function xml.lshow(pattern,report) +-- report = report or fallbackreport + report = report or (texio and texio.write) or io.write + local lp = xml.lpath(pattern) + if lp == false then + report(" -: root\n") + elseif lp == true then + report(" -: wildcard\n") + else + if type(pattern) == "string" then + report(format("pattern: %s\n",pattern)) + end + for k=1,#lp do + local v = lp[k] + if #v > 1 then + local t = { } + for i=2,#v do + local vv = v[i] + if type(vv) == "string" then + t[#t+1] = (vv ~= "" and vv) or "#" + elseif type(vv) == "boolean" then + t[#t+1] = (vv and "==") or "<>" end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) - else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) end + report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) + else + report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) end end end +end - function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } - local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") - else - for i=1,#e do - report(tostring(e[i]) .. "\n") - end +function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e + local t = { ... } +-- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport + local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write + if e == nil then + report("<!-- no element -->\n") + elseif type(e) ~= "table" then + report(tostring(e)) + elseif e.tg then + report(tostring(e) .. "\n") + else + for i=1,#e do + report(tostring(e[i]) .. "\n") end end - end --[[ldx-- @@ -3376,161 +3764,174 @@ advance what we want to do with the found element the handle gets three argument functions.</p> --ldx]]-- -do - - local functions = xml.functions - local expressions = xml.expressions +local functions = xml.functions +local expressions = xml.expressions - expressions.contains = string.find - expressions.find = string.find - expressions.upper = string.upper - expressions.lower = string.lower - expressions.number = tonumber - expressions.boolean = toboolean +expressions.contains = string.find +expressions.find = string.find +expressions.upper = string.upper +expressions.lower = string.lower +expressions.number = tonumber +expressions.boolean = toboolean - expressions.oneof = function(s,...) -- slow - local t = {...} for i=1,#t do if s == t[i] then return true end end return false - end - - expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") - return false - end +expressions.oneof = function(s,...) -- slow + local t = {...} for i=1,#t do if s == t[i] then return true end end return false +end - functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" - end - end +expressions.error = function(str) + xml.error_handler("unknown function in lpath expression",str or "?") + return false +end - functions.name = function(d,k,n) -- ns + tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end +functions.text = function(root,k,n) -- unchecked, maybe one deeper + local t = type(t) + if t == "string" then + return t + else -- todo n + local rdt = root.dt + return (rdt and rdt[k]) or root[k] or "" + end +end + +functions.name = function(d,k,n) -- ns + tg + local found = false + n = n or 0 + if not k then + -- not found + elseif n == 0 then + local dk = d[k] + found = dk and (type(dk) == "table") and dk + elseif n < 0 then + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 end end - else - for i=k+1,#d,1 do - local di = d[i] - if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + end + else + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 end end end - if found then - local ns, tg = found.rn or found.ns or "", found.tg - if ns ~= "" then - return ns .. ":" .. tg - else - return tg - end + end + if found then + local ns, tg = found.rn or found.ns or "", found.tg + if ns ~= "" then + return ns .. ":" .. tg else - return "" + return tg end + else + return "" end +end - functions.tag = function(d,k,n) -- only tg - local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk - elseif n < 0 then - for i=k-1,1,-1 do - local di = d[i] - if type(di) == "table" then - if n == -1 then - found = di - break - else - n = n + 1 - end +functions.tag = function(d,k,n) -- only tg + local found = false + n = n or 0 + if not k then + -- not found + elseif n == 0 then + local dk = d[k] + found = dk and (type(dk) == "table") and dk + elseif n < 0 then + for i=k-1,1,-1 do + local di = d[i] + if type(di) == "table" then + if n == -1 then + found = di + break + else + n = n + 1 end end - else - for i=k+1,#d,1 do - local di = d[i] - if type(di) == "table" then - if n == 1 then - found = di - break - else - n = n - 1 - end + end + else + for i=k+1,#d,1 do + local di = d[i] + if type(di) == "table" then + if n == 1 then + found = di + break + else + n = n - 1 end end end - return (found and found.tg) or "" end + return (found and found.tg) or "" +end - expressions.text = functions.text - expressions.name = functions.name - expressions.tag = functions.tag +expressions.text = functions.text +expressions.name = functions.name +expressions.tag = functions.tag - local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end +local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces + if not root then -- error + return false + elseif pattern == false then -- root + handle(root,root.dt,root.ri) + return false + elseif pattern == true then -- wildcard + local rootdt = root.dt + if rootdt then + local start, stop, step = 1, #rootdt, 1 + if reverse then + start, stop, step = stop, start, -1 end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end + for k=start,stop,step do + if handle(root,rootdt,root.ri or k) then return false end + if not traverse(rootdt[k],true,handle,reverse) then return false end + end + end + return false + elseif root.dt then + index = index or 1 + local action = pattern[index] + local command = action[1] + if command == 29 then -- fast case /oeps + local rootdt = root.dt + for k=1,#rootdt do + local e = rootdt[k] + local tg = e.tg + if e.tg then + local ns = e.rn or e.ns + local ns_a, tg_a = action[3], action[4] + local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) + if not action[2] then matched = not matched end + if matched then + if handle(root,rootdt,k) then return false end end end - elseif command == 11 then -- parent + end + elseif command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + if (command == 16 or command == 12) and index == 1 then -- initial + -- wildcard = true + wildcard = command == 16 -- ok? + index = index + 1 + action = pattern[index] + command = action and action[1] or 0 -- something is wrong + end + if command == 11 then -- parent local ep = root.__p__ or parent if index < #pattern then if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end @@ -3538,41 +3939,51 @@ do return false end else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then + local rootdt = root.dt + local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 + if command == 30 then + if action[5] < 0 then start, stop, step = stop, start, -1 + dn = -1 end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then + elseif reverse and index == #pattern then + start, stop, step = stop, start, -1 + end + local idx = 0 + local hsh = { } -- this will slooow down the lot + for k=start,stop,step do -- we used to have functions for all but a case is faster + local e = rootdt[k] + local ns, tg = e.rn or e.ns, e.tg + if tg then + -- we can optimize this for simple searches, but it probably does not pay off + hsh[tg] = (hsh[tg] or 0) + 1 + idx = idx + 1 + if command == 30 then + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + if matched then + n = n + dn + if n == action[5] then + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + break + end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + end + else + local matched, multiple = false, false + if command == 20 then -- match local ns_a, tg_a = action[3], action[4] if tg == tg_a then matched = ns_a == "*" or ns == ns_a @@ -3582,193 +3993,166 @@ do matched = false end if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end + elseif command == 21 then -- match one of + multiple = true + for i=3,#action,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true break end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end + if not action[2] then matched = not matched end + elseif command == 22 then -- eq + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end + matched = matched and e.at[action[6]] == action[7] + elseif command == 23 then -- ne + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = mached and e.at[action[6]] ~= action[7] + elseif command == 24 then -- one of eq + multiple = true + for i=3,#action-2,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] == action[#action] + elseif command == 25 then -- one of ne + multiple = true + for i=3,#action-2,2 do + local ns_a, tg_a = action[i], action[i+1] + if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then + matched = true + break end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] ~= action[#action] + elseif command == 27 then -- has attribute + local ns_a, tg_a = action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = matched and e.at[action[5]] + elseif command == 28 then -- has value + local edt, ns_a, tg_a = e.dt, action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + matched = matched and edt and edt[1] == action[5] + elseif command == 31 then + local edt, ns_a, tg_a = e.dt, action[3], action[4] + if tg == tg_a then + matched = ns_a == "*" or ns == ns_a + elseif tg_a == '*' then + matched, multiple = ns_a == "*" or ns == ns_a, true + else + matched = false + end + if not action[2] then matched = not matched end + if matched then + matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) + end + end + if matched then -- combine tg test and at test + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + if wildcard then + if multiple then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + else + -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml + if not traverse(e,pattern,handle,reverse,index,root) then return false end end - elseif handle(root,rootdt,k) then - return false end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 14 then -- any + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 15 then -- many + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end end - end - else -- not here : 11 - if command == 11 then -- parent + elseif command == 11 then -- parent local ep = e.__p__ or parent if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end + elseif handle(root,rootdt,k) then + return false + end + elseif command == 40 and e.special and tg == "@pi@" then -- pi + local pi = action[2] + if pi ~= "" then + local pt = e.dt[1] + if pt and pt:find(pi) then + if handle(root,rootdt,k) then + return false + end + end elseif handle(root,rootdt,k) then return false end - break -- else loop + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end end + else + -- not here : 11 + if command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + break -- else loop + end end end end end - return true end - - xml.traverse = traverse - + return true end +xml.traverse = traverse + --[[ldx-- <p>Next come all kind of locators and manipulators. The most generic function here is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace @@ -3779,399 +4163,414 @@ local r, d, k = xml.filter(root,"/a/b/c/position(4)" </typing> --ldx]]-- -do +local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert - local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +xml.filters = { } - xml.filters = { } +function xml.filters.default(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) + return dt and dt[dk], rt, dt, dk +end - function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk - end - function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk - else - return ekat, rt, dt, dk - end +function xml.filters.attributes(root,pattern,arguments) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) + if ekat then + if arguments then + return ekat[arguments] or "", rt, dt, dk else - return { }, rt, dt, dk + return ekat, rt, dt, dk end + else + return { }, rt, dt, dk end - function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk - end - function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n - end - function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t - end - function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t - end - function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk - end - function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk +end + +function xml.filters.reverse(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') + return dt and dt[dk], rt, dt, dk +end + +function xml.filters.count(root,pattern,everything) + local n = 0 + traverse(root, lpath(pattern), function(r,d,t) + if everything or type(d[t]) == "table" then + n = n + 1 + end + end) + return n +end + +function xml.filters.elements(root, pattern) -- == all + local t = { } + traverse(root, lpath(pattern), function(r,d,k) + local e = d[k] + if e then + t[#t+1] = e + end + end) + return t +end + +function xml.filters.texts(root, pattern) + local t = { } + traverse(root, lpath(pattern), function(r,d,k) + local e = d[k] + if e and e.dt then + t[#t+1] = e.dt + end + end) + return t +end + +function xml.filters.first(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) + return dt and dt[dk], rt, dt, dk +end + +function xml.filters.last(root,pattern) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') + return dt and dt[dk], rt, dt, dk +end + +function xml.filters.index(root,pattern,arguments) + local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 + if i and i ~= 0 then + if i < 0 then + reverse, i = true, -i + end + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) + if i == 0 then + return dt and dt[dk], rt, dt, dk + end end - function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk - end - end - return nil, nil, nil, nil - end - function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - return (ekat and (ekat[arguments] or ekat[arguments:gsub("^([\"\'])(.*)%1$","%2")])) or "" - end - function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk - end - else + return nil, nil, nil, nil +end + +function xml.filters.attribute(root,pattern,arguments) + local rt, dt, dk + traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) + return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" +end + +function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow + local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) + if dtk then -- n + local dtkdt = dtk.dt + if not dtkdt then return "", rt, dt, dk + elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then + return dtkdt[1], rt, dt, dk + else + return xml.tostring(dtkdt), rt, dt, dk end + else + return "", rt, dt, dk end - function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag - end - function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag - end +end - --[[ldx-- - <p>For splitting the filter function from the path specification, we can - use string matching or lpeg matching. Here the difference in speed is - neglectable but the lpeg variant is more robust.</p> - --ldx]]-- +function xml.filters.tag(root,pattern,n) + local tag = "" + traverse(root, lpath(pattern), function(r,d,k) + tag = xml.functions.tag(d,k,n and tonumber(n)) + return true + end) + return tag +end - -- not faster but hipper ... although ... i can't get rid of the trailing / in the path +function xml.filters.name(root,pattern,n) + local tag = "" + traverse(root, lpath(pattern), function(r,d,k) + tag = xml.functions.name(d,k,n and tonumber(n)) + return true + end) + return tag +end - local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +--[[ldx-- +<p>For splitting the filter function from the path specification, we can +use string matching or lpeg matching. Here the difference in speed is +neglectable but the lpeg variant is more robust.</p> +--ldx]]-- - local slash = P('/') - local name = (R("az","AZ","--","__"))^1 - local path = C(((1-slash)^0 * slash)^1) - local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } - local action = Cc(1) * path * C(name) * argument - local attribute = Cc(2) * path * P('@') * C(name) - local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument +-- not faster but hipper ... although ... i can't get rid of the trailing / in the path - local parser = direct + action + attribute +local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc - local filters = xml.filters - local attribute_filter = xml.filters.attributes - local default_filter = xml.filters.default +local slash = P('/') +local name = (R("az","AZ","--","__"))^1 +local path = C(((1-slash)^0 * slash)^1) +local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } +local action = Cc(1) * path * C(name) * argument +local attribute = Cc(2) * path * P('@') * C(name) +local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument - -- todo: also hash, could be gc'd +local parser = direct + action + attribute - function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) ---~ if xml.trace_lpath then ---~ print(pattern,kind,a,b,c) ---~ end - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) - else - return default_filter(root,pattern) - end - end - - --~ slightly faster, but first we need a proper test file - --~ - --~ local hash = { } - --~ - --~ function xml.filter(root,pattern) - --~ local h = hash[pattern] - --~ if not h then - --~ local kind, a, b, c = parser:match(pattern) - --~ if kind == 1 then - --~ h = { kind, filters[b] or default_filter, a, b, c } - --~ elseif kind == 2 then - --~ h = { kind, attribute_filter, a, b, c } - --~ else - --~ h = { kind, default_filter, a, b, c } - --~ end - --~ hash[pattern] = h - --~ end - --~ local kind = h[1] - --~ if kind == 1 then - --~ return h[2](root,h[2],h[4]) - --~ elseif kind == 2 then - --~ return h[2](root,h[2],h[3]) - --~ else - --~ return h[2](root,pattern) - --~ end - --~ end - - --[[ldx-- - <p>The following functions collect elements and texts.</p> - --ldx]]-- - - -- still somewhat bugged - - function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end +local filters = xml.filters +local attribute_filter = xml.filters.attributes +local default_filter = xml.filters.default + +-- todo: also hash, could be gc'd + +function xml.filter(root,pattern) + local kind, a, b, c = parser:match(pattern) + if kind == 1 or kind == 3 then + return (filters[b] or default_filter)(root,a,c) + elseif kind == 2 then + return attribute_filter(root,a,b) + else + return default_filter(root,pattern) + end +end + +--~ slightly faster, but first we need a proper test file +--~ +--~ local hash = { } +--~ +--~ function xml.filter(root,pattern) +--~ local h = hash[pattern] +--~ if not h then +--~ local kind, a, b, c = parser:match(pattern) +--~ if kind == 1 then +--~ h = { kind, filters[b] or default_filter, a, b, c } +--~ elseif kind == 2 then +--~ h = { kind, attribute_filter, a, b, c } +--~ else +--~ h = { kind, default_filter, a, b, c } +--~ end +--~ hash[pattern] = h +--~ end +--~ local kind = h[1] +--~ if kind == 1 then +--~ return h[2](root,h[2],h[4]) +--~ elseif kind == 2 then +--~ return h[2](root,h[2],h[3]) +--~ else +--~ return h[2](root,pattern) +--~ end +--~ end + +--[[ldx-- +<p>The following functions collect elements and texts.</p> +--ldx]]-- + +-- still somewhat bugged + +function xml.collect_elements(root, pattern, ignorespaces) + local rr, dd = { }, { } + traverse(root, lpath(pattern), function(r,d,k) + local dk = d and d[k] + if dk then + if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then + -- ignore + else + local n = #rr+1 + rr[n], dd[n] = r, dk end - end) - return dd, rr - end - - function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" - else - t[#t+1] = "" - end + end + end) + return dd, rr +end + +function xml.collect_texts(root, pattern, flatten) + local t = { } -- no r collector + traverse(root, lpath(pattern), function(r,d,k) + if d then + local ek = d[k] + local tx = ek and ek.dt + if flatten then + if tx then + t[#t+1] = xml.tostring(tx) or "" else - t[#t+1] = tx or "" + t[#t+1] = "" end else - t[#t+1] = "" + t[#t+1] = tx or "" end - end) - return t - end + else + t[#t+1] = "" + end + end) + return t +end - function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg - end +function xml.collect_tags(root, pattern, nonamespace) + local t = { } + xml.traverse(root, xml.lpath(pattern), function(r,d,k) + local dk = d and d[k] + if dk and type(dk) == "table" then + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg -- if needed we can return an extra table + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg end - end) - return #t > 0 and {} - end + end + end) + return #t > 0 and {} +end - --[[ldx-- - <p>Often using an iterators looks nicer in the code than passing handler - functions. The <l n='lua'/> book describes how to use coroutines for that - purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits - code like:</p> +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> - <typing> - for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) - end - </typing> +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) +end +</typing> - <p>Which will print all the titles in the document. The iterator variant takes - 1.5 times the runtime of the function variant which is due to the overhead in - creating the wrapper. So, instead of:</p> +<p>Which will print all the titles in the document. The iterator variant takes +1.5 times the runtime of the function variant which is due to the overhead in +creating the wrapper. So, instead of:</p> - <typing> - function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk - end - return nil, nil, nil, nil +<typing> +function xml.filters.first(root,pattern) + for rt,dt,dk in xml.elements(root,pattern) + return dt and dt[dk], rt, dt, dk end - </typing> + return nil, nil, nil, nil +end +</typing> - <p>We use the function variants in the filters.</p> - --ldx]]-- +<p>We use the function variants in the filters.</p> +--ldx]]-- - local wrap, yield = coroutine.wrap, coroutine.yield +local wrap, yield = coroutine.wrap, coroutine.yield - function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) - end +function xml.elements(root,pattern,reverse) + return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +end - function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) - end +function xml.elements_only(root,pattern,reverse) + return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) +end - function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok - end +function xml.each_element(root, pattern, handle, reverse) + local ok + traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) + return ok +end - function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end +function xml.process_elements(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then + for i=1,#dkdt do + local v = dkdt[i] + if v.tg then handle(v) end end - end) - end + end + end) +end - function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil - end - end) - end +function xml.process_attributes(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local ek = d[k] + local a = ek.at or { } + handle(a) + if next(a) then -- next is faster than type (and >0 test) + ek.at = a + else + ek.at = nil + end + end) +end - --[[ldx-- - <p>We've now arrives at the functions that manipulate the tree.</p> - --ldx]]-- +--[[ldx-- +<p>We've now arrives at the functions that manipulate the tree.</p> +--ldx]]-- - function xml.inject_element(root, pattern, element, prepend) - if root and element then - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt +function xml.inject_element(root, pattern, element, prepend) + if root and element then + local matches, collect = { }, nil + if type(element) == "string" then + element = convert(element,true) + end + if element then + collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end + traverse(root, lpath(pattern), collect) + for i=1,#matches do + local m = matches[i] + local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil + if element.ri then + element = element.dt[element.ri].dt + else + element = element.dt + end + if r.ri then + edt = r.dt[r.ri].dt + else + edt = d and d[k] and d[k].dt + end + if edt then + local be, af + if prepend then + be, af = xml.copy(element), edt else - element = element.dt + be, af = edt, xml.copy(element) end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt + for i=1,#af do + be[#be+1] = af[i] end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt - else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] - end - if r.ri then - r.dt[r.ri].dt = be - else - d[k].dt = be - end + if r.ri then + r.dt[r.ri].dt = be else - -- r.dt = element.dt -- todo + d[k].dt = be end + else + -- r.dt = element.dt -- todo end end end end +end - -- todo: copy ! +-- todo: copy ! - function xml.insert_element(root, pattern, element, before) -- todo: element als functie - if root and element then - if pattern == "/" then - xml.inject_element(root, pattern, element, before) - else - local matches, collect = { }, nil - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested - elseif element.dt then - for _,v in ipairs(element.dt) do -- i added - insert(d,k,v) +function xml.insert_element(root, pattern, element, before) -- todo: element als functie + if root and element then + if pattern == "/" then + xml.inject_element(root, pattern, element, before) + else + local matches, collect = { }, nil + if type(element) == "string" then + element = convert(element,true) + end + if element and element.ri then + element = element.dt[element.ri] + end + if element then + collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end + traverse(root, lpath(pattern), collect) + for i=#matches,1,-1 do + local m = matches[i] + local r, d, k, element = m[1], m[2], m[3], m[4] + if not before then k = k + 1 end + if element.tg then + insert(d,k,element) -- untested +--~ elseif element.dt then +--~ for _,v in ipairs(element.dt) do -- i added +--~ insert(d,k,v) +--~ k = k + 1 +--~ end +--~ end + else + local edt = element.dt + if edt then + for i=1,#edt do + insert(d,k,edt[i]) k = k + 1 end end @@ -4180,176 +4579,176 @@ do end end end +end - xml.insert_element_after = xml.insert_element - xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end - xml.inject_element_after = xml.inject_element - xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end +xml.insert_element_after = xml.insert_element +xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end +xml.inject_element_after = xml.inject_element +xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end - function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = table.remove(m[2],m[3]) - end - return deleted +function xml.delete_element(root, pattern) + local matches, deleted = { }, { } + local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end + traverse(root, lpath(pattern), collect) + for i=#matches,1,-1 do + local m = matches[i] + deleted[#deleted+1] = remove(m[2],m[3]) end + return deleted +end - function xml.replace_element(root, pattern, element) - if type(element) == "string" then - element = convert(element,true) - end - if element and element.ri then - element = element.dt[element.ri] - end - if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) - end +function xml.replace_element(root, pattern, element) + if type(element) == "string" then + element = convert(element,true) end - - local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() - end - return data + if element and element.ri then + element = element.dt[element.ri] end + if element then + traverse(root, lpath(pattern), function(rm, d, k) + d[k] = element.dt -- maybe not clever enough + end) + end +end - function xml.include(xmldata,pattern,attribute,recursive,loaddata) - -- parse="text" (default: xml), encoding="" (todo) - -- attribute = attribute or 'href' - pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then - local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then - for a in (attribute or "href"):gmatch("([^|]+)") do - name = ek.at[a] - if name then break end - end +local function load_data(name) -- == io.loaddata + local f, data = io.open(name), "" + if f then + data = f:read("*all",'b') -- 'b' ? + f:close() + end + return data +end + +function xml.include(xmldata,pattern,attribute,recursive,loaddata) + -- parse="text" (default: xml), encoding="" (todo) + -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or load_data + local function include(r,d,k) + local ek, name = d[k], nil + if not attribute or attribute == "" then + local ekdt = ek.dt + name = (type(ekdt) == "table" and ekdt[1]) or ekdt + end + if not name then + if ek.at then + for a in gmatch(attribute or "href","([^|]+)") do + name = ek.at[a] + if name then break end end end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then + end + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + xml.empty(d,k) + elseif ek.at["parse"] == "text" then -- for the moment hard coded + d[k] = xml.escaped(data) + else + local xi = xml.convert(data) + if not xi then xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) - else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) - end - xml.assign(d,k,xi) + if recursive then + xml.include(xi,pattern,attribute,recursive,loaddata) end + xml.assign(d,k,xi) end end - xml.each_element(xmldata, pattern, include) end + xml.each_element(xmldata, pattern, include) +end - function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then +function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then -- can be optimized + local t = { } + for i=1,#dkdt do + local str = dkdt[i] + if type(str) == "string" then + if str == "" then + -- stripped + else + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end if str == "" then -- stripped else - if nolines then - str = str:gsub("[ \n\r\t]+"," ") - end - if str == "" then - -- stripped - else - t[#t+1] = str - end + t[#t+1] = str end - else - t[#t+1] = str end + else + t[#t+1] = str end - d[k].dt = t end - end) - end + d[k].dt = t + end + end) +end - function xml.rename_space(root, oldspace, newspace) -- fast variant - local ndt = #root.dt - local rename = xml.rename_space - for i=1,ndt or 0 do - local e = root[i] - if type(e) == "table" then - if e.ns == oldspace then - e.ns = newspace - if e.rn then - e.rn = newspace - end - end - local edt = e.dt - if edt then - rename(edt, oldspace, newspace) +local function rename_space(root, oldspace, newspace) -- fast variant + local ndt = #root.dt + for i=1,ndt or 0 do + local e = root[i] + if type(e) == "table" then + if e.ns == oldspace then + e.ns = newspace + if e.rn then + e.rn = newspace end end + local edt = e.dt + if edt then + rename_space(edt, oldspace, newspace) + end end end +end - function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) - end - function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) - end - function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns - end - end) - end - function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) - end +xml.rename_space = rename_space - function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true - else - found = true - end - return true - end) - return found - end +function xml.remap_tag(root, pattern, newtg) + traverse(root, lpath(pattern), function(r,d,k) + d[k].tg = newtg + end) +end +function xml.remap_namespace(root, pattern, newns) + traverse(root, lpath(pattern), function(r,d,k) + d[k].ns = newns + end) +end +function xml.check_namespace(root, pattern, newns) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + if (not dk.rn or dk.rn == "") and dk.ns == "" then + dk.rn = newns + end + end) +end +function xml.remap_name(root, pattern, newtg, newns, newrn) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + dk.tg = newtg + dk.ns = newns + dk.rn = newrn + end) +end +function xml.filters.found(root,pattern,check_content) + local found = false + traverse(root, lpath(pattern), function(r,d,k) + if check_content then + local dk = d and d[k] + found = dk and dk.dt and next(dk.dt) and true + else + found = true + end + return true + end) + return found end --[[ldx-- @@ -4387,10 +4786,12 @@ put them here instead of loading mode modules there then needed.</p> --ldx]]-- function xml.gsub(t,old,new) - if t.dt then - for k,v in ipairs(t.dt) do + local dt = t.dt + if dt then + for k=1,#dt do + local v = dt[k] if type(v) == "string" then - t.dt[k] = v:gsub(old,new) + dt[k] = gsub(v,old,new) else xml.gsub(v,old,new) end @@ -4415,45 +4816,41 @@ end --~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } --~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end ---~ function xml.escaped (str) return str:gsub("(.)" , xml.escapes ) end ---~ function xml.unescaped(str) return str:gsub("(&.-;)", xml.unescapes) end ---~ function xml.cleansed (str) return str:gsub("<.->" , '' ) end -- "%b<>" +--~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end +--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end +--~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" -do +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - - -- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg - -- - -- 1021:0335:0287:0247 +-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg +-- +-- 1021:0335:0287:0247 - -- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" - -- - -- 1559:0257:0288:0190 (last one suggested by roberto) +-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" +-- +-- 1559:0257:0288:0190 (last one suggested by roberto) - -- escaped = Cs((S("<&>") / xml.escapes + 1)^0) - -- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) - local normal = (1 - S("<&>"))^0 - local special = P("<")/"<" + P(">")/">" + P("&")/"&" - local escaped = Cs(normal * (special * normal)^0) +-- escaped = Cs((S("<&>") / xml.escapes + 1)^0) +-- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +local normal = (1 - S("<&>"))^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local escaped = Cs(normal * (special * normal)^0) - -- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) +-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - -- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) - -- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) - local normal = (1 - S"&")^0 - local special = P("<")/"<" + P(">")/">" + P("&")/"&" - local unescaped = Cs(normal * (special * normal)^0) +-- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +-- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) +local normal = (1 - S"&")^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local unescaped = Cs(normal * (special * normal)^0) - -- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) +-- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) +local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - function xml.escaped (str) return escaped :match(str) end - function xml.unescaped(str) return unescaped:match(str) end - function xml.cleansed (str) return cleansed :match(str) end - -end +function xml.escaped (str) return escaped :match(str) end +function xml.unescaped(str) return unescaped:match(str) end +function xml.cleansed (str) return cleansed :match(str) end function xml.join(t,separator,lastseparator) if #t > 0 then @@ -4471,115 +4868,6 @@ function xml.join(t,separator,lastseparator) end end - ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- - -do if unicode and unicode.utf8 then - - xml.entities = xml.entities or { } -- xml.entity_handler == function - - function xml.entity_handler(e) - return format("[%s]",e) - end - - local char = unicode.utf8.char - - local function toutf(s) - return char(tonumber(s,16)) - end - - function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if dk:find("&#x.-;") then - d[k] = dk:gsub("&#x(.-);",toutf) - end - else - utfize(dk) - end - end - end - - xml.utfize = utfize - - local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if e:find("#x") then - return char(tonumber(e:sub(3),16)) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" - end - end - end - - local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if dk:find("&.-;") then - d[k] = dk:gsub("&(.-);",resolve) - end - else - resolve_entities(dk) - end - end - end - end - - xml.resolve_entities = resolve_entities - - function xml.utfize_text(str) - if str:find("&#") then - return (str:gsub("&#x(.-);",toutf)) - else - return str - end - end - - function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if str:find("&") then - return (str:gsub("&(.-);",resolve)) - else - return str - end - end - - function xml.show_text_entities(str) - if str:find("&") then - return (str:gsub("&(.-);","[%1]")) - else - return str - end - end - - -- experimental, this will be done differently - - function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in pairs(documententities) do - allentities[k] = v - end - end - end - -end end - function xml.statistics() return { lpathcalls = lpathcalls, @@ -4606,7 +4894,7 @@ end --~ </a> --~ ]]) ---~ xml.trace_lpath = true +--~ xml.settrace("lpath",true) --~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) --~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) @@ -4633,157 +4921,490 @@ end --~ print(xml.filter(x,"b/tag(1)")) --- filename : l-utils.lua --- comment : split off from luat-lib --- author : Hans Hagen, PRAGMA-ADE, Hasselt NL --- copyright: PRAGMA ADE / ConTeXt Development Team --- license : see context related readme files +end -- of closure -if not versions then versions = { } end versions['l-utils'] = 1.001 +do -- create closure to overcome 200 locals limit -if not utils then utils = { } end -if not utils.merger then utils.merger = { } end -if not utils.lua then utils.lua = { } end +if not modules then modules = { } end modules ['lxml-ent'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -utils.merger.m_begin = "begin library merge" -utils.merger.m_end = "end library merge" -utils.merger.pattern = - "%c+" .. - "%-%-%s+" .. utils.merger.m_begin .. - "%c+(.-)%c+" .. - "%-%-%s+" .. utils.merger.m_end .. - "%c+" +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub, find = string.format, string.gsub, string.find +local utfchar = unicode.utf8.char -function utils.merger._self_fake_() - return - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. utils.merger.m_begin .. "\n\n" .. - "-- " .. utils.merger.m_end .. "\n\n" +--[[ldx-- +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> +--ldx]]-- + +xml.entities = xml.entities or { } -- xml.entity_handler == function + +function xml.entity_handler(e) + return format("[%s]",e) end -function utils.report(...) - print(...) +local function toutf(s) + return utfchar(tonumber(s,16)) end -utils.merger.strip_comment = true +local function utfize(root) + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + -- test prevents copying if no match + if find(dk,"&#x.-;") then + d[k] = gsub(dk,"&#x(.-);",toutf) + end + else + utfize(dk) + end + end +end -function utils.merger._self_load_(name) - local f, data = io.open(name), "" - if f then - utils.report("reading merge from %s",name) - data = f:read("*all") - f:close() +xml.utfize = utfize + +local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks + if find(e,"^#x") then + return utfchar(tonumber(e:sub(3),16)) + elseif find(e,"^#") then + return utfchar(tonumber(e:sub(2))) else - utils.report("unknown file to merge %s",name) + local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) + if ee then + return ee + else + local h = xml.entity_handler + return (h and h(e)) or "&" .. e .. ";" + end end - if data and utils.merger.strip_comment then - -- saves some 20K - data = data:gsub("%-%-~[^\n\r]*[\r\n]", "") +end + +local function resolve_entities(root) + if not root.special or root.tg == "@rt@" then + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + if find(dk,"&.-;") then + d[k] = gsub(dk,"&(.-);",resolve) + end + else + resolve_entities(dk) + end + end end - return data or "" end -function utils.merger._self_save_(name, data) - if data ~= "" then - local f = io.open(name,'w') - if f then - utils.report("saving merge from %s",name) - f:write(data) - f:close() +xml.resolve_entities = resolve_entities + +function xml.utfize_text(str) + if find(str,"&#") then + return (gsub(str,"&#x(.-);",toutf)) + else + return str + end +end + +function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline + if find(str,"&") then + return (gsub(str,"&(.-);",resolve)) + else + return str + end +end + +function xml.show_text_entities(str) + if find(str,"&") then + return (gsub(str,"&(.-);","[%1]")) + else + return str + end +end + +-- experimental, this will be done differently + +function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in next, documententities do + allentities[k] = v end end end -function utils.merger._self_swap_(data,code) - if data ~= "" then - return (data:gsub(utils.merger.pattern, function(s) - return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" - end, 1)) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub = string.format, string.gsub + +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- + +function xml.gsub(t,old,new) + local dt = t.dt + if dt then + for k=1,#dt do + local v = dt[k] + if type(v) == "string" then + dt[k] = gsub(v,old,new) + else + xml.gsub(v,old,new) + end + end + end +end + +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k and d[k-1] and type(d[k-1]) == "string" then + local s = d[k-1]:match("\n(%s+)") + xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + end +end + +function xml.serialize_path(root,lpath,handle) + local dk, r, d, k = xml.first(root,lpath) + dk = xml.copy(dk) + xml.strip_leading_spaces(dk,d,k) + xml.serialize(dk,handle) +end + +--~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } +--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end + +--~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end +--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end +--~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" + +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs + +-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg +-- +-- 1021:0335:0287:0247 + +-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" +-- +-- 1559:0257:0288:0190 (last one suggested by roberto) + +-- escaped = Cs((S("<&>") / xml.escapes + 1)^0) +-- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +local normal = (1 - S("<&>"))^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local escaped = Cs(normal * (special * normal)^0) + +-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) + +-- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +-- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) +local normal = (1 - S"&")^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local unescaped = Cs(normal * (special * normal)^0) + +-- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) + +local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) + +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed + +function xml.escaped (str) return escaped :match(str) end +function xml.unescaped(str) return unescaped:match(str) end +function xml.cleansed (str) return cleansed :match(str) end + +function xml.join(t,separator,lastseparator) + if #t > 0 then + local result = { } + for k,v in pairs(t) do + result[k] = xml.tostring(v) + end + if lastseparator then + return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] + else + return concat(result,separator) + end else return "" end end -function utils.merger._self_libs_(libs,list) - local result, f = { }, nil - if type(libs) == 'string' then libs = { libs } end - if type(list) == 'string' then list = { list } end - for _, lib in ipairs(libs) do - for _, pth in ipairs(list) do - local name = string.gsub(pth .. "/" .. lib,"\\","/") - f = io.open(name) - if f then - utils.report("merging library %s",name) - result[#result+1] = f:read("*all") - f:close() - list = { pth } -- speed up the search - break - else - utils.report("no library %s",name) + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +debugger = debugger or { } + +local counters = { } +local names = { } +local getinfo = debug.getinfo +local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in pairs(counters) do + if count > threshold then + local name = getname(func) + if not name:find("for generator") then + printer(format("%8i %s", count, name)) + total = total + count end end + grandtotal = grandtotal + count + functions = functions + 1 end - return table.concat(result, "\n\n") + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) end -function utils.merger.selfcreate(libs,list,target) - if target then - utils.merger._self_save_( - target, - utils.merger._self_swap_( - utils.merger._self_fake_(), - utils.merger._self_libs_(libs,list) - ) - ) +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in pairs(counters) do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() end end -function utils.merger.selfmerge(name,libs,list,target) - utils.merger._self_save_( - target or name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - utils.merger._self_libs_(libs,list) - ) - ) +function debugger.enable() + debug.sethook(hook,"c") end -function utils.merger.selfclean(name) - utils.merger._self_save_( - name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - "" - ) - ) +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil end -function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true - -- utils.report("compiling",luafile,"into",lucfile) - os.remove(lucfile) - local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) - if strip ~= false then - command = "-s " .. command +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false end - local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - -- utils.report("removing",luafile) - os.remove(luafile) +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +trackers = trackers or { } + +local data, done = { }, { } + +local function set(what,value) + for w in gmatch(lower(what),"[^, ]+") do + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](value) + end + end + end + end +end + +local function reset() + for d, f in next, data do + for i=1,#f do + f[i](false) + end end - return done end +function trackers.register(what,...) + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(fnc,value,nesting) end + end + end +end + +function trackers.enable(what) + done = { } + set(what,true) +end + +function trackers.disable(what) + done = { } + if not what or what == "" then + trackers.reset(what) + else + set(what,false) + end +end + +function trackers.reset(what) + done = { } + reset() +end + +function trackers.list() -- pattern + local list = table.sortedkeys(data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + + +end -- of closure +do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-lib'] = { +if not modules then modules = { } end modules ['luat-env'] = { version = 1.001, + comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", - comment = "companion to luat-lib.tex", + license = "see context related readme files" } --- most code already moved to the l-*.lua and other luat-*.lua files +-- A former version provided functionality for non embeded core +-- scripts i.e. runtime library loading. Given the amount of +-- Lua code we use now, this no longer makes sense. Much of this +-- evolved before bytecode arrays were available and so a lot of +-- code has disappeared already. + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + +local format = string.format + +-- precautions os.setlocale(nil,nil) -- useless feature and even dangerous in luatex @@ -4791,15 +5412,27 @@ function os.setlocale() -- no way you can mess with it end +-- dirty tricks + if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil end +if profiler and os.env["MTX_PROFILE_RUN"] == "YES" then + profiler.start("luatex-profile.log") +end + +-- environment + environment = environment or { } environment.arguments = { } environment.files = { } environment.sortedflags = nil +if not environment.jobname or environment.jobname == "" then if tex then environment.jobname = tex.jobname end end +if not environment.version or environment.version == "" then environment.version = "unknown" end +if not environment.jobname then environment.jobname = "unknown" end + function environment.initialize_arguments(arg) local arguments, files = { }, { } environment.arguments, environment.files, environment.sortedflags = arguments, files, nil @@ -4821,24 +5454,20 @@ function environment.initialize_arguments(arg) environment.ownname = environment.ownname or arg[0] or 'unknown.lua' end -function environment.showarguments() - for k,v in pairs(environment.arguments) do - print(k .. " : " .. tostring(v)) - end - if #environment.files > 0 then - print("files : " .. table.concat(environment.files, " ")) - end -end - function environment.setargument(name,value) environment.arguments[name] = value end -function environment.argument(name) -- todo: default (plus typecheck on default) +-- todo: defaults, better checks e.g on type (boolean versus string) +-- +-- tricky: too many hits when we support partials unless we add +-- a registration of arguments so from now on we have 'partial' + +function environment.argument(name,partial) local arguments, sortedflags = environment.arguments, environment.sortedflags if arguments[name] then return arguments[name] - else + elseif partial then if not sortedflags then sortedflags = { } for _,v in pairs(table.sortedkeys(arguments)) do @@ -4846,6 +5475,7 @@ function environment.argument(name) -- todo: default (plus typecheck on default) end environment.sortedflags = sortedflags end + -- example of potential clash: ^mode ^modefile for _,v in ipairs(sortedflags) do if name:find(v) then return arguments[v:sub(2,#v)] @@ -4869,43 +5499,17 @@ function environment.split_arguments(separator) -- rather special, cut-off befor return before, after end ---~ function environment.reconstruct_commandline(arg) ---~ if not arg then arg = environment.original_arguments end ---~ local result = { } ---~ for _,a in ipairs(arg) do -- ipairs 1 .. #n ---~ local kk, vv = a:match("^(%-+.-)=(.+)$") ---~ if kk and vv then ---~ if vv:find(" ") then ---~ vv = vv:unquote() ---~ vv = vv:gsub('"','\\"') ---~ result[#result+1] = kk .. "=" .. vv:quote() ---~ else ---~ a = a:unquote() ---~ a = a:gsub('"','\\"') ---~ result[#result+1] = a ---~ end ---~ elseif a:find(" ") then ---~ a = a:unquote() ---~ a = a:gsub('"','\\"') ---~ result[#result+1] = a:quote() ---~ else ---~ result[#result+1] = a ---~ end ---~ end ---~ return table.join(result," ") ---~ end - function environment.reconstruct_commandline(arg,noquote) - if not arg then arg = environment.original_arguments end + arg = arg or environment.original_arguments if noquote and #arg == 1 then local a = arg[1] - a = input.resolve(a) + a = resolvers.resolve(a) a = a:unquote() return a - elseif #arg == 1 then + elseif next(arg) then local result = { } for _,a in ipairs(arg) do -- ipairs 1 .. #n - a = input.resolve(a) + a = resolvers.resolve(a) a = a:unquote() a = a:gsub('"','\\"') -- tricky if a:find(" ") then @@ -4915,6 +5519,8 @@ function environment.reconstruct_commandline(arg,noquote) end end return table.join(result," ") + else + return "" end end @@ -4950,8 +5556,563 @@ if arg then end +-- weird place ... depends on a not yet loaded module + +function environment.texfile(filename) + return resolvers.find_file(filename,'tex') +end + +function environment.luafile(filename) + local resolved = resolvers.find_file(filename,'tex') or "" + if resolved ~= "" then + return resolved + end + resolved = resolvers.find_file(filename,'texmfscripts') or "" + if resolved ~= "" then + return resolved + end + return resolvers.find_file(filename,'luatexlibs') or "" +end + +environment.loadedluacode = loadfile -- can be overloaded + +--~ function environment.loadedluacode(name) +--~ if os.spawn("texluac -s -o texluac.luc " .. name) == 0 then +--~ local chunk = loadstring(io.loaddata("texluac.luc")) +--~ os.remove("texluac.luc") +--~ return chunk +--~ else +--~ environment.loadedluacode = loadfile -- can be overloaded +--~ return loadfile(name) +--~ end +--~ end + +function environment.luafilechunk(filename) -- used for loading lua bytecode in the format + filename = file.replacesuffix(filename, "lua") + local fullname = environment.luafile(filename) + if fullname and fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading file %s", fullname) + end + return environment.loadedluacode(fullname) + else + if trace_verbose then + logs.report("fileio","unknown file %s", filename) + end + return nil + end +end + +-- the next ones can use the previous ones / combine + +function environment.loadluafile(filename, version) + local lucname, luaname, chunk + local basename = file.removesuffix(filename) + if basename == filename then + lucname, luaname = basename .. ".luc", basename .. ".lua" + else + lucname, luaname = nil, basename -- forced suffix + end + -- when not overloaded by explicit suffix we look for a luc file first + local fullname = (lucname and environment.luafile(lucname)) or "" + if fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + end + if chunk then + assert(chunk)() + if version then + -- we check of the version number of this chunk matches + local v = version -- can be nil + if modules and modules[filename] then + v = modules[filename].version -- new method + elseif versions and versions[filename] then + v = versions[filename] -- old method + end + if v == version then + return true + else + if trace_verbose then + logs.report("fileio","version mismatch for %s: lua=%s, luc=%s", filename, v, version) + end + environment.loadluafile(filename) + end + else + return true + end + end + fullname = (luaname and environment.luafile(luaname)) or "" + if fullname ~= "" then + if trace_verbose then + logs.report("fileio","loading %s", fullname) + end + chunk = loadfile(fullname) -- this way we don't need a file exists check + if not chunk then + if verbose then + logs.report("fileio","unknown file %s", filename) + end + else + assert(chunk)() + return true + end + end + return false +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-inf'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format = string.format + +local statusinfo, n, registered = { }, 0, { } + +statistics = statistics or { } + +statistics.enable = true +statistics.threshold = 0.05 + +-- timing functions + +local clock = os.gettimeofday or os.clock + +function statistics.hastimer(instance) + return instance and instance.starttime +end + +function statistics.starttiming(instance) + if instance then + local it = instance.timing + if not it then + it = 0 + end + if it == 0 then + instance.starttime = clock() + if not instance.loadtime then + instance.loadtime = 0 + end + end + instance.timing = it + 1 + end +end + +function statistics.stoptiming(instance, report) + if instance then + local it = instance.timing + if it > 1 then + instance.timing = it - 1 + else + local starttime = instance.starttime + if starttime then + local stoptime = clock() + local loadtime = stoptime - starttime + instance.stoptime = stoptime + instance.loadtime = instance.loadtime + loadtime + if report then + statistics.report("load time %0.3f",loadtime) + end + instance.timing = 0 + return loadtime + end + end + end + return 0 +end + +function statistics.elapsedtime(instance) + return format("%0.3f",(instance and instance.loadtime) or 0) +end + +function statistics.elapsedindeed(instance) + local t = (instance and instance.loadtime) or 0 + return t > statistics.threshold +end + +-- general function + +function statistics.register(tag,fnc) + if statistics.enable and type(fnc) == "function" then + local rt = registered[tag] or (#statusinfo + 1) + statusinfo[rt] = { tag, fnc } + registered[tag] = rt + if #tag > n then n = #tag end + end +end + +function statistics.show(reporter) + if statistics.enable then + if not reporter then reporter = function(tag,data,n) texio.write_nl(tag .. " " .. data) end end + -- this code will move + local register = statistics.register + register("luatex banner", function() + return string.lower(status.banner) + end) + register("control sequences", function() + return format("%s of %s", status.cs_count, status.hash_size+status.hash_extra) + end) + register("callbacks", function() + local total, indirect = status.callbacks or 0, status.indirect_callbacks or 0 + return format("direct: %s, indirect: %s, total: %s", total-indirect, indirect, total) + end) + register("current memory usage", statistics.memused) + register("runtime",statistics.runtime) +-- -- + for i=1,#statusinfo do + local s = statusinfo[i] + local r = s[2]() + if r then + reporter(s[1],r,n) + end + end + statistics.enable = false + end +end + +function statistics.show_job_stat(tag,data,n) + texio.write_nl(format("%-15s: %s - %s","mkiv lua stats",tag:rpadd(n," "),data)) +end + +function statistics.memused() -- no math.round yet -) + local round = math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000), round(status.luastate_bytes/1000000)) +end + +if statistics.runtime then + -- already loaded and set +elseif luatex and luatex.starttime then + statistics.starttime = luatex.starttime + statistics.loadtime = 0 + statistics.timing = 0 +else + statistics.starttiming(statistics) +end + +function statistics.runtime() + statistics.stoptiming(statistics) + return statistics.formatruntime(statistics.elapsedtime(statistics)) +end + +function statistics.formatruntime(runtime) + return format("%s seconds", statistics.elapsedtime(statistics)) +end + +function statistics.timed(action,report) + local timer = { } + report = report or logs.simple + statistics.starttiming(timer) + action() + statistics.stoptiming(timer) + report("total runtime: %s",statistics.elapsedtime(timer)) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-log'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- this is old code that needs an overhaul + +local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format + +if texlua then + write_nl = print + write = io.write +end + +--[[ldx-- +<p>This is a prelude to a more extensive logging module. For the sake +of parsing log files, in addition to the standard logging we will +provide an <l n='xml'/> structured file. Actually, any logging that +is hooked into callbacks will be \XML\ by default.</p> +--ldx]]-- + +logs = logs or { } +logs.xml = logs.xml or { } +logs.tex = logs.tex or { } + +--[[ldx-- +<p>This looks pretty ugly but we need to speed things up a bit.</p> +--ldx]]-- + +logs.moreinfo = [[ +more information about ConTeXt and the tools that come with it can be found at: + +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] + +logs.levels = { + ['error'] = 1, + ['warning'] = 2, + ['info'] = 3, + ['debug'] = 4, +} + +logs.functions = { + 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct', + 'start_run', 'stop_run', + 'start_page_number', 'stop_page_number', + 'report_output_pages', 'report_output_log', + 'report_tex_stat', 'report_job_stat', + 'show_open', 'show_close', 'show_load', +} + +logs.tracers = { +} + +logs.level = 0 +logs.mode = string.lower((os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex")) + +function logs.set_level(level) + logs.level = logs.levels[level] or level +end + +function logs.set_method(method) + for _, v in next, logs.functions do + logs[v] = logs[method][v] or function() end + end +end + +-- tex logging + +function logs.tex.report(category,fmt,...) -- new + if fmt then + write_nl(category .. " | " .. format(fmt,...)) + else + write_nl(category .. " |") + end +end + +function logs.tex.line(fmt,...) -- new + if fmt then + write_nl(format(fmt,...)) + else + write_nl("") + end +end + +local texcount = tex and tex.count + +function logs.tex.start_page_number() + local real, user, sub = texcount[0], texcount[1], texcount[2] + if real > 0 then + if user > 0 then + if sub > 0 then + write(format("[%s.%s.%s",real,user,sub)) + else + write(format("[%s.%s",real,user)) + end + else + write(format("[%s",real)) + end + else + write("[-") + end +end + +function logs.tex.stop_page_number() + write("]") +end + +logs.tex.report_job_stat = statistics.show_job_stat + +-- xml logging + +function logs.xml.report(category,fmt,...) -- new + if fmt then + write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) + else + write_nl(format("<r category='%s'/>",category)) + end +end +function logs.xml.line(fmt,...) -- new + if fmt then + write_nl(format("<r>%s</r>",format(fmt,...))) + else + write_nl("<r/>") + end +end + +function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end +function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end +function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end +function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end + +function logs.xml.start_run() + write_nl("<?xml version='1.0' standalone='yes'?>") + write_nl("<job>") -- xmlns='www.pragma-ade.com/luatex/schemas/context-job.rng' + write_nl("") +end + +function logs.xml.stop_run() + write_nl("</job>") +end + +function logs.xml.start_page_number() + write_nl(format("<p real='%s' page='%s' sub='%s'", texcount[0], texcount[1], texcount[2])) +end + +function logs.xml.stop_page_number() + write("/>") + write_nl("") +end + +function logs.xml.report_output_pages(p,b) + write_nl(format("<v k='pages' v='%s'/>", p)) + write_nl(format("<v k='bytes' v='%s'/>", b)) + write_nl("") +end + +function logs.xml.report_output_log() +end + +function logs.xml.report_tex_stat(k,v) + texiowrite_nl("log","<v k='"..k.."'>"..tostring(v).."</v>") +end + +local level = 0 + +function logs.xml.show_open(name) + level = level + 1 + texiowrite_nl(format("<f l='%s' n='%s'>",level,name)) +end + +function logs.xml.show_close(name) + texiowrite("</f> ") + level = level - 1 +end + +function logs.xml.show_load(name) + texiowrite_nl(format("<f l='%s' n='%s'/>",level+1,name)) +end + +-- + +local name, banner = 'report', 'context' + +local function report(category,fmt,...) + if fmt then + write_nl(format("%s | %s: %s",name,category,format(fmt,...))) + elseif category then + write_nl(format("%s | %s",name,category)) + else + write_nl(format("%s |",name)) + end +end + +local function simple(fmt,...) + if fmt then + write_nl(format("%s | %s",name,format(fmt,...))) + else + write_nl(format("%s |",name)) + end +end + +function logs.setprogram(_name_,_banner_,_verbose_) + name, banner = _name_, _banner_ + if _verbose_ then + trackers.enable("resolvers.verbose") + end + logs.set_method("tex") + logs.report = report -- also used in libraries + logs.simple = simple -- only used in scripts ! + if utils then + utils.report = simple + end + logs.verbose = _verbose_ +end + +function logs.setverbose(what) + if what then + trackers.enable("resolvers.verbose") + else + trackers.disable("resolvers.verbose") + end + logs.verbose = what or false +end + +function logs.extendbanner(_banner_,_verbose_) + banner = banner .. " | ".. _banner_ + if _verbose_ ~= nil then + logs.setverbose(what) + end +end + +logs.verbose = false +logs.report = logs.tex.report +logs.simple = logs.tex.report + +function logs.reportlines(str) -- todo: <lines></lines> + for line in str:gmatch("(.-)[\n\r]") do + logs.report(line) + end +end + +function logs.reportline() -- for scripts too + logs.report() +end -if not modules then modules = { } end modules ['luat-inp'] = { +logs.simpleline = logs.reportline + +function logs.help(message,option) + logs.report(banner) + logs.reportline() + logs.reportlines(message) + local moreinfo = logs.moreinfo or "" + if moreinfo ~= "" and option ~= "nomoreinfo" then + logs.reportline() + logs.reportlines(moreinfo) + end +end + +logs.set_level('error') +logs.set_method('tex') + +function logs.system(whereto,process,jobname,category,...) + for i=1,10 do + local f = io.open(whereto,"a") + if f then + f:write(format("%s %s => %s => %s => %s\r",os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...))) + f:close() + break + else + sleep(0.1) + end + end +end + +--~ local syslogname = "oeps.xxx" +--~ +--~ for i=1,10 do +--~ logs.system(syslogname,"context","test","fonts","font %s recached due to newer version (%s)","blabla","123") +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { version = 1.001, author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", @@ -4959,12 +6120,20 @@ if not modules then modules = { } end modules ['luat-inp'] = { comment = "companion to luat-lib.tex", } +-- After a few years using the code the large luat-inp.lua file +-- has been split up a bit. In the process some functionality was +-- dropped: +-- +-- * support for reading lsr files +-- * selective scanning (subtrees) +-- * some public auxiliary functions were made private +-- -- TODO: os.getenv -> os.env[] -- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) -- TODO: check escaping in find etc, too much, too slow -- This lib is multi-purpose and can be loaded again later on so that --- additional functionality becomes available. We will split this +-- additional functionality becomes available. We will split thislogs.report("fileio", -- module in components once we're done with prototyping. This is the -- first code I wrote for LuaTeX, so it needs some cleanup. Before changing -- something in this module one can best check with Taco or Hans first; there @@ -4978,82 +6147,205 @@ if not modules then modules = { } end modules ['luat-inp'] = { -- Beware, loading and saving is overloaded in luat-tmp! -if not input then input = { } end -if not input.suffixes then input.suffixes = { } end -if not input.formats then input.formats = { } end -if not input.aux then input.aux = { } end - -if not input.suffixmap then input.suffixmap = { } end - -if not input.locators then input.locators = { } end -- locate databases -if not input.hashers then input.hashers = { } end -- load databases -if not input.generators then input.generators = { } end -- generate databases -if not input.filters then input.filters = { } end -- conversion filters - -local format, concat, sortedkeys = string.format, table.concat, table.sortedkeys - -input.locators.notfound = { nil } -input.hashers.notfound = { nil } -input.generators.notfound = { nil } - -input.cacheversion = '1.0.1' -input.banner = nil -input.verbose = false -input.debug = false -input.cnfname = 'texmf.cnf' -input.luaname = 'texmfcnf.lua' -input.lsrname = 'ls-R' -input.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' - ---~ input.luasuffix = 'tma' ---~ input.lucsuffix = 'tmc' - --- for the moment we have .local but this will disappear -input.cnfdefault = '{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - --- chances are low that the cnf file is in the bin path -input.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' - --- we use a cleaned up list / format=any is a wildcard, as is *name - -input.formats['afm'] = 'AFMFONTS' input.suffixes['afm'] = { 'afm' } -input.formats['enc'] = 'ENCFONTS' input.suffixes['enc'] = { 'enc' } -input.formats['fmt'] = 'TEXFORMATS' input.suffixes['fmt'] = { 'fmt' } -input.formats['map'] = 'TEXFONTMAPS' input.suffixes['map'] = { 'map' } -input.formats['mp'] = 'MPINPUTS' input.suffixes['mp'] = { 'mp' } -input.formats['ocp'] = 'OCPINPUTS' input.suffixes['ocp'] = { 'ocp' } -input.formats['ofm'] = 'OFMFONTS' input.suffixes['ofm'] = { 'ofm', 'tfm' } -input.formats['otf'] = 'OPENTYPEFONTS' input.suffixes['otf'] = { 'otf' } -- 'ttf' -input.formats['opl'] = 'OPLFONTS' input.suffixes['opl'] = { 'opl' } -input.formats['otp'] = 'OTPINPUTS' input.suffixes['otp'] = { 'otp' } -input.formats['ovf'] = 'OVFFONTS' input.suffixes['ovf'] = { 'ovf', 'vf' } -input.formats['ovp'] = 'OVPFONTS' input.suffixes['ovp'] = { 'ovp' } -input.formats['tex'] = 'TEXINPUTS' input.suffixes['tex'] = { 'tex' } -input.formats['tfm'] = 'TFMFONTS' input.suffixes['tfm'] = { 'tfm' } -input.formats['ttf'] = 'TTFONTS' input.suffixes['ttf'] = { 'ttf', 'ttc' } -input.formats['pfb'] = 'T1FONTS' input.suffixes['pfb'] = { 'pfb', 'pfa' } -input.formats['vf'] = 'VFFONTS' input.suffixes['vf'] = { 'vf' } - -input.formats['fea'] = 'FONTFEATURES' input.suffixes['fea'] = { 'fea' } -input.formats['cid'] = 'FONTCIDMAPS' input.suffixes['cid'] = { 'cid', 'cidmap' } - -input.formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new -input.suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' - -input.formats ['lua'] = 'LUAINPUTS' -- new -input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } +local format, gsub, find, lower, upper, match, gmatch = string.format, string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch +local concat, insert, sortedkeys = table.concat, table.insert, table.sortedkeys +local next, type = next, type + +local trace_locating, trace_detail, trace_verbose = false, false, false + +trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) +trackers.register("resolvers.detail", function(v) trace_detail = v trackers.enable("resolvers.verbose,resolvers.detail") end) + +if not resolvers then + resolvers = { + suffixes = { }, + formats = { }, + dangerous = { }, + suffixmap = { }, + alternatives = { }, + locators = { }, -- locate databases + hashers = { }, -- load databases + generators = { }, -- generate databases + } +end + +local resolvers = resolvers + +resolvers.locators .notfound = { nil } +resolvers.hashers .notfound = { nil } +resolvers.generators.notfound = { nil } + +resolvers.cacheversion = '1.0.1' +resolvers.cnfname = 'texmf.cnf' +resolvers.luaname = 'texmfcnf.lua' +resolvers.homedir = os.env[os.platform == "windows" and 'USERPROFILE'] or os.env['HOME'] or '~' +resolvers.cnfdefault = '{$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c}' + +local dummy_path_expr = "^!*unset/*$" + +local formats = resolvers.formats +local suffixes = resolvers.suffixes +local dangerous = resolvers.dangerous +local suffixmap = resolvers.suffixmap +local alternatives = resolvers.alternatives + +formats['afm'] = 'AFMFONTS' suffixes['afm'] = { 'afm' } +formats['enc'] = 'ENCFONTS' suffixes['enc'] = { 'enc' } +formats['fmt'] = 'TEXFORMATS' suffixes['fmt'] = { 'fmt' } +formats['map'] = 'TEXFONTMAPS' suffixes['map'] = { 'map' } +formats['mp'] = 'MPINPUTS' suffixes['mp'] = { 'mp' } +formats['ocp'] = 'OCPINPUTS' suffixes['ocp'] = { 'ocp' } +formats['ofm'] = 'OFMFONTS' suffixes['ofm'] = { 'ofm', 'tfm' } +formats['otf'] = 'OPENTYPEFONTS' suffixes['otf'] = { 'otf' } -- 'ttf' +formats['opl'] = 'OPLFONTS' suffixes['opl'] = { 'opl' } +formats['otp'] = 'OTPINPUTS' suffixes['otp'] = { 'otp' } +formats['ovf'] = 'OVFFONTS' suffixes['ovf'] = { 'ovf', 'vf' } +formats['ovp'] = 'OVPFONTS' suffixes['ovp'] = { 'ovp' } +formats['tex'] = 'TEXINPUTS' suffixes['tex'] = { 'tex' } +formats['tfm'] = 'TFMFONTS' suffixes['tfm'] = { 'tfm' } +formats['ttf'] = 'TTFONTS' suffixes['ttf'] = { 'ttf', 'ttc' } +formats['pfb'] = 'T1FONTS' suffixes['pfb'] = { 'pfb', 'pfa' } +formats['vf'] = 'VFFONTS' suffixes['vf'] = { 'vf' } + +formats['fea'] = 'FONTFEATURES' suffixes['fea'] = { 'fea' } +formats['cid'] = 'FONTCIDMAPS' suffixes['cid'] = { 'cid', 'cidmap' } + +formats ['texmfscripts'] = 'TEXMFSCRIPTS' -- new +suffixes['texmfscripts'] = { 'rb', 'pl', 'py' } -- 'lua' + +formats ['lua'] = 'LUAINPUTS' -- new +suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } + +-- backward compatible ones + +alternatives['map files'] = 'map' +alternatives['enc files'] = 'enc' +alternatives['cid files'] = 'cid' +alternatives['fea files'] = 'fea' +alternatives['opentype fonts'] = 'otf' +alternatives['truetype fonts'] = 'ttf' +alternatives['truetype collections'] = 'ttc' +alternatives['type1 fonts'] = 'pfb' + +-- obscure ones + +formats ['misc fonts'] = '' +suffixes['misc fonts'] = { } + +formats ['sfd'] = 'SFDFONTS' +suffixes ['sfd'] = { 'sfd' } +alternatives['subfont definition files'] = 'sfd' + +-- In practice we will work within one tds tree, but i want to keep +-- the option open to build tools that look at multiple trees, which is +-- why we keep the tree specific data in a table. We used to pass the +-- instance but for practical pusposes we now avoid this and use a +-- instance variable. -- here we catch a few new thingies (todo: add these paths to context.tmf) -- -- FONTFEATURES = .;$TEXMF/fonts/fea// -- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata() -- not yet ok, no time for debugging now - local instance = input.instance +-- we always have one instance active + +resolvers.instance = resolvers.instance or nil -- the current one (slow access) +local instance = resolvers.instance or nil -- the current one (fast access) + +function resolvers.newinstance() + + -- store once, freeze and faster (once reset we can best use + -- instance.environment) maybe better have a register suffix + -- function + + for k, v in next, suffixes do + for i=1,#v do + local vi = v[i] + if vi then + suffixmap[vi] = k + end + end + end + + -- because vf searching is somewhat dangerous, we want to prevent + -- too liberal searching esp because we do a lookup on the current + -- path anyway; only tex (or any) is safe + + for k, v in next, formats do + dangerous[k] = true + end + dangerous.tex = nil + + -- the instance + + local newinstance = { + rootpath = '', + treepath = '', + progname = 'context', + engine = 'luatex', + format = '', + environment = { }, + variables = { }, + expansions = { }, + files = { }, + remap = { }, + configuration = { }, + setup = { }, + order = { }, + found = { }, + foundintrees = { }, + kpsevars = { }, + hashes = { }, + cnffiles = { }, + luafiles = { }, + lists = { }, + remember = true, + diskcache = true, + renewcache = false, + scandisk = true, + cachepath = nil, + loaderror = false, + sortdata = false, + savelists = true, + cleanuppaths = true, + allresults = false, + pattern = nil, -- lists + data = { }, -- only for loading + force_suffixes = true, + fakepaths = { }, + } + + local ne = newinstance.environment + + for k,v in next, os.env do + ne[k] = resolvers.bare_variable(v) + end + + return newinstance + +end + +function resolvers.setinstance(someinstance) + instance = someinstance + resolvers.instance = someinstance + return someinstance +end + +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) +end + +local function reset_hashes() + instance.lists = { } + instance.found = { } +end + +local function check_configuration() -- not yet ok, no time for debugging now + local ie = instance.environment local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" - local p = instance.environment[proname] - local v = instance.environment[varname] + local p, v = ie[proname], ie[varname] if not ((p and p ~= "") or (v and v ~= "")) then instance.variables[varname] = default -- or environment? end @@ -5069,207 +6361,198 @@ function input.checkconfigdata() -- not yet ok, no time for debugging now fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") -- no progname, hm fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("LUATEXLIBS" , ".;$TEXMF/luatex/lua//") end --- backward compatible ones - -input.alternatives = { } - -input.alternatives['map files'] = 'map' -input.alternatives['enc files'] = 'enc' -input.alternatives['cid files'] = 'cid' -input.alternatives['fea files'] = 'fea' -input.alternatives['opentype fonts'] = 'otf' -input.alternatives['truetype fonts'] = 'ttf' -input.alternatives['truetype collections'] = 'ttc' -input.alternatives['type1 fonts'] = 'pfb' - --- obscure ones - -input.formats ['misc fonts'] = '' -input.suffixes['misc fonts'] = { } - -input.formats ['sfd'] = 'SFDFONTS' -input.suffixes ['sfd'] = { 'sfd' } -input.alternatives['subfont definition files'] = 'sfd' - --- In practice we will work within one tds tree, but i want to keep --- the option open to build tools that look at multiple trees, which is --- why we keep the tree specific data in a table. We used to pass the --- instance but for practical pusposes we now avoid this and use a --- instance variable. - -function input.newinstance() - - local instance = { } - - instance.rootpath = '' - instance.treepath = '' - instance.progname = 'context' - instance.engine = 'luatex' - instance.format = '' - instance.environment = { } - instance.variables = { } - instance.expansions = { } - instance.files = { } - instance.remap = { } - instance.configuration = { } - instance.setup = { } - instance.order = { } - instance.found = { } - instance.foundintrees = { } - instance.kpsevars = { } - instance.hashes = { } - instance.cnffiles = { } - instance.luafiles = { } - instance.lists = { } - instance.remember = true - instance.diskcache = true - instance.renewcache = false - instance.scandisk = true - instance.cachepath = nil - instance.loaderror = false - instance.smallcache = false - instance.sortdata = false - instance.savelists = true - instance.cleanuppaths = true - instance.allresults = false - instance.pattern = nil -- lists - instance.kpseonly = false -- lists - instance.loadtime = 0 - instance.starttime = 0 - instance.stoptime = 0 - instance.validfile = function(path,name) return true end - instance.data = { } -- only for loading - instance.force_suffixes = true - instance.dummy_path_expr = "^!*unset/*$" - instance.fakepaths = { } - instance.lsrmode = false - - -- store once, freeze and faster (once reset we can best use instance.environment) +function resolvers.bare_variable(str) -- assumes str is a string + return (gsub(str,"\s*([\"\']?)(.+)%1\s*", "%2")) +end - for k,v in pairs(os.env) do - instance.environment[k] = input.bare_variable(v) +function resolvers.settrace(n) -- no longer number but: 'locating' or 'detail' + if n then + trackers.disable("resolvers.*") + trackers.enable("resolvers."..n) end +end - -- cross referencing, delayed because we can add suffixes +resolvers.settrace(os.getenv("MTX.resolvers.TRACE") or os.getenv("MTX_INPUT_TRACE")) - for k, v in pairs(input.suffixes) do - for _, vv in pairs(v) do - if vv then - input.suffixmap[vv] = k - end +function resolvers.osenv(key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] + if e == nil then + -- value = "" -- false + else + value = resolvers.bare_variable(e) end + ie[key] = value end - - return instance - + return value or "" end -input.instance = input.instance or nil - -function input.reset() - input.instance = input.newinstance() - return input.instance +function resolvers.env(key) + return instance.environment[key] or resolvers.osenv(key) end -function input.reset_hashes() - input.instance.lists = { } - input.instance.found = { } -end +-- -function input.bare_variable(str) -- assumes str is a string - -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") - return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) +local function expand_vars(lst) -- simple vars + local variables, env = instance.variables, resolvers.env + local function resolve(a) + return variables[a] or env(a) + end + for k=1,#lst do + lst[k] = gsub(lst[k],"%$([%a%d%_%-]+)",resolve) + end end -function input.settrace(n) - input.trace = tonumber(n or 0) - if input.trace > 0 then - input.verbose = true +local function expanded_var(var) -- simple vars + local function resolve(a) + return instance.variables[a] or resolvers.env(a) end + return (gsub(var,"%$([%a%d%_%-]+)",resolve)) end -input.log = (texio and texio.write_nl) or print - -function input.report(...) - if input.verbose then - input.log("<<"..format(...)..">>") +local function entry(entries,name) + if name and (name ~= "") then + name = gsub(name,'%$','') + local result = entries[name..'.'..instance.progname] or entries[name] + if result then + return result + else + result = resolvers.env(name) + if result then + instance.variables[name] = result + resolvers.expand_variables() + return instance.expansions[name] or "" + end + end end + return "" end -function input.report(...) - if input.trace > 0 then -- extra test - input.log("<<"..format(...)..">>") +local function is_entry(entries,name) + if name and name ~= "" then + name = gsub(name,'%$','') + return (entries[name..'.'..instance.progname] or entries[name]) ~= nil + else + return false end end -input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRACE") or input.trace or 0)) - --- These functions can be used to test the performance, especially --- loading the database files. +-- {a,b,c,d} +-- a,b,c/{p,q,r},d +-- a,b,c/{p,q,r}/d/{x,y,z}// +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} +-- a{b,c}{d,e}f +-- {a,b,c,d} +-- {a,b,c/{p,q,r},d} +-- {a,b,c/{p,q,r}/d/{x,y,z}//} +-- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} +-- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} +-- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} -do - local clock = os.gettimeofday or os.clock +-- this one is better and faster, but it took me a while to realize +-- that this kind of replacement is cleaner than messy parsing and +-- fuzzy concatenating we can probably gain a bit with selectively +-- applying lpeg, but experiments with lpeg parsing this proved not to +-- work that well; the parsing is ok, but dealing with the resulting +-- table is a pain because we need to work inside-out recursively - function input.starttiming(instance) - if instance then - instance.starttime = clock() - if not instance.loadtime then - instance.loadtime = 0 +local function splitpathexpr(str, t, validate) + -- no need for further optimization as it is only called a + -- few times, we can use lpeg for the sub; we could move + -- the local functions outside the body + t = t or { } + str = gsub(str,",}",",@}") + str = gsub(str,"{,","{@,") + -- str = "@" .. str .. "@" + local ok, done + local function do_first(a,b) + local t = { } + for s in gmatch(b,"[^,]+") do t[#t+1] = a .. s end + return "{" .. concat(t,",") .. "}" + end + local function do_second(a,b) + local t = { } + for s in gmatch(a,"[^,]+") do t[#t+1] = s .. b end + return "{" .. concat(t,",") .. "}" + end + local function do_both(a,b) + local t = { } + for sa in gmatch(a,"[^,]+") do + for sb in gmatch(b,"[^,]+") do + t[#t+1] = sa .. sb end end + return "{" .. concat(t,",") .. "}" end - - function input.stoptiming(instance, report) - if instance then - local starttime = instance.starttime - if starttime then - local stoptime = clock() - local loadtime = stoptime - starttime - instance.stoptime = stoptime - instance.loadtime = instance.loadtime + loadtime - if report then - input.report("load time %0.3f",loadtime) - end - return loadtime - end + local function do_three(a,b,c) + return a .. b.. c + end + while true do + done = false + while true do + str, ok = gsub(str,"([^{},]+){([^{}]+)}",do_first) + if ok > 0 then done = true else break end end - return 0 + while true do + str, ok = gsub(str,"{([^{}]+)}([^{},]+)",do_second) + if ok > 0 then done = true else break end + end + while true do + str, ok = gsub(str,"{([^{}]+)}{([^{}]+)}",do_both) + if ok > 0 then done = true else break end + end + str, ok = gsub(str,"({[^{}]*){([^{}]+)}([^{}]*})",do_three) + if ok > 0 then done = true end + if not done then break end end - -end - -function input.elapsedtime(instance) - return format("%0.3f",(instance and instance.loadtime) or 0) -end - -function input.report_loadtime(instance) - if instance then - input.report('total load time %s', input.elapsedtime(instance)) + str = gsub(str,"[{}]", "") + str = gsub(str,"@","") + if validate then + for s in gmatch(str,"[^,]+") do + s = validate(s) + if s then t[#t+1] = s end + end + else + for s in gmatch(str,"[^,]+") do + t[#t+1] = s + end end + return t end -input.loadtime = input.elapsedtime - -function input.env(key) - return input.instance.environment[key] or input.osenv(key) -end - -function input.osenv(key) - local ie = input.instance.environment - local value = ie[key] - if value == nil then - -- local e = os.getenv(key) - local e = os.env[key] - if e == nil then - -- value = "" -- false - else - value = input.bare_variable(e) +local function expanded_path_from_list(pathlist) -- maybe not a list, just a path + -- a previous version fed back into pathlist + local newlist, ok = { }, false + for k=1,#pathlist do + if find(pathlist[k],"[{}]") then + ok = true + break end - ie[key] = value end - return value or "" + if ok then + local function validate(s) + s = file.collapse_path(s) + return s ~= "" and not find(s,dummy_path_expr) and s + end + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + else + for k=1,#pathlist do + for p in gmatch(pathlist[k],"([^,]+)") do + p = file.collapse_path(p) + if p ~= "" then newlist[#newlist+1] = p end + end + end + end + return newlist end -- we follow a rather traditional approach: @@ -5280,20 +6563,20 @@ end -- also we now follow the stupid route: if not set then just assume *one* -- cnf file under texmf (i.e. distribution) -input.ownpath = input.ownpath or nil -input.ownbin = input.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" -input.autoselfdir = true -- false may be handy for debugging +resolvers.ownpath = resolvers.ownpath or nil +resolvers.ownbin = resolvers.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" +resolvers.autoselfdir = true -- false may be handy for debugging -function input.getownpath() - if not input.ownpath then - if input.autoselfdir and os.selfdir then - input.ownpath = os.selfdir +function resolvers.getownpath() + if not resolvers.ownpath then + if resolvers.autoselfdir and os.selfdir then + resolvers.ownpath = os.selfdir else - local binary = input.ownbin + local binary = resolvers.ownbin if os.platform == "windows" then binary = file.replacesuffix(binary,"exe") end - for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do local b = file.join(p,binary) if lfs.isfile(b) then -- we assume that after changing to the path the currentdir function @@ -5303,162 +6586,91 @@ function input.getownpath() local olddir = lfs.currentdir() if lfs.chdir(p) then local pp = lfs.currentdir() - if input.verbose and p ~= pp then - input.report("following symlink %s to %s",p,pp) + if trace_verbose and p ~= pp then + logs.report("fileio","following symlink %s to %s",p,pp) end - input.ownpath = pp + resolvers.ownpath = pp lfs.chdir(olddir) else - if input.verbose then - input.report("unable to check path %s",p) + if trace_verbose then + logs.report("fileio","unable to check path %s",p) end - input.ownpath = p + resolvers.ownpath = p end break end end end - if not input.ownpath then input.ownpath = '.' end + if not resolvers.ownpath then resolvers.ownpath = '.' end end - return input.ownpath + return resolvers.ownpath end -function input.identify_own() - local instance = input.instance - local ownpath = input.getownpath() or lfs.currentdir() +local own_places = { "SELFAUTOLOC", "SELFAUTODIR", "SELFAUTOPARENT", "TEXMFCNF" } + +local function identify_own() + local ownpath = resolvers.getownpath() or lfs.currentdir() local ie = instance.environment if ownpath then - if input.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end - if input.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end - if input.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end + if resolvers.env('SELFAUTOLOC') == "" then os.env['SELFAUTOLOC'] = file.collapse_path(ownpath) end + if resolvers.env('SELFAUTODIR') == "" then os.env['SELFAUTODIR'] = file.collapse_path(ownpath .. "/..") end + if resolvers.env('SELFAUTOPARENT') == "" then os.env['SELFAUTOPARENT'] = file.collapse_path(ownpath .. "/../..") end else - input.verbose = true - input.report("error: unable to locate ownpath") + logs.report("fileio","error: unable to locate ownpath") os.exit() end - if input.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = input.cnfdefault end - if input.env('TEXOS') == "" then os.env['TEXOS'] = input.env('SELFAUTODIR') end - if input.env('TEXROOT') == "" then os.env['TEXROOT'] = input.env('SELFAUTOPARENT') end - if input.verbose then - for _,v in ipairs({"SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF"}) do - input.report("variable %s set to %s",v,input.env(v) or "unknown") + if resolvers.env('TEXMFCNF') == "" then os.env['TEXMFCNF'] = resolvers.cnfdefault end + if resolvers.env('TEXOS') == "" then os.env['TEXOS'] = resolvers.env('SELFAUTODIR') end + if resolvers.env('TEXROOT') == "" then os.env['TEXROOT'] = resolvers.env('SELFAUTOPARENT') end + if trace_verbose then + for i=1,#own_places do + local v = own_places[i] + logs.report("fileio","variable %s set to %s",v,resolvers.env(v) or "unknown") end end - function input.identify_own() end + identify_own = function() end end -function input.identify_cnf() - local instance = input.instance +function resolvers.identify_cnf() if #instance.cnffiles == 0 then -- fallback - input.identify_own() + identify_own() -- the real search - input.expand_variables() - local t = input.split_path(input.env('TEXMFCNF')) - t = input.aux.expanded_path(t) - input.aux.expand_vars(t) -- redundant + resolvers.expand_variables() + local t = resolvers.split_path(resolvers.env('TEXMFCNF')) + t = expanded_path_from_list(t) + expand_vars(t) -- redundant local function locate(filename,list) - for _,v in ipairs(t) do - local texmfcnf = input.normalize_name(file.join(v,filename)) + for i=1,#t do + local ti = t[i] + local texmfcnf = file.collapse_path(file.join(ti,filename)) if lfs.isfile(texmfcnf) then - table.insert(list,texmfcnf) + list[#list+1] = texmfcnf end end end - locate(input.luaname,instance.luafiles) - locate(input.cnfname,instance.cnffiles) + locate(resolvers.luaname,instance.luafiles) + locate(resolvers.cnfname,instance.cnffiles) end end -function input.load_cnf() - local instance = input.instance - local function loadoldconfigdata() - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(fname) - end - end - -- instance.cnffiles contain complete names now ! - if #instance.cnffiles == 0 then - input.report("no cnf files found (TEXMFCNF may not be set/known)") - else - instance.rootpath = instance.cnffiles[1] - for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = input.normalize_name(instance.rootpath) - if instance.lsrmode then - loadoldconfigdata() - elseif instance.diskcache and not instance.renewcache then - input.loadoldconfig(instance.cnffiles) - if instance.loaderror then - loadoldconfigdata() - input.saveoldconfig() - end - else - loadoldconfigdata() - if instance.renewcache then - input.saveoldconfig() - end - end - input.aux.collapse_cnf_data() - end - input.checkconfigdata() -end - -function input.load_lua() - local instance = input.instance - if #instance.luafiles == 0 then - -- yet harmless - else - instance.rootpath = instance.luafiles[1] - for k,fname in ipairs(instance.luafiles) do - instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) - end - for i=1,3 do - instance.rootpath = file.dirname(instance.rootpath) - end - instance.rootpath = input.normalize_name(instance.rootpath) - input.loadnewconfig() - input.aux.collapse_cnf_data() - end - input.checkconfigdata() -end - -function input.aux.collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) - local instance = input.instance - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) - end - end - end - end -end - -function input.aux.load_cnf(fname) - local instance = input.instance - fname = input.clean_path(fname) +local function load_cnf_file(fname) + fname = resolvers.clean_path(fname) local lname = file.replacesuffix(fname,'lua') local f = io.open(lname) if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_configuration(dname,lname) + resolvers.load_data(dname,'configuration',lname and file.basename(lname)) instance.order[#instance.order+1] = instance.configuration[dname] end else f = io.open(fname) if f then - input.report("loading %s", fname) + if trace_verbose then + logs.report("fileio","loading %s", fname) + end local line, data, n, k, v local dname = file.dirname(fname) if not instance.configuration[dname] then @@ -5470,17 +6682,19 @@ function input.aux.load_cnf(fname) local line, n = f:read(), 0 if line then while true do -- join lines - line, n = line:gsub("\\%s*$", "") + line, n = gsub(line,"\\%s*$", "") if n > 0 then line = line .. f:read() else break end end - if not line:find("^[%%#]") then - local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + if not find(line,"^[%%#]") then + local l = gsub(line,"%s*%%.*$","") + local k, v = match(l,"%s*(.-)%s*=%s*(.-)%s*$") if k and v and not data[k] then - data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") + v = gsub(v,"[%%#].*",'') + data[k] = gsub(v,"~","$HOME") instance.kpsevars[k] = true end end @@ -5489,53 +6703,119 @@ function input.aux.load_cnf(fname) end end f:close() + elseif trace_verbose then + logs.report("fileio","skipping %s", fname) + end + end +end + +local function collapse_cnf_data() -- potential optimization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in next, c do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = resolvers.bare_variable(v) + end + end + end + end +end + +function resolvers.load_cnf() + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + load_cnf_file(fname) + end + end + -- instance.cnffiles contain complete names now ! + if #instance.cnffiles == 0 then + if trace_verbose then + logs.report("fileio","no cnf files found (TEXMFCNF may not be set/known)") + end + else + instance.rootpath = instance.cnffiles[1] + for k,fname in ipairs(instance.cnffiles) do + instance.cnffiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + if instance.diskcache and not instance.renewcache then + resolvers.loadoldconfig(instance.cnffiles) + if instance.loaderror then + loadoldconfigdata() + resolvers.saveoldconfig() + end else - input.report("skipping %s", fname) + loadoldconfigdata() + if instance.renewcache then + resolvers.saveoldconfig() + end end + collapse_cnf_data() end + check_configuration() +end + +function resolvers.load_lua() + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = file.collapse_path(gsub(fname,"\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = file.collapse_path(instance.rootpath) + resolvers.loadnewconfig() + collapse_cnf_data() + end + check_configuration() end -- database loading -function input.load_hash() - local instance = input.instance - input.locatelists() - if instance.lsrmode then - input.loadlists() - elseif instance.diskcache and not instance.renewcache then - input.loadfiles() +function resolvers.load_hash() + resolvers.locatelists() + if instance.diskcache and not instance.renewcache then + resolvers.loadfiles() if instance.loaderror then - input.loadlists() - input.savefiles() + resolvers.loadlists() + resolvers.savefiles() end else - input.loadlists() + resolvers.loadlists() if instance.renewcache then - input.savefiles() + resolvers.savefiles() end end end -function input.aux.append_hash(type,tag,name) - if input.trace > 0 then - input.logger("= hash append: %s",tag) +function resolvers.append_hash(type,tag,name) + if trace_locating then + logs.report("fileio","= hash append: %s",tag) end - table.insert(input.instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) + insert(instance.hashes, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.prepend_hash(type,tag,name) - if input.trace > 0 then - input.logger("= hash prepend: %s",tag) +function resolvers.prepend_hash(type,tag,name) + if trace_locating then + logs.report("fileio","= hash prepend: %s",tag) end - table.insert(input.instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) + insert(instance.hashes, 1, { ['type']=type, ['tag']=tag, ['name']=name } ) end -function input.aux.extend_texmf_var(specification) -- crap, we could better prepend the hash - local instance = input.instance --- local t = input.expanded_path_list('TEXMF') -- full expansion - local t = input.split_path(input.env('TEXMF')) - table.insert(t,1,specification) - local newspec = table.join(t,";") +function resolvers.extend_texmf_var(specification) -- crap, we could better prepend the hash +-- local t = resolvers.expanded_path_list('TEXMF') -- full expansion + local t = resolvers.split_path(resolvers.env('TEXMF')) + insert(t,1,specification) + local newspec = concat(t,";") if instance.environment["TEXMF"] then instance.environment["TEXMF"] = newspec elseif instance.variables["TEXMF"] then @@ -5543,182 +6823,142 @@ function input.aux.extend_texmf_var(specification) -- crap, we could better prep else -- weird end - input.expand_variables() - input.reset_hashes() + resolvers.expand_variables() + reset_hashes() end -- locators -function input.locatelists() - local instance = input.instance - for _, path in pairs(input.clean_path_list('TEXMF')) do - input.report("locating list of %s",path) - input.locatedatabase(input.normalize_name(path)) +function resolvers.locatelists() + for _, path in ipairs(resolvers.clean_path_list('TEXMF')) do + if trace_verbose then + logs.report("fileio","locating list of %s",path) + end + resolvers.locatedatabase(file.collapse_path(path)) end end -function input.locatedatabase(specification) - return input.methodhandler('locators', specification) +function resolvers.locatedatabase(specification) + return resolvers.methodhandler('locators', specification) end -function input.locators.tex(specification) +function resolvers.locators.tex(specification) if specification and specification ~= '' and lfs.isdir(specification) then - if input.trace > 0 then - input.logger('! tex locator found: %s',specification) + if trace_locating then + logs.report("fileio",'! tex locator found: %s',specification) end - input.aux.append_hash('file',specification,filename) - elseif input.trace > 0 then - input.logger('? tex locator not found: %s',specification) + resolvers.append_hash('file',specification,filename) + elseif trace_locating then + logs.report("fileio",'? tex locator not found: %s',specification) end end -- hashers -function input.hashdatabase(tag,name) - return input.methodhandler('hashers',tag,name) +function resolvers.hashdatabase(tag,name) + return resolvers.methodhandler('hashers',tag,name) end -function input.loadfiles() - local instance = input.instance +function resolvers.loadfiles() instance.loaderror = false instance.files = { } if not instance.renewcache then for _, hash in ipairs(instance.hashes) do - input.hashdatabase(hash.tag,hash.name) + resolvers.hashdatabase(hash.tag,hash.name) if instance.loaderror then break end end end end -function input.hashers.tex(tag,name) - input.aux.load_files(tag) +function resolvers.hashers.tex(tag,name) + resolvers.load_data(tag,'files') end -- generators: -function input.loadlists() - for _, hash in ipairs(input.instance.hashes) do - input.generatedatabase(hash.tag) +function resolvers.loadlists() + for _, hash in ipairs(instance.hashes) do + resolvers.generatedatabase(hash.tag) end end -function input.generatedatabase(specification) - return input.methodhandler('generators', specification) +function resolvers.generatedatabase(specification) + return resolvers.methodhandler('generators', specification) end -local weird = lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +-- starting with . or .. etc or funny char + +local weird = lpeg.P(".")^1 + lpeg.anywhere(lpeg.S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) -function input.generators.tex(specification) - local instance = input.instance +function resolvers.generators.tex(specification) local tag = specification - if not instance.lsrmode and lfs.dir then - input.report("scanning path %s",specification) - instance.files[tag] = { } - local files = instance.files[tag] - local n, m, r = 0, 0, 0 - local spec = specification .. '/' - local attributes = lfs.attributes - local directory = lfs.dir - local small = instance.smallcache - local function action(path) - local mode, full - if path then - full = spec .. path .. '/' - else - full = spec - end - for name in directory(full) do - if name:find("^%.") then - -- skip - -- elseif name:find("[%~%`%!%#%$%%%^%&%*%(%)%=%{%}%[%]%:%;\"\'%|%<%>%,%?\n\r\t]") then -- too much escaped - elseif weird:match(name) then - -- texio.write_nl("skipping " .. name) - -- skip - else - mode = attributes(full..name,'mode') - if mode == 'directory' then - m = m + 1 - if path then - action(path..'/'..name) - else - action(name) - end - elseif path and mode == 'file' then + if trace_verbose then + logs.report("fileio","scanning path %s",specification) + end + instance.files[tag] = { } + local files = instance.files[tag] + local n, m, r = 0, 0, 0 + local spec = specification .. '/' + local attributes = lfs.attributes + local directory = lfs.dir + local function action(path) + local full + if path then + full = spec .. path .. '/' + else + full = spec + end + for name in directory(full) do + if not weird:match(name) then + local mode = attributes(full..name,'mode') + if mode == 'file' then + if path then n = n + 1 local f = files[name] if f then - if not small then - if type(f) == 'string' then - files[name] = { f, path } - else - f[#f+1] = path - end + if type(f) == 'string' then + files[name] = { f, path } + else + f[#f+1] = path end - else + else -- probably unique anyway files[name] = path - local lower = name:lower() + local lower = lower(name) if name ~= lower then files["remap:"..lower] = name r = r + 1 end end end - end - end - end - action() - input.report("%s files found on %s directories with %s uppercase remappings",n,m,r) - else - local fullname = file.join(specification,input.lsrname) - local path = '.' - local f = io.open(fullname) - if f then - instance.files[tag] = { } - local files = instance.files[tag] - local small = instance.smallcache - input.report("loading lsr file %s",fullname) - -- for line in f:lines() do -- much slower then the next one - for line in (f:read("*a")):gmatch("(.-)\n") do - if line:find("^[%a%d]") then - local fl = files[line] - if fl then - if not small then - if type(fl) == 'string' then - files[line] = { fl, path } -- table - else - fl[#fl+1] = path - end - end + elseif mode == 'directory' then + m = m + 1 + if path then + action(path..'/'..name) else - files[line] = path -- string - local lower = line:lower() - if line ~= lower then - files["remap:"..lower] = line - end + action(name) end - else - path = line:match("%.%/(.-)%:$") or path -- match could be nil due to empty line end end - f:close() end end + action() + if trace_verbose then + logs.report("fileio","%s files found on %s directories with %s uppercase remappings",n,m,r) + end end -- savers, todo -function input.savefiles() - input.aux.save_data('files', function(k,v) - return input.instance.validfile(k,v) -- path, name - end) +function resolvers.savefiles() + resolvers.save_data('files') end -- A config (optionally) has the paths split in tables. Internally -- we join them and split them after the expansion has taken place. This -- is more convenient. -function input.splitconfig() - for i,c in ipairs(input.instance) do +function resolvers.splitconfig() + for i,c in ipairs(instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -5730,23 +6970,23 @@ function input.splitconfig() end end -function input.joinconfig() - for i,c in ipairs(input.instance.order) do - for k,v in pairs(c) do +function resolvers.joinconfig() + for i,c in ipairs(instance.order) do + for k,v in pairs(c) do -- ipairs? if type(v) == 'table' then c[k] = file.join_path(v) end end end end -function input.split_path(str) +function resolvers.split_path(str) if type(str) == 'table' then return str else return file.split_path(str) end end -function input.join_path(str) +function resolvers.join_path(str) if type(str) == 'table' then return file.join_path(str) else @@ -5754,11 +6994,11 @@ function input.join_path(str) end end -function input.splitexpansions() - local ie = input.instance.expansions - for k,v in pairs(ie) do +function resolvers.splitexpansions() + local ie = instance.expansions + for k,v in next, ie do local t, h = { }, { } - for _,vv in pairs(file.split_path(v)) do + for _,vv in ipairs(file.split_path(v)) do if vv ~= "" and not h[vv] then t[#t+1] = vv h[vv] = true @@ -5774,27 +7014,27 @@ end -- end of split/join code -function input.saveoldconfig() - input.splitconfig() - input.aux.save_data('configuration', nil) - input.joinconfig() +function resolvers.saveoldconfig() + resolvers.splitconfig() + resolvers.save_data('configuration') + resolvers.joinconfig() end -input.configbanner = [[ +resolvers.configbanner = [[ -- This is a Luatex configuration file created by 'luatools.lua' or -- 'luatex.exe' directly. For comment, suggestions and questions you can -- contact the ConTeXt Development Team. This configuration file is -- not copyrighted. [HH & TH] ]] -function input.serialize(files) +function resolvers.serialize(files) -- This version is somewhat optimized for the kind of -- tables that we deal with, so it's much faster than -- the generic serializer. This makes sense because -- luatools and mtxtools are called frequently. Okay, -- we pay a small price for properly tabbed tables. local t = { } - local function dump(k,v,m) + local function dump(k,v,m) -- could be moved inline if type(v) == 'string' then return m .. "['" .. k .. "']='" .. v .. "'," elseif #v == 1 then @@ -5804,12 +7044,12 @@ function input.serialize(files) end end t[#t+1] = "return {" - if input.instance.sortdata then - for _, k in pairs(sortedkeys(files)) do + if instance.sortdata then + for _, k in pairs(sortedkeys(files)) do -- ipairs local fk = files[k] if type(fk) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sortedkeys(fk)) do + for _, kk in pairs(sortedkeys(fk)) do -- ipairs t[#t+1] = dump(kk,fk[kk],"\t\t") end t[#t+1] = "\t}," @@ -5818,10 +7058,10 @@ function input.serialize(files) end end else - for k, v in pairs(files) do + for k, v in next, files do if type(v) == 'table' then t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do + for kk,vv in next, v do t[#t+1] = dump(kk,vv,"\t\t") end t[#t+1] = "\t}," @@ -5834,62 +7074,67 @@ function input.serialize(files) return concat(t,"\n") end -if not texmf then texmf = {} end -- no longer needed, at least not here - -function input.aux.save_data(dataname, check, makename) -- untested without cache overload - for cachename, files in pairs(input.instance[dataname]) do +function resolvers.save_data(dataname, makename) -- untested without cache overload + for cachename, files in next, instance[dataname] do local name = (makename or file.join)(cachename,dataname) local luaname, lucname = name .. ".lua", name .. ".luc" - input.report("preparing %s for %s",dataname,cachename) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false + if trace_verbose then + logs.report("fileio","preparing %s for %s",dataname,cachename) + end + for k, v in next, files do + if type(v) == "table" and #v == 1 then + files[k] = v[1] end end local data = { type = dataname, root = cachename, - version = input.cacheversion, + version = resolvers.cacheversion, date = os.date("%Y-%m-%d"), time = os.date("%H:%M:%S"), content = files, } - local ok = io.savedata(luaname,input.serialize(data)) + local ok = io.savedata(luaname,resolvers.serialize(data)) if ok then - input.report("%s saved in %s",dataname,luaname) + if trace_verbose then + logs.report("fileio","%s saved in %s",dataname,luaname) + end if utils.lua.compile(luaname,lucname,false,true) then -- no cleanup but strip - input.report("%s compiled to %s",dataname,lucname) + if trace_verbose then + logs.report("fileio","%s compiled to %s",dataname,lucname) + end else - input.report("compiling failed for %s, deleting file %s",dataname,lucname) + if trace_verbose then + logs.report("fileio","compiling failed for %s, deleting file %s",dataname,lucname) + end os.remove(lucname) end - else - input.report("unable to save %s in %s (access error)",dataname,luaname) + elseif trace_verbose then + logs.report("fileio","unable to save %s in %s (access error)",dataname,luaname) end end end -function input.aux.load_data(pathname,dataname,filename,makename) -- untested without cache overload - local instance = input.instance +function resolvers.load_data(pathname,dataname,filename,makename) -- untested without cache overload filename = ((not filename or (filename == "")) and dataname) or filename filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) local blob = loadfile(filename .. ".luc") or loadfile(filename .. ".lua") if blob then local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading %s for %s from %s",dataname,pathname,filename) + if data and data.content and data.type == dataname and data.version == resolvers.cacheversion then + if trace_verbose then + logs.report("fileio","loading %s for %s from %s",dataname,pathname,filename) + end instance[dataname][pathname] = data.content else - input.report("skipping %s for %s from %s",dataname,pathname,filename) + if trace_verbose then + logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) + end instance[dataname][pathname] = { } instance.loaderror = true end - else - input.report("skipping %s for %s from %s",dataname,pathname,filename) + elseif trace_verbose then + logs.report("fileio","skipping %s for %s from %s",dataname,pathname,filename) end end @@ -5902,195 +7147,139 @@ end -- TEXMFBOGUS = 'effe checken of dit werkt', -- } -function input.aux.load_texmfcnf(dataname,pathname) - local instance = input.instance - local filename = file.join(pathname,input.luaname) - local blob = loadfile(filename) - if blob then - local data = blob() - if data then - input.report("loading configuration file %s",filename) - if true then - -- flatten to variable.progname - local t = { } - for k, v in pairs(data) do -- v = progname - if type(v) == "string" then - t[k] = v - else - for kk, vv in pairs(v) do -- vv = variable - if type(vv) == "string" then - t[vv.."."..v] = kk +function resolvers.resetconfig() + identify_own() + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function resolvers.loadnewconfig() + for _, cnf in ipairs(instance.luafiles) do + local pathname = file.dirname(cnf) + local filename = file.join(pathname,resolvers.luaname) + local blob = loadfile(filename) + if blob then + local data = blob() + if data then + if trace_verbose then + logs.report("fileio","loading configuration file %s",filename) + end + if true then + -- flatten to variable.progname + local t = { } + for k, v in next, data do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in next, v do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end end end end + instance['setup'][pathname] = t + else + instance['setup'][pathname] = data end - instance[dataname][pathname] = t else - instance[dataname][pathname] = data + if trace_verbose then + logs.report("fileio","skipping configuration file %s",filename) + end + instance['setup'][pathname] = { } + instance.loaderror = true end - else - input.report("skipping configuration file %s",filename) - instance[dataname][pathname] = { } - instance.loaderror = true + elseif trace_verbose then + logs.report("fileio","skipping configuration file %s",filename) end - else - input.report("skipping configuration file %s",filename) - end -end - -function input.aux.load_configuration(dname,lname) - input.aux.load_data(dname,'configuration',lname and file.basename(lname)) -end -function input.aux.load_files(tag) - input.aux.load_data(tag,'files') -end - -function input.resetconfig() - input.identify_own() - local instance = input.instance - instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false -end - -function input.loadnewconfig() - local instance = input.instance - for _, cnf in ipairs(instance.luafiles) do - local dname = file.dirname(cnf) - input.aux.load_texmfcnf('setup',dname) - instance.order[#instance.order+1] = instance.setup[dname] + instance.order[#instance.order+1] = instance.setup[pathname] if instance.loaderror then break end end end -function input.loadoldconfig() - local instance = input.instance +function resolvers.loadoldconfig() if not instance.renewcache then for _, cnf in ipairs(instance.cnffiles) do local dname = file.dirname(cnf) - input.aux.load_configuration(dname) + resolvers.load_data(dname,'configuration') instance.order[#instance.order+1] = instance.configuration[dname] if instance.loaderror then break end end end - input.joinconfig() + resolvers.joinconfig() end -function input.expand_variables() - local instance = input.instance +function resolvers.expand_variables() local expansions, environment, variables = { }, instance.environment, instance.variables - local env = input.env + local env = resolvers.env instance.expansions = expansions if instance.engine ~= "" then environment['engine'] = instance.engine end if instance.progname ~= "" then environment['progname'] = instance.progname end - for k,v in pairs(environment) do - local a, b = k:match("^(%a+)%_(.*)%s*$") + for k,v in next, environment do + local a, b = match(k,"^(%a+)%_(.*)%s*$") if a and b then expansions[a..'.'..b] = v else expansions[k] = v end end - for k,v in pairs(environment) do -- move environment to expansions + for k,v in next, environment do -- move environment to expansions if not expansions[k] then expansions[k] = v end end - for k,v in pairs(variables) do -- move variables to expansions + for k,v in next, variables do -- move variables to expansions if not expansions[k] then expansions[k] = v end end + local busy = false + local function resolve(a) + busy = true + return expansions[a] or env(a) + end while true do - local busy = false - for k,v in pairs(expansions) do - local s, n = v:gsub("%$([%a%d%_%-]+)", function(a) - busy = true - return expansions[a] or env(a) - end) - local s, m = s:gsub("%$%{([%a%d%_%-]+)%}", function(a) - busy = true - return expansions[a] or env(a) - end) + busy = false + for k,v in next, expansions do + local s, n = gsub(v,"%$([%a%d%_%-]+)",resolve) + local s, m = gsub(s,"%$%{([%a%d%_%-]+)%}",resolve) if n > 0 or m > 0 then expansions[k]= s end end if not busy then break end end - for k,v in pairs(expansions) do - expansions[k] = v:gsub("\\", '/') + for k,v in next, expansions do + expansions[k] = gsub(v,"\\", '/') end end -function input.aux.expand_vars(lst) -- simple vars - local instance = input.instance - local variables, env = instance.variables, input.env - for k,v in pairs(lst) do - lst[k] = v:gsub("%$([%a%d%_%-]+)", function(a) - return variables[a] or env(a) - end) - end +function resolvers.variable(name) + return entry(instance.variables,name) end -function input.aux.expanded_var(var) -- simple vars - local instance = input.instance - return var:gsub("%$([%a%d%_%-]+)", function(a) - return instance.variables[a] or input.env(a) - end) +function resolvers.expansion(name) + return entry(instance.expansions,name) end -function input.aux.entry(entries,name) - if name and (name ~= "") then - local instance = input.instance - name = name:gsub('%$','') - local result = entries[name..'.'..instance.progname] or entries[name] - if result then - return result - else - result = input.env(name) - if result then - instance.variables[name] = result - input.expand_variables() - return instance.expansions[name] or "" - end - end - end - return "" -end -function input.variable(name) - return input.aux.entry(input.instance.variables,name) -end -function input.expansion(name) - return input.aux.entry(input.instance.expansions,name) +function resolvers.is_variable(name) + return is_entry(instance.variables,name) end -function input.aux.is_entry(entries,name) - if name and name ~= "" then - name = name:gsub('%$','') - return (entries[name..'.'..input.instance.progname] or entries[name]) ~= nil - else - return false - end +function resolvers.is_expansion(name) + return is_entry(instance.expansions,name) end -function input.is_variable(name) - return input.aux.is_entry(input.instance.variables,name) +function resolvers.unexpanded_path_list(str) + local pth = resolvers.variable(str) + local lst = resolvers.split_path(pth) + return expanded_path_from_list(lst) end -function input.is_expansion(name) - return input.aux.is_entry(input.instance.expansions,name) +function resolvers.unexpanded_path(str) + return file.join_path(resolvers.unexpanded_path_list(str)) end -function input.unexpanded_path_list(str) - local pth = input.variable(str) - local lst = input.split_path(pth) - return input.aux.expanded_path(lst) -end +do -- no longer needed -function input.unexpanded_path(str) - return file.join_path(input.unexpanded_path_list(str)) -end - -do local done = { } - function input.reset_extra_path() - local instance = input.instance + function resolvers.reset_extra_path() local ep = instance.extra_paths if not ep then ep, done = { }, { } @@ -6100,26 +7289,25 @@ do end end - function input.register_extra_path(paths,subpaths) - local instance = input.instance + function resolvers.register_extra_path(paths,subpaths) local ep = instance.extra_paths or { } local n = #ep if paths and paths ~= "" then if subpaths and subpaths ~= "" then - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = p .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end end else - for p in paths:gmatch("[^,]+") do + for p in gmatch(paths,"[^,]+") do if not done[p] then - ep[#ep+1] = input.clean_path(p) + ep[#ep+1] = resolvers.clean_path(p) done[p] = true end end @@ -6127,10 +7315,10 @@ do elseif subpaths and subpaths ~= "" then for i=1,n do -- we gmatch each step again, not that fast, but used seldom - for s in subpaths:gmatch("[^,]+") do + for s in gmatch(subpaths,"[^,]+") do local ps = ep[i] .. "/" .. s if not done[ps] then - ep[#ep+1] = input.clean_path(ps) + ep[#ep+1] = resolvers.clean_path(ps) done[ps] = true end end @@ -6146,306 +7334,196 @@ do end -function input.expanded_path_list(str) - local instance = input.instance - local function made_list(list) - local ep = instance.extra_paths - if not ep or #ep == 0 then - return list - else - local done, new = { }, { } - -- honour . .. ../.. but only when at the start - for k, v in ipairs(list) do - if not done[v] then - if v:find("^[%.%/]$") then - done[v] = true - new[#new+1] = v - else - break - end - end - end - -- first the extra paths - for k, v in ipairs(ep) do - if not done[v] then +local function made_list(instance,list) + local ep = instance.extra_paths + if not ep or #ep == 0 then + return list + else + local done, new = { }, { } + -- honour . .. ../.. but only when at the start + for k=1,#list do + local v = list[k] + if not done[v] then + if find(v,"^[%.%/]$") then done[v] = true new[#new+1] = v + else + break end end - -- next the formal paths - for k, v in ipairs(list) do - if not done[v] then - done[v] = true - new[#new+1] = v - end + end + -- first the extra paths + for k=1,#ep do + local v = ep[k] + if not done[v] then + done[v] = true + new[#new+1] = v end - return new end - end - if not str then - return ep or { } - elseif instance.savelists then - -- engine+progname hash - str = str:gsub("%$","") - if not instance.lists[str] then -- cached - local lst = made_list(input.split_path(input.expansion(str))) - instance.lists[str] = input.aux.expanded_path(lst) + -- next the formal paths + for k=1,#list do + local v = list[k] + if not done[v] then + done[v] = true + new[#new+1] = v + end end - return instance.lists[str] - else - local lst = input.split_path(input.expansion(str)) - return made_list(input.aux.expanded_path(lst)) + return new end end - -function input.clean_path_list(str) - local t = input.expanded_path_list(str) +function resolvers.clean_path_list(str) + local t = resolvers.expanded_path_list(str) if t then for i=1,#t do - t[i] = file.collapse_path(input.clean_path(t[i])) + t[i] = file.collapse_path(resolvers.clean_path(t[i])) end end return t end -function input.expand_path(str) - return file.join_path(input.expanded_path_list(str)) +function resolvers.expand_path(str) + return file.join_path(resolvers.expanded_path_list(str)) +end + +function resolvers.expanded_path_list(str) + if not str then + return ep or { } + elseif instance.savelists then + -- engine+progname hash + str = gsub(str,"%$","") + if not instance.lists[str] then -- cached + local lst = made_list(instance,resolvers.split_path(resolvers.expansion(str))) + instance.lists[str] = expanded_path_from_list(lst) + end + return instance.lists[str] + else + local lst = resolvers.split_path(resolvers.expansion(str)) + return made_list(instance,expanded_path_from_list(lst)) + end end -function input.expanded_path_list_from_var(str) -- brrr - local tmp = input.var_of_format_or_suffix(str:gsub("%$","")) +function resolvers.expanded_path_list_from_var(str) -- brrr + local tmp = resolvers.var_of_format_or_suffix(gsub(str,"%$","")) if tmp ~= "" then - return input.expanded_path_list(str) + return resolvers.expanded_path_list(str) else - return input.expanded_path_list(tmp) + return resolvers.expanded_path_list(tmp) end end -function input.expand_path_from_var(str) - return file.join_path(input.expanded_path_list_from_var(str)) + +function resolvers.expand_path_from_var(str) + return file.join_path(resolvers.expanded_path_list_from_var(str)) end -function input.format_of_var(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.format_of_var(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.format_of_suffix(str) - return input.suffixmap[file.extname(str)] or 'tex' +function resolvers.format_of_suffix(str) + return suffixmap[file.extname(str)] or 'tex' end -function input.variable_of_format(str) - return input.formats[str] or input.formats[input.alternatives[str]] or '' +function resolvers.variable_of_format(str) + return formats[str] or formats[alternatives[str]] or '' end -function input.var_of_format_or_suffix(str) - local v = input.formats[str] +function resolvers.var_of_format_or_suffix(str) + local v = formats[str] if v then return v end - v = input.formats[input.alternatives[str]] + v = formats[alternatives[str]] if v then return v end - v = input.suffixmap[file.extname(str)] + v = suffixmap[file.extname(str)] if v then - return input.formats[isf] + return formats[isf] end return '' end -function input.expand_braces(str) -- output variable and brace expansion of STRING - local ori = input.variable(str) - local pth = input.aux.expanded_path(input.split_path(ori)) +function resolvers.expand_braces(str) -- output variable and brace expansion of STRING + local ori = resolvers.variable(str) + local pth = expanded_path_from_list(resolvers.split_path(ori)) return file.join_path(pth) end --- {a,b,c,d} --- a,b,c/{p,q,r},d --- a,b,c/{p,q,r}/d/{x,y,z}// --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a,b,c/{p,q/{x,y,z},r},d/{p,q,r} --- a{b,c}{d,e}f --- {a,b,c,d} --- {a,b,c/{p,q,r},d} --- {a,b,c/{p,q,r}/d/{x,y,z}//} --- {a,b,c/{p,q/{x,y,z}},d/{p,q,r}} --- {a,b,c/{p,q/{x,y,z},w}v,d/{p,q,r}} --- {$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,.local,}/web2c} +resolvers.isreadable = { } --- this one is better and faster, but it took me a while to realize --- that this kind of replacement is cleaner than messy parsing and --- fuzzy concatenating we can probably gain a bit with selectively --- applying lpeg, but experiments with lpeg parsing this proved not to --- work that well; the parsing is ok, but dealing with the resulting --- table is a pain because we need to work inside-out recursively - -function input.aux.splitpathexpr(str, t, validate) - -- no need for optimization, only called a few times, we can use lpeg for the sub - t = t or { } - str = str:gsub(",}",",@}") - str = str:gsub("{,","{@,") - -- str = "@" .. str .. "@" - while true do - local done = false - while true do - local ok = false - str = str:gsub("([^{},]+){([^{}]+)}", function(a,b) - local t = { } - for s in b:gmatch("[^,]+") do t[#t+1] = a .. s end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]+)}([^{},]+)", function(a,b) - local t = { } - for s in a:gmatch("[^,]+") do t[#t+1] = s .. b end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - while true do - local ok = false - str = str:gsub("{([^{}]+)}{([^{}]+)}", function(a,b) - local t = { } - for sa in a:gmatch("[^,]+") do - for sb in b:gmatch("[^,]+") do - t[#t+1] = sa .. sb - end - end - ok, done = true, true - return "{" .. concat(t,",") .. "}" - end) - if not ok then break end - end - str = str:gsub("({[^{}]*){([^{}]+)}([^{}]*})", function(a,b,c) - done = true - return a .. b.. c - end) - if not done then break end - end - str = str:gsub("[{}]", "") - str = str:gsub("@","") - if validate then - for s in str:gmatch("[^,]+") do - s = validate(s) - if s then t[#t+1] = s end - end - else - for s in str:gmatch("[^,]+") do - t[#t+1] = s - end - end - return t -end - -function input.aux.expanded_path(pathlist) -- maybe not a list, just a path - local instance = input.instance - -- a previous version fed back into pathlist - local newlist, ok = { }, false - for _,v in ipairs(pathlist) do - if v:find("[{}]") then - ok = true - break - end - end - if ok then - for _, v in ipairs(pathlist) do - input.aux.splitpathexpr(v, newlist, function(s) - s = file.collapse_path(s) - return s ~= "" and not s:find(instance.dummy_path_expr) and s - end) - end - else - for _,v in ipairs(pathlist) do - for vv in string.gmatch(v..',',"(.-),") do - vv = file.collapse_path(v) - if vv ~= "" then newlist[#newlist+1] = vv end - end - end - end - return newlist -end - -input.is_readable = { } - -function input.aux.is_readable(readable, name) - if input.trace > 2 then +function resolvers.isreadable.file(name) + local readable = lfs.isfile(name) -- brrr + if trace_detail then if readable then - input.logger("+ readable: %s",name) + logs.report("fileio","+ readable: %s",name) else - input.logger("- readable: %s", name) + logs.report("fileio","- readable: %s", name) end end return readable end -function input.is_readable.file(name) - return input.aux.is_readable(lfs.isfile(name), name) -end - -input.is_readable.tex = input.is_readable.file +resolvers.isreadable.tex = resolvers.isreadable.file -- name -- name/name -function input.aux.collect_files(names) - local instance = input.instance +local function collect_files(names) local filelist = { } - for _, fname in pairs(names) do - if fname then - if input.trace > 2 then - input.logger("? blobpath asked: %s",fname) - end - local bname = file.basename(fname) - local dname = file.dirname(fname) - if dname == "" or dname:find("^%.") then - dname = false - else - dname = "/" .. dname .. "$" - end - for _, hash in ipairs(instance.hashes) do - local blobpath = hash.tag - local files = blobpath and instance.files[blobpath] - if files then - if input.trace > 2 then - input.logger('? blobpath do: %s (%s)',blobpath,bname) + for k=1,#names do + local fname = names[k] + if trace_detail then + logs.report("fileio","? blobpath asked: %s",fname) + end + local bname = file.basename(fname) + local dname = file.dirname(fname) + if dname == "" or find(dname,"^%.") then + dname = false + else + dname = "/" .. dname .. "$" + end + local hashes = instance.hashes + for h=1,#hashes do + local hash = hashes[h] + local blobpath = hash.tag + local files = blobpath and instance.files[blobpath] + if files then + if trace_detail then + logs.report("fileio",'? blobpath do: %s (%s)',blobpath,bname) + end + local blobfile = files[bname] + if not blobfile then + local rname = "remap:"..bname + blobfile = files[rname] + if blobfile then + bname = files[rname] + blobfile = files[bname] end - local blobfile = files[bname] - if not blobfile then - local rname = "remap:"..bname - blobfile = files[rname] - if blobfile then - bname = files[rname] - blobfile = files[bname] + end + if blobfile then + if type(blobfile) == 'string' then + if not dname or find(blobfile,dname) then + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + resolvers.concatinators[hash.type](blobpath,blobfile,bname) -- result + } end - end - if blobfile then - if type(blobfile) == 'string' then - if not dname or blobfile:find(dname) then + else + for kk=1,#blobfile do + local vv = blobfile[kk] + if not dname or find(vv,dname) then filelist[#filelist+1] = { hash.type, - file.join(blobpath,blobfile,bname), -- search - input.concatinators[hash.type](blobpath,blobfile,bname) -- result + file.join(blobpath,vv,bname), -- search + resolvers.concatinators[hash.type](blobpath,vv,bname) -- result } end - else - for _, vv in pairs(blobfile) do - if not dname or vv:find(dname) then - filelist[#filelist+1] = { - hash.type, - file.join(blobpath,vv,bname), -- search - input.concatinators[hash.type](blobpath,vv,bname) -- result - } - end - end end end - elseif input.trace > 1 then - input.logger('! blobpath no: %s (%s)',blobpath,bname) end + elseif trace_locating then + logs.report("fileio",'! blobpath no: %s (%s)',blobpath,bname) end end end @@ -6456,102 +7534,95 @@ function input.aux.collect_files(names) end end -function input.suffix_of_format(str) - if input.suffixes[str] then - return input.suffixes[str][1] +function resolvers.suffix_of_format(str) + if suffixes[str] then + return suffixes[str][1] else return "" end end -function input.suffixes_of_format(str) - if input.suffixes[str] then - return input.suffixes[str] +function resolvers.suffixes_of_format(str) + if suffixes[str] then + return suffixes[str] else return {} end end -do - - -- called about 700 times for an empty doc (font initializations etc) - -- i need to weed the font files for redundant calls - - local letter = lpeg.R("az","AZ") - local separator = lpeg.P("://") - - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local rootbased = lpeg.P("/") + letter*lpeg.P(":") - - -- ./name ../name /name c: :// - function input.aux.qualified_path(filename) - return qualified:match(filename) - end - function input.aux.rootbased_path(filename) - return rootbased:match(filename) - end - - function input.normalize_name(original) - return original +function resolvers.register_in_trees(name) + if not find(name,"^%.") then + instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one end - - input.normalize_name = file.collapse_path - end -function input.aux.register_in_trees(name) - if not name:find("^%.") then - local instance = input.instance - instance.foundintrees[name] = (instance.foundintrees[name] or 0) + 1 -- maybe only one +-- split the next one up for readability (bu this module needs a cleanup anyway) + +local function can_be_dir(name) -- can become local + local fakepaths = instance.fakepaths + if not fakepaths[name] then + if lfs.isdir(name) then + fakepaths[name] = 1 -- directory + else + fakepaths[name] = 2 -- no directory + end end + return (fakepaths[name] == 1) end --- split the next one up, better for jit - -function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) - local instance = input.instance - local result = { } +local function collect_instance_files(filename,collected) -- todo : plugin (scanners, checkers etc) + local result = collected or { } local stamp = nil - filename = input.normalize_name(filename) -- elsewhere - filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere + filename = file.collapse_path(filename) -- elsewhere + filename = file.collapse_path(gsub(filename,"\\","/")) -- elsewhere -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format if instance.found[stamp] then - if input.trace > 0 then - input.logger('! remembered: %s',filename) + if trace_locating then + logs.report("fileio",'! remembered: %s',filename) end return instance.found[stamp] end end - if filename:find('%*') then - if input.trace > 0 then - input.logger('! wildcard: %s', filename) + if not dangerous[instance.format or "?"] then + if resolvers.isreadable.file(filename) then + if trace_detail then + logs.report("fileio",'= found directly: %s',filename) + end + instance.found[stamp] = { filename } + return { filename } + end + end + if find(filename,'%*') then + if trace_locating then + logs.report("fileio",'! wildcard: %s', filename) end - result = input.find_wildcard_files(filename) - elseif input.aux.qualified_path(filename) then - if input.is_readable.file(filename) then - if input.trace > 0 then - input.logger('! qualified: %s', filename) + result = resolvers.find_wildcard_files(filename) + elseif file.is_qualified_path(filename) then + if resolvers.isreadable.file(filename) then + if trace_locating then + logs.report("fileio",'! qualified: %s', filename) end result = { filename } else - local forcedname, ok = "", false - if file.extname(filename) == "" then + local forcedname, ok, suffix = "", false, file.extname(filename) + if suffix == "" then -- why if instance.format == "" then forcedname = filename .. ".tex" - if input.is_readable.file(forcedname) then - if input.trace > 0 then - input.logger('! no suffix, forcing standard filetype: tex') + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio",'! no suffix, forcing standard filetype: tex') end result, ok = { forcedname }, true end else - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do forcedname = filename .. "." .. s - if input.is_readable.file(forcedname) then - if input.trace > 0 then - input.logger('! no suffix, forcing format filetype: %s', s) + if resolvers.isreadable.file(forcedname) then + if trace_locating then + logs.report("fileio",'! no suffix, forcing format filetype: %s', s) end result, ok = { forcedname }, true break @@ -6559,8 +7630,49 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) end end end - if not ok and input.trace > 0 then - input.logger('? qualified: %s', filename) + if not ok and suffix ~= "" then + -- try to find in tree (no suffix manipulation), here we search for the + -- matching last part of the name + local basename = file.basename(filename) + local pattern = (filename .. "$"):gsub("([%.%-])","%%%1") + local savedformat = instance.format + local format = savedformat or "" + if format == "" then + instance.format = resolvers.format_of_suffix(suffix) + end + if not format then + instance.format = "othertextfiles" -- kind of everything, maybe texinput is better + end + -- + local resolved = collect_instance_files(basename) + if #result == 0 then + local lowered = lower(basename) + if filename ~= lowered then + resolved = collect_instance_files(lowered) + end + end + resolvers.format = savedformat + -- + for r=1,#resolved do + local rr = resolved[r] + if rr:find(pattern) then + result[#result+1], ok = rr, true + end + end + -- a real wildcard: + -- + -- if not ok then + -- local filelist = collect_files({basename}) + -- for f=1,#filelist do + -- local ff = filelist[f][3] or "" + -- if ff:find(pattern) then + -- result[#result+1], ok = ff, true + -- end + -- end + -- end + end + if not ok and trace_locating then + logs.report("fileio",'? qualified: %s', filename) end end else @@ -6577,45 +7689,47 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) if ext == "" then local forcedname = filename .. '.tex' wantedfiles[#wantedfiles+1] = forcedname - filetype = input.format_of_suffix(forcedname) - if input.trace > 0 then - input.logger('! forcing filetype: %s',filetype) - end + filetype = resolvers.format_of_suffix(forcedname) + if trace_locating then + logs.report("fileio",'! forcing filetype: %s',filetype) + end else - filetype = input.format_of_suffix(filename) - if input.trace > 0 then - input.logger('! using suffix based filetype: %s',filetype) + filetype = resolvers.format_of_suffix(filename) + if trace_locating then + logs.report("fileio",'! using suffix based filetype: %s',filetype) end end else if ext == "" then - for _, s in pairs(input.suffixes_of_format(instance.format)) do + local suffixes = resolvers.suffixes_of_format(instance.format) + for _, s in next, suffixes do wantedfiles[#wantedfiles+1] = filename .. "." .. s end end filetype = instance.format - if input.trace > 0 then - input.logger('! using given filetype: %s',filetype) + if trace_locating then + logs.report("fileio",'! using given filetype: %s',filetype) end end - local typespec = input.variable_of_format(filetype) - local pathlist = input.expanded_path_list(typespec) + local typespec = resolvers.variable_of_format(filetype) + local pathlist = resolvers.expanded_path_list(typespec) if not pathlist or #pathlist == 0 then -- no pathlist, access check only / todo == wildcard - if input.trace > 2 then - input.logger('? filename: %s',filename) - input.logger('? filetype: %s',filetype or '?') - input.logger('? wanted files: %s',concat(wantedfiles," | ")) - end - for _, fname in pairs(wantedfiles) do - if fname and input.is_readable.file(fname) then + if trace_detail then + logs.report("fileio",'? filename: %s',filename) + logs.report("fileio",'? filetype: %s',filetype or '?') + logs.report("fileio",'? wanted files: %s',concat(wantedfiles," | ")) + end + for k=1,#wantedfiles do + local fname = wantedfiles[k] + if fname and resolvers.isreadable.file(fname) then filename, done = fname, true result[#result+1] = file.join('.',fname) break end end -- this is actually 'other text files' or 'any' or 'whatever' - local filelist = input.aux.collect_files(wantedfiles) + local filelist = collect_files(wantedfiles) local fl = filelist and filelist[1] if fl then filename = fl[3] @@ -6624,56 +7738,53 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) end else -- list search - local filelist = input.aux.collect_files(wantedfiles) + local filelist = collect_files(wantedfiles) local doscan, recurse - if input.trace > 2 then - input.logger('? filename: %s',filename) - -- if pathlist then input.logger('? path list: %s',concat(pathlist," | ")) end - -- if filelist then input.logger('? file list: %s',concat(filelist," | ")) end + if trace_detail then + logs.report("fileio",'? filename: %s',filename) end -- a bit messy ... esp the doscan setting here - for _, path in pairs(pathlist) do - if path:find("^!!") then doscan = false else doscan = true end - if path:find("//$") then recurse = true else recurse = false end - local pathname = path:gsub("^!+", '') + for k=1,#pathlist do + local path = pathlist[k] + if find(path,"^!!") then doscan = false else doscan = true end + if find(path,"//$") then recurse = true else recurse = false end + local pathname = gsub(path,"^!+", '') done = false -- using file list if filelist and not (done and not instance.allresults) and recurse then -- compare list entries with permitted pattern - pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences - pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless + pathname = gsub(pathname,"([%-%.])","%%%1") -- this also influences + pathname = gsub(pathname,"/+$", '/.*') -- later usage of pathname + pathname = gsub(pathname,"//", '/.-/') -- not ok for /// but harmless local expr = "^" .. pathname - -- input.debug('?',expr) - for _, fl in ipairs(filelist) do + for k=1,#filelist do + local fl = filelist[k] local f = fl[2] - if f:find(expr) then - -- input.debug('T',' '..f) - if input.trace > 2 then - input.logger('= found in hash: %s',f) + if find(f,expr) then + if trace_detail then + logs.report("fileio",'= found in hash: %s',f) end --- todo, test for readable result[#result+1] = fl[3] - input.aux.register_in_trees(f) -- for tracing used files + resolvers.register_in_trees(f) -- for tracing used files done = true if not instance.allresults then break end - else - -- input.debug('F',' '..f) end end end if not done and doscan then -- check if on disk / unchecked / does not work at all / also zips - if input.method_is_file(pathname) then -- ? - local pname = pathname:gsub("%.%*$",'') - if not pname:find("%*") then - local ppname = pname:gsub("/+$","") - if input.aux.can_be_dir(ppname) then - for _, w in pairs(wantedfiles) do + if resolvers.splitmethod(pathname).scheme == 'file' then -- ? + local pname = gsub(pathname,"%.%*$",'') + if not find(pname,"%*") then + local ppname = gsub(pname,"/+$","") + if can_be_dir(ppname) then + for k=1,#wantedfiles do + local w = wantedfiles[k] local fname = file.join(ppname,w) - if input.is_readable.file(fname) then - if input.trace > 2 then - input.logger('= found by scanning: %s',fname) + if resolvers.isreadable.file(fname) then + if trace_detail then + logs.report("fileio",'= found by scanning: %s',fname) end result[#result+1] = fname done = true @@ -6693,8 +7804,8 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) end end end - for k,v in pairs(result) do - result[k] = file.collapse_path(v) + for k=1,#result do + result[k] = file.collapse_path(result[k]) end if instance.remember then instance.found[stamp] = result @@ -6702,38 +7813,12 @@ function input.aux.find_file(filename) -- todo : plugin (scanners, checkers etc) return result end -input.aux._find_file_ = input.aux.find_file -- frozen variant +if not resolvers.concatinators then resolvers.concatinators = { } end -function input.aux.find_file(filename) -- maybe make a lowres cache too - local result = input.aux._find_file_(filename) - if #result == 0 then - local lowered = filename:lower() - if filename ~= lowered then - return input.aux._find_file_(lowered) - end - end - return result -end +resolvers.concatinators.tex = file.join +resolvers.concatinators.file = resolvers.concatinators.tex -function input.aux.can_be_dir(name) - local instance = input.instance - if not instance.fakepaths[name] then - if lfs.isdir(name) then - instance.fakepaths[name] = 1 -- directory - else - instance.fakepaths[name] = 2 -- no directory - end - end - return (instance.fakepaths[name] == 1) -end - -if not input.concatinators then input.concatinators = { } end - -input.concatinators.tex = file.join -input.concatinators.file = input.concatinators.tex - -function input.find_files(filename,filetype,mustexist) - local instance = input.instance +function resolvers.find_files(filename,filetype,mustexist) if type(mustexist) == boolean then -- all set elseif type(filetype) == 'boolean' then @@ -6742,19 +7827,26 @@ function input.find_files(filename,filetype,mustexist) filetype, mustexist = nil, false end instance.format = filetype or '' - local t = input.aux.find_file(filename,true) + local result = collect_instance_files(filename) + if #result == 0 then + local lowered = lower(filename) + if filename ~= lowered then + return collect_instance_files(lowered) + end + end instance.format = '' - return t + return result end -function input.find_file(filename,filetype,mustexist) - return (input.find_files(filename,filetype,mustexist)[1] or "") +function resolvers.find_file(filename,filetype,mustexist) + return (resolvers.find_files(filename,filetype,mustexist)[1] or "") end -function input.find_given_files(filename) - local instance = input.instance +function resolvers.find_given_files(filename) local bname, result = file.basename(filename), { } - for k, hash in ipairs(instance.hashes) do + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] local files = instance.files[hash.tag] local blist = files[bname] if not blist then @@ -6767,11 +7859,12 @@ function input.find_given_files(filename) end if blist then if type(blist) == 'string' then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,blist,bname) or "" if not instance.allresults then break end else - for kk,vv in pairs(blist) do - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" + for kk=1,#blist do + local vv = blist[kk] + result[#result+1] = resolvers.concatinators[hash.type](hash.tag,vv,bname) or "" if not instance.allresults then break end end end @@ -6780,61 +7873,68 @@ function input.find_given_files(filename) return result end -function input.find_given_file(filename) - return (input.find_given_files(filename)[1] or "") +function resolvers.find_given_file(filename) + return (resolvers.find_given_files(filename)[1] or "") end -function input.find_wildcard_files(filename) -- todo: remap: - local instance = input.instance +local function doit(path,blist,bname,tag,kind,result,allresults) + local done = false + if blist and kind then + if type(blist) == 'string' then + -- make function and share code + if find(lower(blist),path) then + result[#result+1] = resolvers.concatinators[kind](tag,blist,bname) or "" + done = true + end + else + for kk=1,#blist do + local vv = blist[kk] + if find(lower(vv),path) then + result[#result+1] = resolvers.concatinators[kind](tag,vv,bname) or "" + done = true + if not allresults then break end + end + end + end + end + return done +end + +function resolvers.find_wildcard_files(filename) -- todo: remap: local result = { } local bname, dname = file.basename(filename), file.dirname(filename) - local path = dname:gsub("^*/","") - path = path:gsub("*",".*") - path = path:gsub("-","%%-") + local path = gsub(dname,"^*/","") + path = gsub(path,"*",".*") + path = gsub(path,"-","%%-") if dname == "" then path = ".*" end local name = bname - name = name:gsub("*",".*") - name = name:gsub("-","%%-") - path = path:lower() - name = name:lower() - local function doit(blist,bname,hash,allresults) - local done = false - if blist then - if type(blist) == 'string' then - -- make function and share code - if (blist:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,blist,bname) or "" - done = true - end - else - for kk,vv in pairs(blist) do - if (vv:lower()):find(path) then - result[#result+1] = input.concatinators[hash.type](hash.tag,vv,bname) or "" - done = true - if not allresults then break end - end - end - end - end - return done - end + name = gsub(name,"*",".*") + name = gsub(name,"-","%%-") + path = lower(path) + name = lower(name) local files, allresults, done = instance.files, instance.allresults, false - if name:find("%*") then - for k, hash in ipairs(instance.hashes) do - for kk, hh in pairs(files[hash.tag]) do - if not kk:find("^remap:") then - if (kk:lower()):find(name) then - if doit(hh,kk,hash,allresults) then done = true end + if find(name,"%*") then + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + for kk, hh in next, files[hash.tag] do + if not find(kk,"^remap:") then + if find(lower(kk),name) then + if doit(path,hh,kk,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end end end else - for k, hash in ipairs(instance.hashes) do - if doit(files[hash.tag][bname],bname,hash,allresults) then done = true end + local hashes = instance.hashes + for k=1,#hashes do + local hash = hashes[k] + local tag, kind = hash.tag, hash.type + if doit(path,files[tag][bname],bname,tag,kind,result,allresults) then done = true end if done and not allresults then break end end end @@ -6843,67 +7943,49 @@ function input.find_wildcard_files(filename) -- todo: remap: return result end -function input.find_wildcard_file(filename) - return (input.find_wildcard_files(filename)[1] or "") +function resolvers.find_wildcard_file(filename) + return (resolvers.find_wildcard_files(filename)[1] or "") end -- main user functions -function input.save_used_files_in_trees(filename,jobname) - local instance = input.instance - if not filename then filename = 'luatex.jlg' end - local f = io.open(filename,'w') - if f then - f:write("<?xml version='1.0' standalone='yes'?>\n") - f:write("<rl:job>\n") - if jobname then - f:write("\t<rl:name>" .. jobname .. "</rl:name>\n") - end - f:write("\t<rl:files>\n") - for _,v in pairs(sorted(instance.foundintrees)) do -- ipairs - f:write("\t\t<rl:file n='" .. instance.foundintrees[v] .. "'>" .. v .. "</rl:file>\n") - end - f:write("\t</rl:files>\n") - f:write("</rl:usedfiles>\n") - f:close() - end -end - -function input.automount() +function resolvers.automount() -- implemented later end -function input.load() - input.starttiming(input.instance) - input.resetconfig() - input.identify_cnf() - input.load_lua() - input.expand_variables() - input.load_cnf() - input.expand_variables() - input.load_hash() - input.automount() - input.stoptiming(input.instance) +function resolvers.load(option) + statistics.starttiming(instance) + resolvers.resetconfig() + resolvers.identify_cnf() + resolvers.load_lua() + resolvers.expand_variables() + resolvers.load_cnf() + resolvers.expand_variables() + if option ~= "nofiles" then + resolvers.load_hash() + resolvers.automount() + end + statistics.stoptiming(instance) end -function input.for_files(command, files, filetype, mustexist) +function resolvers.for_files(command, files, filetype, mustexist) if files and #files > 0 then local function report(str) - if input.verbose then - input.report(str) -- has already verbose + if trace_verbose then + logs.report("fileio",str) -- has already verbose else print(str) end end - if input.verbose then + if trace_verbose then report('') end - for _, file in pairs(files) do + for _, file in ipairs(files) do local result = command(file,filetype,mustexist) if type(result) == 'string' then report(result) else - for _,v in pairs(result) do + for _,v in ipairs(result) do report(v) end end @@ -6913,19 +7995,19 @@ end -- strtab -input.var_value = input.variable -- output the value of variable $STRING. -input.expand_var = input.expansion -- output variable expansion of STRING. +resolvers.var_value = resolvers.variable -- output the value of variable $STRING. +resolvers.expand_var = resolvers.expansion -- output variable expansion of STRING. -function input.show_path(str) -- output search path for file type NAME - return file.join_path(input.expanded_path_list(input.format_of_var(str))) +function resolvers.show_path(str) -- output search path for file type NAME + return file.join_path(resolvers.expanded_path_list(resolvers.format_of_var(str))) end --- input.find_file(filename) --- input.find_file(filename, filetype, mustexist) --- input.find_file(filename, mustexist) --- input.find_file(filename, filetype) +-- resolvers.find_file(filename) +-- resolvers.find_file(filename, filetype, mustexist) +-- resolvers.find_file(filename, mustexist) +-- resolvers.find_file(filename, filetype) -function input.aux.register_file(files, name, path) +function resolvers.register_file(files, name, path) if files[name] then if type(files[name]) == 'string' then files[name] = { files[name], path } @@ -6937,170 +8019,77 @@ function input.aux.register_file(files, name, path) end end -if not input.finders then input.finders = { } end -if not input.openers then input.openers = { } end -if not input.loaders then input.loaders = { } end - -input.finders.notfound = { nil } -input.openers.notfound = { nil } -input.loaders.notfound = { false, nil, 0 } - -function input.splitmethod(filename) +function resolvers.splitmethod(filename) if not filename then return { } -- safeguard elseif type(filename) == "table" then return filename -- already split - elseif not filename:find("://") then + elseif not find(filename,"://") then return { scheme="file", path = filename, original=filename } -- quick hack else return url.hashed(filename) end end -function input.method_is_file(filename) - return input.splitmethod(filename).scheme == 'file' -end - function table.sequenced(t,sep) -- temp here local s = { } - for k, v in pairs(t) do + for k, v in pairs(t) do -- pairs? s[#s+1] = k .. "=" .. v end return concat(s, sep or " | ") end -function input.methodhandler(what, filename, filetype) -- ... - local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb +function resolvers.methodhandler(what, filename, filetype) -- ... + local specification = (type(filename) == "string" and resolvers.splitmethod(filename)) or filename -- no or { }, let it bomb local scheme = specification.scheme - if input[what][scheme] then - if input.trace > 0 then - input.logger('= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) + if resolvers[what][scheme] then + if trace_locating then + logs.report("fileio",'= handler: %s -> %s -> %s',specification.original,what,table.sequenced(specification)) end - return input[what][scheme](filename,filetype) -- todo: specification + return resolvers[what][scheme](filename,filetype) -- todo: specification else - return input[what].tex(filename,filetype) -- todo: specification + return resolvers[what].tex(filename,filetype) -- todo: specification end end --- also inside next test? - -function input.findtexfile(filename, filetype) - return input.methodhandler('finders',input.normalize_name(filename), filetype) -end -function input.opentexfile(filename) - return input.methodhandler('openers',input.normalize_name(filename)) -end - -function input.findbinfile(filename, filetype) - return input.methodhandler('finders',input.normalize_name(filename), filetype) -end -function input.openbinfile(filename) - return input.methodhandler('loaders',input.normalize_name(filename)) -end - -function input.loadbinfile(filename, filetype) - local fname = input.findbinfile(input.normalize_name(filename), filetype) - if fname and fname ~= "" then - return input.openbinfile(fname) - else - return unpack(input.loaders.notfound) - end -end - -function input.texdatablob(filename, filetype) - local ok, data, size = input.loadbinfile(filename, filetype) - return data or "" -end - -input.loadtexfile = input.texdatablob - -function input.openfile(filename) - local fullname = input.findtexfile(filename) - if fullname and (fullname ~= "") then - return input.opentexfile(fullname) - else - return nil - end -end - -function input.logmode() - return (os.getenv("MTX.LOG.MODE") or os.getenv("MTX_LOG_MODE") or "tex"):lower() -end - --- this is a prelude to engine/progname specific configuration files --- in which case we can omit files meant for other programs and --- packages - ---- ctx - --- maybe texinputs + font paths --- maybe positive selection tex/context fonts/tfm|afm|vf|opentype|type1|map|enc - -input.validators = { } -input.validators.visibility = { } - -function input.validators.visibility.default(path, name) - return true -end - -function input.validators.visibility.context(path, name) - path = path[1] or path -- some day a loop - return not ( - path:find("latex") or --- path:find("doc") or - path:find("tex4ht") or - path:find("source") or --- path:find("config") or --- path:find("metafont") or - path:find("lists$") or - name:find("%.tpm$") or - name:find("%.bak$") - ) -end - --- todo: describe which functions are public (maybe input.private. ... ) - --- beware: i need to check where we still need a / on windows: - -function input.clean_path(str) +function resolvers.clean_path(str) if str then - str = str:gsub("\\","/") - str = str:gsub("^!+","") - str = str:gsub("^~",input.homedir) + str = gsub(str,"\\","/") + str = gsub(str,"^!+","") + str = gsub(str,"^~",resolvers.homedir) return str else return nil end end -function input.do_with_path(name,func) - for _, v in pairs(input.expanded_path_list(name)) do - func("^"..input.clean_path(v)) +function resolvers.do_with_path(name,func) + for _, v in pairs(resolvers.expanded_path_list(name)) do -- pairs? + func("^"..resolvers.clean_path(v)) end end -function input.do_with_var(name,func) - func(input.aux.expanded_var(name)) +function resolvers.do_with_var(name,func) + func(expanded_var(name)) end -function input.with_files(pattern,handle) - local instance = input.instance +function resolvers.with_files(pattern,handle) for _, hash in ipairs(instance.hashes) do local blobpath = hash.tag local blobtype = hash.type if blobpath then local files = instance.files[blobpath] if files then - for k,v in pairs(files) do - if k:find("^remap:") then + for k,v in next, files do + if find(k,"^remap:") then k = files[k] v = files[k] -- chained end - if k:find(pattern) then + if find(k,pattern) then if type(v) == "string" then handle(blobtype,blobpath,v,k) else - for _,vv in pairs(v) do + for _,vv in pairs(v) do -- ipairs? handle(blobtype,blobpath,vv,k) end end @@ -7111,122 +8100,32 @@ function input.with_files(pattern,handle) end end -function input.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix - local scriptpath = "scripts/context/lua" - newname = file.addsuffix(newname,"lua") - local oldscript = input.clean_path(oldname) - input.report("to be replaced old script %s", oldscript) - local newscripts = input.find_files(newname) or { } - if #newscripts == 0 then - input.report("unable to locate new script") - else - for _, newscript in ipairs(newscripts) do - newscript = input.clean_path(newscript) - input.report("checking new script %s", newscript) - if oldscript == newscript then - input.report("old and new script are the same") - elseif not newscript:find(scriptpath) then - input.report("new script should come from %s",scriptpath) - elseif not (oldscript:find(file.removesuffix(newname).."$") or oldscript:find(newname.."$")) then - input.report("invalid new script name") - else - local newdata = io.loaddata(newscript) - if newdata then - input.report("old script content replaced by new content") - io.savedata(oldscript,newdata) - break - else - input.report("unable to load new script") - end - end - end - end -end - - ---~ print(table.serialize(input.aux.splitpathexpr("/usr/share/texmf-{texlive,tetex}", {}))) - --- command line resolver: - ---~ print(input.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) - -do - - local resolvers = { } - - resolvers.environment = function(str) - return input.clean_path(os.getenv(str) or os.getenv(str:upper()) or os.getenv(str:lower()) or "") - end - resolvers.relative = function(str,n) - if io.exists(str) then - -- nothing - elseif io.exists("./" .. str) then - str = "./" .. str - else - local p = "../" - for i=1,n or 2 do - if io.exists(p .. str) then - str = p .. str - break - else - p = p .. "../" - end - end - end - return input.clean_path(str) +function resolvers.locate_format(name) + local barename, fmtname = name:gsub("%.%a+$",""), "" + if resolvers.usecache then + local path = file.join(caches.setpath("formats")) -- maybe platform + fmtname = file.join(path,barename..".fmt") or "" end - resolvers.locate = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path((fullname ~= "" and fullname) or str) + if fmtname == "" then + fmtname = resolvers.find_files(barename..".fmt")[1] or "" end - resolvers.filename = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path(file.basename((fullname ~= "" and fullname) or str)) - end - resolvers.pathname = function(str) - local fullname = input.find_given_file(str) or "" - return input.clean_path(file.dirname((fullname ~= "" and fullname) or str)) - end - - resolvers.env = resolvers.environment - resolvers.rel = resolvers.relative - resolvers.loc = resolvers.locate - resolvers.kpse = resolvers.locate - resolvers.full = resolvers.locate - resolvers.file = resolvers.filename - resolvers.path = resolvers.pathname - - local function resolve(str) - if type(str) == "table" then - for k, v in pairs(str) do - str[k] = resolve(v) or v - end - elseif str and str ~= "" then - str = str:gsub("([a-z]+):([^ \"\']*)", function(method,target) - if resolvers[method] then - return resolvers[method](target) - else - return method .. ":" .. target - end - end) + fmtname = resolvers.clean_path(fmtname) + if fmtname ~= "" then + local barename = file.removesuffix(fmtname) + local luaname, lucname, luiname = barename .. ".lua", barename .. ".luc", barename .. ".lui" + if lfs.isfile(luiname) then + return barename, luiname + elseif lfs.isfile(lucname) then + return barename, lucname + elseif lfs.isfile(luaname) then + return barename, luaname end - return str end - - if os.uname then - for k, v in pairs(os.uname()) do - if not resolvers[k] then - resolvers[k] = function() return v end - end - end - end - - input.resolve = resolve - + return nil, nil end -function input.boolean_variable(str,default) - local b = input.expansion(str) +function resolvers.boolean_variable(str,default) + local b = resolvers.expansion(str) if b == "" then return default else @@ -7235,165 +8134,20 @@ function input.boolean_variable(str,default) end end +texconfig.kpse_init = false -if not modules then modules = { } end modules ['luat-log'] = { - version = 1.001, - comment = "companion to luat-lib.tex", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - ---[[ldx-- -<p>This is a prelude to a more extensive logging module. For the sake -of parsing log files, in addition to the standard logging we will -provide an <l n='xml'/> structured file. Actually, any logging that -is hooked into callbacks will be \XML\ by default.</p> ---ldx]]-- - --- input.logger -> special tracing, driven by log level (only input) --- input.report -> goes to terminal, depends on verbose, has banner --- logs.report -> module specific tracing and reporting, no banner but class - - -input = input or { } -logs = logs or { } - ---[[ldx-- -<p>This looks pretty ugly but we need to speed things up a bit.</p> ---ldx]]-- - -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4 -} - -logs.functions = { - 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' -} - -logs.callbacks = { - 'start_page_number', - 'stop_page_number', - 'report_output_pages', - 'report_output_log' -} +kpse = { original = kpse } setmetatable(kpse, { __index = function(k,v) return resolvers[v] end } ) -logs.tracers = { -} +-- for a while -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } +input = resolvers -logs.level = 0 -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +end -- of closure -if texlua then - write_nl = print - write = io.write -end - -function logs.xml.report(category,fmt,...) -- new - write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) -end -function logs.xml.line(fmt,...) -- new - write_nl(format("<r>%s</r>",format(fmt,...))) -end - -function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end -function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end -function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end -function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end - -function logs.tex.report(category,fmt,...) -- new - -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . - write_nl(category .. " | " .. format(fmt,...)) -end -function logs.tex.line(fmt,...) -- new - write_nl(format(fmt,...)) -end - -function logs.set_level(level) - logs.level = logs.levels[level] or level -end - -function logs.set_method(method) - for _, v in pairs(logs.functions) do - logs[v] = logs[method][v] or function() end - end - if callback and input[method] then - for _, cb in pairs(logs.callbacks) do - callback.register(cb, input[method][cb]) - end - end -end +do -- create closure to overcome 200 locals limit -function logs.xml.start_page_number() - write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) -end - -function logs.xml.stop_page_number() - write("/>") - write_nl("") -end - -function logs.xml.report_output_pages(p,b) - write_nl(format("<v k='pages' v='%s'/>", p)) - write_nl(format("<v k='bytes' v='%s'/>", b)) - write_nl("") -end - -function logs.xml.report_output_log() -end - -function input.logger(...) -- assumes test for input.trace > n - if input.trace > 0 then - logs.report(...) - end -end - -function input.report(fmt,...) - if input.verbose then - logs.report(input.banner or "report",format(fmt,...)) - end -end - -function input.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do - logs.report(input.banner or "report",line) - end -end - -input.moreinfo = [[ -more information about ConTeXt and the tools that come with it can be found at: - -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] - -function input.help(banner,message) - if not input.verbose then - input.verbose = true - -- input.report(banner,"\n") - end - input.report(banner,"\n") - input.report("") - input.reportlines(message) - if input.moreinfo and input.moreinfo ~= "" then - input.report("") - input.reportlines(input.moreinfo) - end -end - -logs.set_level('error') -logs.set_method('tex') - - -if not modules then modules = { } end modules ['luat-tmp'] = { +if not modules then modules = { } end modules ['data-tmp'] = { version = 1.001, comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -7417,17 +8171,16 @@ being written at the same time is small. We also need to extend luatools with a recache feature.</p> --ldx]]-- -local format = string.format +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) caches = caches or { } -dir = dir or { } -texmf = texmf or { } caches.path = caches.path or nil caches.base = caches.base or "luatex-cache" caches.more = caches.more or "context" caches.direct = false -- true is faster but may need huge amounts of memory -caches.trace = false caches.tree = false caches.paths = caches.paths or nil caches.force = false @@ -7437,13 +8190,14 @@ function caches.temp() local cachepath = nil local function check(list,isenv) if not cachepath then - for _, v in ipairs(list) do + for k=1,#list do + local v = list[k] cachepath = (isenv and (os.env[v] or "")) or v or "" if cachepath == "" then -- next else - cachepath = input.clean_path(cachepath) - if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" + cachepath = resolvers.clean_path(cachepath) + if lfs.isdir(cachepath) and file.iswritable(cachepath) then -- lfs.attributes(cachepath,"mode") == "directory" break elseif caches.force or io.ask(format("\nShould I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then dir.mkdirs(cachepath) @@ -7456,7 +8210,7 @@ function caches.temp() end end end - check(input.clean_path_list("TEXMFCACHE") or { }) + check(resolvers.clean_path_list("TEXMFCACHE") or { }) check(caches.defaults,true) if not cachepath then print("\nfatal error: there is no valid (writable) cache path defined\n") @@ -7465,7 +8219,7 @@ function caches.temp() print(format("\nfatal error: cache path %s is not a directory\n",cachepath)) os.exit() end - cachepath = input.normalize_name(cachepath) + cachepath = file.collapse_path(cachepath) function caches.temp() return cachepath end @@ -7473,24 +8227,13 @@ function caches.temp() end function caches.configpath() - return table.concat(input.instance.cnffiles,";") + return table.concat(resolvers.instance.cnffiles,";") end function caches.hashed(tree) - return md5.hex((tree:lower()):gsub("[\\\/]+","/")) + return md5.hex(gsub(lower(tree),"[\\\/]+","/")) end ---~ tracing: - ---~ function caches.hashed(tree) ---~ tree = (tree:lower()):gsub("[\\\/]+","/") ---~ local hash = md5.hex(tree) ---~ if input.verbose then -- temp message ---~ input.report("hashing %s => %s",tree,hash) ---~ end ---~ return hash ---~ end - function caches.treehash() local tree = caches.configpath() if not tree or tree == "" then @@ -7505,21 +8248,19 @@ function caches.setpath(...) if not caches.path then caches.path = caches.temp() end - caches.path = input.clean_path(caches.path) -- to be sure - if lfs then - caches.tree = caches.tree or caches.treehash() - if caches.tree then - caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) - else - caches.path = dir.mkdirs(caches.path,caches.base,caches.more) - end + caches.path = resolvers.clean_path(caches.path) -- to be sure + caches.tree = caches.tree or caches.treehash() + if caches.tree then + caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree) + else + caches.path = dir.mkdirs(caches.path,caches.base,caches.more) end end if not caches.path then caches.path = '.' end - caches.path = input.clean_path(caches.path) - if lfs and not table.is_empty({...}) then + caches.path = resolvers.clean_path(caches.path) + if not table.is_empty({...}) then local pth = dir.mkdirs(caches.path,...) return pth end @@ -7547,9 +8288,14 @@ function caches.loaddata(path,name) end end -function caches.is_writable(filepath,filename) +--~ function caches.loaddata(path,name) +--~ local tmaname, tmcname = caches.setluanames(path,name) +--~ return dofile(tmcname) or dofile(tmaname) +--~ end + +function caches.iswritable(filepath,filename) local tmaname, tmcname = caches.setluanames(filepath,filename) - return file.is_writable(tmaname) + return file.iswritable(tmaname) end function caches.savedata(filepath,filename,data,raw) @@ -7559,23 +8305,177 @@ function caches.savedata(filepath,filename,data,raw) reduce, simplify = false, false end if caches.direct then - file.savedata(tmaname, table.serialize(data,'return',true,true,false)) -- no hex + file.savedata(tmaname, table.serialize(data,'return',false,true,false)) -- no hex else - table.tofile(tmaname, data,'return',true,true,false) -- maybe not the last true + table.tofile(tmaname, data,'return',false,true,false) -- maybe not the last true end - local cleanup = input.boolean_variable("PURGECACHE", false) - local strip = input.boolean_variable("LUACSTRIP", true) + local cleanup = resolvers.boolean_variable("PURGECACHE", false) + local strip = resolvers.boolean_variable("LUACSTRIP", true) utils.lua.compile(tmaname, tmcname, cleanup, strip) end -- here we use the cache for format loading (texconfig.[formatname|jobname]) --~ if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then -if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and input.instance then +if tex and texconfig and (not texconfig.formatname or texconfig.formatname == "") and input and resolvers.instance then if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end -- or luc - texconfig.formatname = caches.setpath("formats") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt") + texconfig.formatname = caches.setpath("formats") .. "/" .. gsub(texconfig.luaname,"%.lu.$",".fmt") +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-res'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--~ print(resolvers.resolve("abc env:tmp file:cont-en.tex path:cont-en.tex full:cont-en.tex rel:zapf/one/p-chars.tex")) + +local upper, lower, gsub = string.upper, string.lower, string.gsub + +local prefixes = { } + +prefixes.environment = function(str) + return resolvers.clean_path(os.getenv(str) or os.getenv(upper(str)) or os.getenv(lower(str)) or "") +end + +prefixes.relative = function(str,n) + if io.exists(str) then + -- nothing + elseif io.exists("./" .. str) then + str = "./" .. str + else + local p = "../" + for i=1,n or 2 do + if io.exists(p .. str) then + str = p .. str + break + else + p = p .. "../" + end + end + end + return resolvers.clean_path(str) +end + +prefixes.locate = function(str) + local fullname = resolvers.find_given_file(str) or "" + return resolvers.clean_path((fullname ~= "" and fullname) or str) +end + +prefixes.filename = function(str) + local fullname = resolvers.find_given_file(str) or "" + return resolvers.clean_path(file.basename((fullname ~= "" and fullname) or str)) +end + +prefixes.pathname = function(str) + local fullname = resolvers.find_given_file(str) or "" + return resolvers.clean_path(file.dirname((fullname ~= "" and fullname) or str)) end +prefixes.env = prefixes.environment +prefixes.rel = prefixes.relative +prefixes.loc = prefixes.locate +prefixes.kpse = prefixes.locate +prefixes.full = prefixes.locate +prefixes.file = prefixes.filename +prefixes.path = prefixes.pathname + +local function _resolve_(method,target) + if prefixes[method] then + return prefixes[method](target) + else + return method .. ":" .. target + end +end + +local function resolve(str) + if type(str) == "table" then + for k, v in pairs(str) do -- ipairs + str[k] = resolve(v) or v + end + elseif str and str ~= "" then + str = gsub(str,"([a-z]+):([^ \"\']*)",_resolve_) + end + return str +end + +resolvers.resolve = resolve + +if os.uname then + + for k, v in pairs(os.uname()) do + if not prefixes[k] then + prefixes[k] = function() return v end + end + end + +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-inp'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +resolvers.finders = resolvers.finders or { } +resolvers.openers = resolvers.openers or { } +resolvers.loaders = resolvers.loaders or { } + +resolvers.finders.notfound = { nil } +resolvers.openers.notfound = { nil } +resolvers.loaders.notfound = { false, nil, 0 } + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-out'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +outputs = outputs or { } + + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-con'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_cache = false trackers.register("resolvers.cache", function(v) trace_cache = v end) +local trace_containers = false trackers.register("resolvers.containers", function(v) trace_containers = v end) +local trace_storage = false trackers.register("resolvers.storage", function(v) trace_storage = v end) +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + --[[ldx-- <p>Once we found ourselves defining similar cache constructs several times, containers were introduced. Containers are used @@ -7589,126 +8489,145 @@ table structures without bothering about the disk cache.</p> <p>Examples of usage can be found in the font related code.</p> --ldx]]-- -containers = { } -containers.trace = false +containers = containers or { } -do -- local report +containers.usecache = true - local function report(container,tag,name) - if caches.trace or containers.trace or container.trace then - logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') - end +local function report(container,tag,name) + if trace_cache or trace_containers then + logs.report(format("%s cache",container.subcategory),"%s: %s",tag,name or 'invalid') end +end - local allocated = { } +local allocated = { } - -- tracing +-- tracing - function containers.define(category, subcategory, version, enabled) - return function() - if category and subcategory then - local c = allocated[category] - if not c then - c = { } - allocated[category] = c - end - local s = c[subcategory] - if not s then - s = { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(category,subcategory), - } - c[subcategory] = s - end - return s - else - return nil - end +function containers.define(category, subcategory, version, enabled) + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches and caches.setpath and caches.setpath(category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil end end +end - function containers.is_usable(container, name) - return container.enabled and caches.is_writable(container.path, name) +function containers.is_usable(container, name) + return container.enabled and caches and caches.iswritable(container.path, name) +end + +function containers.is_valid(container, name) + if name and name ~= "" then + local storage = container.storage[name] + return storage and not table.is_empty(storage) and storage.cache_version == container.version + else + return false end +end - function containers.is_valid(container, name) - if name and name ~= "" then - local storage = container.storage[name] - return storage and not table.is_empty(storage) and storage.cache_version == container.version +function containers.read(container,name) + if container.enabled and caches and not container.storage[name] and containers.usecache then + container.storage[name] = caches.loaddata(container.path,name) + if containers.is_valid(container,name) then + report(container,"loaded",name) else - return false + container.storage[name] = nil end end - - function containers.read(container,name) - if container.enabled and not container.storage[name] then - container.storage[name] = caches.loaddata(container.path,name) - if containers.is_valid(container,name) then - report(container,"loaded",name) - else - container.storage[name] = nil - end - end - if container.storage[name] then - report(container,"reusing",name) - end - return container.storage[name] + if container.storage[name] then + report(container,"reusing",name) end + return container.storage[name] +end - function containers.write(container, name, data) - if data then - data.cache_version = container.version - if container.enabled then - local unique, shared = data.unique, data.shared - data.unique, data.shared = nil, nil - caches.savedata(container.path, name, data) - report(container,"saved",name) - data.unique, data.shared = unique, shared - end - report(container,"stored",name) - container.storage[name] = data +function containers.write(container, name, data) + if data then + data.cache_version = container.version + if container.enabled and caches then + local unique, shared = data.unique, data.shared + data.unique, data.shared = nil, nil + caches.savedata(container.path, name, data) + report(container,"saved",name) + data.unique, data.shared = unique, shared end - return data + report(container,"stored",name) + container.storage[name] = data end + return data +end - function containers.content(container,name) - return container.storage[name] - end +function containers.content(container,name) + return container.storage[name] +end +function containers.cleanname(name) + return (gsub(lower(name),"[^%w%d]+","-")) end + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-use'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local format, lower, gsub = string.format, string.lower, string.gsub + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v trackers.enable("resolvers.verbose") end) + -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. -local save_data = input.aux.save_data -local load_data = input.aux.load_data +local save_data = resolvers.save_data +local load_data = resolvers.load_data -input.cachepath = nil -- public, for tracing -input.usecache = true -- public, for tracing +resolvers.cachepath = nil -- public, for tracing +resolvers.usecache = true -- public, for tracing -function input.aux.save_data(dataname, check) - save_data(dataname, check, function(cachename,dataname) - input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) - if input.usecache then - input.cachepath = input.cachepath or caches.definepath("trees") - return file.join(input.cachepath(),caches.hashed(cachename)) +function resolvers.save_data(dataname) + save_data(dataname, function(cachename,dataname) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(cachename)) else return file.join(cachename,dataname) end end) end -function input.aux.load_data(pathname,dataname,filename) +function resolvers.load_data(pathname,dataname,filename) load_data(pathname,dataname,filename,function(dataname,filename) - input.usecache = not toboolean(input.expansion("CACHEINTDS") or "false",true) - if input.usecache then - input.cachepath = input.cachepath or caches.definepath("trees") - return file.join(input.cachepath(),caches.hashed(pathname)) + resolvers.usecache = not toboolean(resolvers.expansion("CACHEINTDS") or "false",true) + if resolvers.usecache then + resolvers.cachepath = resolvers.cachepath or caches.definepath("trees") + return file.join(resolvers.cachepath(),caches.hashed(pathname)) else if not filename or (filename == "") then filename = dataname @@ -7720,15 +8639,15 @@ end -- we will make a better format, maybe something xml or just text or lua -input.automounted = input.automounted or { } +resolvers.automounted = resolvers.automounted or { } -function input.automount(usecache) - local mountpaths = input.clean_path_list(input.expansion('TEXMFMOUNT')) +function resolvers.automount(usecache) + local mountpaths = resolvers.clean_path_list(resolvers.expansion('TEXMFMOUNT')) if table.is_empty(mountpaths) and usecache then mountpaths = { caches.setpath("mount") } end if not table.is_empty(mountpaths) then - input.starttiming(input.instance) + statistics.starttiming(resolvers.instance) for k, root in pairs(mountpaths) do local f = io.open(root.."/url.tmi") if f then @@ -7737,98 +8656,72 @@ function input.automount(usecache) if line:find("^[%%#%-]") then -- or %W -- skip elseif line:find("^zip://") then - input.report("mounting %s",line) - table.insert(input.automounted,line) - input.usezipfile(line) + if trace_locating then + logs.report("fileio","mounting %s",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) end end end f:close() end end - input.stoptiming(input.instance) + statistics.stoptiming(resolvers.instance) end end --- store info in format +-- status info -input.storage = { } -input.storage.data = { } -input.storage.min = 0 -- 500 -input.storage.max = input.storage.min - 1 -input.storage.trace = false -- true -input.storage.done = input.storage.done or 0 -input.storage.evaluators = { } --- (evaluate,message,names) +statistics.register("used config path", function() return caches.configpath() end) +statistics.register("used cache path", function() return caches.temp() or "?" end) -function input.storage.register(...) - input.storage.data[#input.storage.data+1] = { ... } -end +-- experiment (code will move) -function input.storage.evaluate(name) - input.storage.evaluators[#input.storage.evaluators+1] = name -end - -function input.storage.finalize() -- we can prepend the string with "evaluate:" - for _, t in ipairs(input.storage.evaluators) do - for i, v in pairs(t) do - if type(v) == "string" then - t[i] = loadstring(v)() - elseif type(v) == "table" then - for _, vv in pairs(v) do - if type(vv) == "string" then - t[i] = loadstring(vv)() - end - end - end - end +function statistics.save_fmt_status(texname,formatbanner,sourcefile) -- texname == formatname + local enginebanner = status.list().banner + if formatbanner and enginebanner and sourcefile then + local luvname = file.replacesuffix(texname,"luv") + local luvdata = { + enginebanner = enginebanner, + formatbanner = formatbanner, + sourcehash = md5.hex(io.loaddata(resolvers.find_file(sourcefile)) or "unknown"), + sourcefile = sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) end end -function input.storage.dump() - for name, data in ipairs(input.storage.data) do - local evaluate, message, original, target = data[1], data[2], data[3] ,data[4] - local name, initialize, finalize, code = nil, "", "", "" - for str in target:gmatch("([^%.]+)") do - if name then - name = name .. "." .. str +function statistics.check_fmt_status(texname) + local enginebanner = status.list().banner + if enginebanner and texname then + local luvname = file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv = dofile(luvname) + if luv and luv.sourcefile then + local sourcehash = md5.hex(io.loaddata(resolvers.find_file(luv.sourcefile)) or "unknown") + if luv.enginebanner and luv.enginebanner ~= enginebanner then + return "engine mismatch" + end + if luv.sourcehash and luv.sourcehash ~= sourcehash then + return "source mismatch" + end else - name = str + return "invalid status file" end - initialize = format("%s %s = %s or {} ", initialize, name, name) - end - if evaluate then - finalize = "input.storage.evaluate(" .. name .. ")" - end - input.storage.max = input.storage.max + 1 - if input.storage.trace then - logs.report('storage','saving %s in slot %s',message,input.storage.max) - code = - initialize .. - format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) .. - table.serialize(original,name) .. - finalize else - code = initialize .. table.serialize(original,name) .. finalize + return "missing status file" end - lua.bytecode[input.storage.max] = loadstring(code) end + return true end --- we also need to count at generation time (nicer for message) -if lua.bytecode then -- from 0 upwards - local i = input.storage.min - while lua.bytecode[i] do - lua.bytecode[i]() - lua.bytecode[i] = nil - i = i + 1 - end - input.storage.done = i -end +end -- of closure +do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['luat-log'] = { +if not modules then modules = { } end modules ['data-zip'] = { version = 1.001, comment = "companion to luat-lib.tex", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -7836,154 +8729,556 @@ if not modules then modules = { } end modules ['luat-log'] = { license = "see context related readme files" } ---[[ldx-- -<p>This is a prelude to a more extensive logging module. For the sake -of parsing log files, in addition to the standard logging we will -provide an <l n='xml'/> structured file. Actually, any logging that -is hooked into callbacks will be \XML\ by default.</p> ---ldx]]-- +local format, find = string.format, string.find --- input.logger -> special tracing, driven by log level (only input) --- input.report -> goes to terminal, depends on verbose, has banner --- logs.report -> module specific tracing and reporting, no banner but class +local trace_locating, trace_verbose = false, false +trackers.register("resolvers.verbose", function(v) trace_verbose = v end) +trackers.register("resolvers.locating", function(v) trace_locating = v trace_verbose = v end) -input = input or { } -logs = logs or { } +zip = zip or { } +zip.archives = zip.archives or { } +zip.registeredfiles = zip.registeredfiles or { } ---[[ldx-- -<p>This looks pretty ugly but we need to speed things up a bit.</p> ---ldx]]-- +local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders +local locators, hashers, concatinators = resolvers.locators, resolvers.hashers, resolvers.concatinators -logs.levels = { - ['error'] = 1, - ['warning'] = 2, - ['info'] = 3, - ['debug'] = 4 -} +local archives = zip.archives -logs.functions = { - 'report', 'start', 'stop', 'push', 'pop', 'line', 'direct' -} - -logs.callbacks = { - 'start_page_number', - 'stop_page_number', - 'report_output_pages', - 'report_output_log' -} +-- zip:///oeps.zip?name=bla/bla.tex +-- zip:///oeps.zip?tree=tex/texmf-local -logs.tracers = { -} +local function validzip(str) -- todo: use url splitter + if not find(str,"^zip://") then + return "zip:///" .. str + else + return str + end +end -logs.xml = logs.xml or { } -logs.tex = logs.tex or { } +function zip.openarchive(name) + if not name or name == "" then + return nil + else + local arch = archives[name] + if not arch then + local full = resolvers.find_file(name) or "" + arch = (full ~= "" and zip.open(full)) or false + archives[name] = arch + end + return arch + end +end -logs.level = 0 +function zip.closearchive(name) + if not name or (name == "" and archives[name]) then + zip.close(archives[name]) + archives[name] = nil + end +end -local write_nl, write, format = texio.write_nl or print, texio.write or io.write, string.format +-- zip:///texmf.zip?tree=/tex/texmf +-- zip:///texmf.zip?tree=/tex/texmf-local +-- zip:///texmf-mine.zip?tree=/tex/texmf-projects -if texlua then - write_nl = print - write = io.write +function locators.zip(specification) -- where is this used? startup zips (untested) + specification = resolvers.splitmethod(specification) + local zipfile = specification.path + local zfile = zip.openarchive(name) -- tricky, could be in to be initialized tree + if trace_locating then + if zfile then + logs.report("fileio",'! zip locator, found: %s',specification.original) + else + logs.report("fileio",'? zip locator, not found: %s',specification.original) + end + end end -function logs.xml.report(category,fmt,...) -- new - write_nl(format("<r category='%s'>%s</r>",category,format(fmt,...))) +function hashers.zip(tag,name) + if trace_verbose then + logs.report("fileio","loading zip file %s as %s",name,tag) + end + resolvers.usezipfile(format("%s?tree=%s",tag,name)) end -function logs.xml.line(fmt,...) -- new - write_nl(format("<r>%s</r>",format(fmt,...))) + +function concatinators.zip(tag,path,name) + if not path or path == "" then + return format('%s?name=%s',tag,name) + else + return format('%s?name=%s/%s',tag,path,name) + end end -function logs.xml.start() if logs.level > 0 then tw("<%s>" ) end end -function logs.xml.stop () if logs.level > 0 then tw("</%s>") end end -function logs.xml.push () if logs.level > 0 then tw("<!-- ") end end -function logs.xml.pop () if logs.level > 0 then tw(" -->" ) end end +function resolvers.isreadable.zip(name) + return true +end -function logs.tex.report(category,fmt,...) -- new - -- write_nl(format("%s | %s",category,format(fmt,...))) -- arg to format can be tex comment so .. . - write_nl(category .. " | " .. format(fmt,...)) +function finders.zip(specification,filetype) + specification = resolvers.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(specification.path) + if zfile then + if trace_locating then + logs.report("fileio",'! zip finder, path: %s',specification.path) + end + local dfile = zfile:open(q.name) + if dfile then + dfile = zfile:close() + if trace_locating then + logs.report("fileio",'+ zip finder, name: %s',q.name) + end + return specification.original + end + elseif trace_locating then + logs.report("fileio",'? zip finder, path %s',specification.path) + end + end + end + if trace_locating then + logs.report("fileio",'- zip finder, name: %s',filename) + end + return unpack(finders.notfound) end -function logs.tex.line(fmt,...) -- new - write_nl(format(fmt,...)) + +function openers.zip(specification) + local zipspecification = resolvers.splitmethod(specification) + if zipspecification.path then + local q = url.query(zipspecification.query) + if q.name then + local zfile = zip.openarchive(zipspecification.path) + if zfile then + if trace_locating then + logs.report("fileio",'+ zip starter, path: %s',zipspecification.path) + end + local dfile = zfile:open(q.name) + if dfile then + logs.show_open(specification) + return openers.text_opener(specification,dfile,'zip') + end + elseif trace_locating then + logs.report("fileio",'- zip starter, path %s',zipspecification.path) + end + end + end + if trace_locating then + logs.report("fileio",'- zip opener, name: %s',filename) + end + return unpack(openers.notfound) end -function logs.set_level(level) - logs.level = logs.levels[level] or level +function loaders.zip(specification) + specification = resolvers.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(specification.path) + if zfile then + if trace_locating then + logs.report("fileio",'+ zip starter, path: %s',specification.path) + end + local dfile = zfile:open(q.name) + if dfile then + logs.show_load(filename) + if trace_locating then + logs.report("fileio",'+ zip loader, name: %s',filename) + end + local s = dfile:read("*all") + dfile:close() + return true, s, #s + end + elseif trace_locating then + logs.report("fileio",'- zip starter, path: %s',specification.path) + end + end + end + if trace_locating then + logs.report("fileio",'- zip loader, name: %s',filename) + end + return unpack(openers.notfound) end -function logs.set_method(method) - for _, v in pairs(logs.functions) do - logs[v] = logs[method][v] or function() end +-- zip:///somefile.zip +-- zip:///somefile.zip?tree=texmf-local -> mount + +function resolvers.usezipfile(zipname) + zipname = validzip(zipname) + if trace_locating then + logs.report("fileio",'! zip use, file: %s',zipname) end - if callback and input[method] then - for _, cb in pairs(logs.callbacks) do - callback.register(cb, input[method][cb]) + local specification = resolvers.splitmethod(zipname) + local zipfile = specification.path + if zipfile and not zip.registeredfiles[zipname] then + local tree = url.query(specification.query).tree or "" + if trace_locating then + logs.report("fileio",'! zip register, file: %s',zipname) + end + local z = zip.openarchive(zipfile) + if z then + local instance = resolvers.instance + if trace_locating then + logs.report("fileio","= zipfile, registering: %s",zipname) + end + statistics.starttiming(instance) + resolvers.prepend_hash('zip',zipname,zipfile) + resolvers.extend_texmf_var(zipname) -- resets hashes too + zip.registeredfiles[zipname] = z + instance.files[zipname] = resolvers.register_zip_file(z,tree or "") + statistics.stoptiming(instance) + elseif trace_locating then + logs.report("fileio","? zipfile, unknown: %s",zipname) end + elseif trace_locating then + logs.report("fileio",'! zip register, no file: %s',zipname) end end -function logs.xml.start_page_number() - write_nl(format("<p real='%s' page='%s' sub='%s'", tex.count[0], tex.count[1], tex.count[2])) +function resolvers.register_zip_file(z,tree) + local files, filter = { }, "" + if tree == "" then + filter = "^(.+)/(.-)$" + else + filter = format("^%s/(.+)/(.-)$",tree) + end + if trace_locating then + logs.report("fileio",'= zip filter: %s',filter) + end + local register, n = resolvers.register_file, 0 + for i in z:files() do + local path, name = i.filename:match(filter) + if path then + if name and name ~= '' then + register(files, name, path) + n = n + 1 + else + -- directory + end + else + register(files, i.filename, '') + n = n + 1 + end + end + logs.report("fileio",'= zip entries: %s',n) + return files end -function logs.xml.stop_page_number() - write("/>") - write_nl("") + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-crl'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +curl = curl or { } + +curl.cached = { } +curl.cachepath = caches.definepath("curl") + +local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders + +function curl.fetch(protocol, name) + local cachename = curl.cachepath() .. "/" .. name:gsub("[^%a%d%.]+","-") +-- cachename = cachename:gsub("[\\/]", io.fileseparator) + cachename = cachename:gsub("[\\]", "/") -- cleanup + if not curl.cached[name] then + if not io.exists(cachename) then + curl.cached[name] = cachename + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" + os.spawn(command) + end + if io.exists(cachename) then + curl.cached[name] = cachename + else + curl.cached[name] = "" + end + end + return curl.cached[name] end -function logs.xml.report_output_pages(p,b) - write_nl(format("<v k='pages' v='%s'/>", p)) - write_nl(format("<v k='bytes' v='%s'/>", b)) - write_nl("") +function finders.curl(protocol,filename) + local foundname = curl.fetch(protocol, filename) + return finders.generic(protocol,foundname,filetype) end -function logs.xml.report_output_log() +function openers.curl(protocol,filename) + return openers.generic(protocol,filename) end -function input.logger(...) -- assumes test for input.trace > n - if input.trace > 0 then - logs.report(...) - end +function loaders.curl(protocol,filename) + return loaders.generic(protocol,filename) end -function input.report(fmt,...) - if input.verbose then - logs.report(input.banner or "report",format(fmt,...)) - end +-- todo: metamethod + +function curl.install(protocol) + finders[protocol] = function (filename,filetype) return finders.curl(protocol,filename) end + openers[protocol] = function (filename) return openers.curl(protocol,filename) end + loaders[protocol] = function (filename) return loaders.curl(protocol,filename) end end -function input.reportlines(str) -- todo: <lines></lines> - for line in str:gmatch("(.-)[\n\r]") do - logs.report(input.banner or "report",line) +curl.install('http') +curl.install('https') +curl.install('ftp') + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['luat-kps'] = { + version = 1.001, + comment = "companion to luatools.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +--[[ldx-- +<p>This file is used when we want the input handlers to behave like +<type>kpsewhich</type>. What to do with the following:</p> + +<typing> +{$SELFAUTOLOC,$SELFAUTODIR,$SELFAUTOPARENT}{,{/share,}/texmf{-local,}/web2c} +$SELFAUTOLOC : /usr/tex/bin/platform +$SELFAUTODIR : /usr/tex/bin +$SELFAUTOPARENT : /usr/tex +</typing> + +<p>How about just forgetting about them?</p> +--ldx]]-- + +local suffixes = resolvers.suffixes +local formats = resolvers.formats + +suffixes['gf'] = { '<resolution>gf' } +suffixes['pk'] = { '<resolution>pk' } +suffixes['base'] = { 'base' } +suffixes['bib'] = { 'bib' } +suffixes['bst'] = { 'bst' } +suffixes['cnf'] = { 'cnf' } +suffixes['mem'] = { 'mem' } +suffixes['mf'] = { 'mf' } +suffixes['mfpool'] = { 'pool' } +suffixes['mft'] = { 'mft' } +suffixes['mppool'] = { 'pool' } +suffixes['graphic/figure'] = { 'eps', 'epsi' } +suffixes['texpool'] = { 'pool' } +suffixes['PostScript header'] = { 'pro' } +suffixes['ist'] = { 'ist' } +suffixes['web'] = { 'web', 'ch' } +suffixes['cweb'] = { 'w', 'web', 'ch' } +suffixes['cmap files'] = { 'cmap' } +suffixes['lig files'] = { 'lig' } +suffixes['bitmap font'] = { } +suffixes['MetaPost support'] = { } +suffixes['TeX system documentation'] = { } +suffixes['TeX system sources'] = { } +suffixes['dvips config'] = { } +suffixes['type42 fonts'] = { } +suffixes['web2c files'] = { } +suffixes['other text files'] = { } +suffixes['other binary files'] = { } +suffixes['opentype fonts'] = { 'otf' } + +suffixes['fmt'] = { 'fmt' } +suffixes['texmfscripts'] = { 'rb','lua','py','pl' } + +suffixes['pdftex config'] = { } +suffixes['Troff fonts'] = { } + +suffixes['ls-R'] = { } + +--[[ldx-- +<p>If you wondered abou tsome of the previous mappings, how about +the next bunch:</p> +--ldx]]-- + +formats['bib'] = '' +formats['bst'] = '' +formats['mft'] = '' +formats['ist'] = '' +formats['web'] = '' +formats['cweb'] = '' +formats['MetaPost support'] = '' +formats['TeX system documentation'] = '' +formats['TeX system sources'] = '' +formats['Troff fonts'] = '' +formats['dvips config'] = '' +formats['graphic/figure'] = '' +formats['ls-R'] = '' +formats['other text files'] = '' +formats['other binary files'] = '' + +formats['gf'] = '' +formats['pk'] = '' +formats['base'] = 'MFBASES' +formats['cnf'] = '' +formats['mem'] = 'MPMEMS' +formats['mf'] = 'MFINPUTS' +formats['mfpool'] = 'MFPOOL' +formats['mppool'] = 'MPPOOL' +formats['texpool'] = 'TEXPOOL' +formats['PostScript header'] = 'TEXPSHEADERS' +formats['cmap files'] = 'CMAPFONTS' +formats['type42 fonts'] = 'T42FONTS' +formats['web2c files'] = 'WEB2C' +formats['pdftex config'] = 'PDFTEXCONFIG' +formats['texmfscripts'] = 'TEXMFSCRIPTS' +formats['bitmap font'] = '' +formats['lig files'] = 'LIGFONTS' + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-aux'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local find = string.find + +local trace_verbose = false trackers.register("resolvers.verbose", function(v) trace_verbose = v end) + +function resolvers.update_script(oldname,newname) -- oldname -> own.name, not per se a suffix + local scriptpath = "scripts/context/lua" + newname = file.addsuffix(newname,"lua") + local oldscript = resolvers.clean_path(oldname) + if trace_verbose then + logs.report("fileio","to be replaced old script %s", oldscript) + end + local newscripts = resolvers.find_files(newname) or { } + if #newscripts == 0 then + if trace_verbose then + logs.report("fileio","unable to locate new script") + end + else + for i=1,#newscripts do + local newscript = resolvers.clean_path(newscripts[i]) + if trace_verbose then + logs.report("fileio","checking new script %s", newscript) + end + if oldscript == newscript then + if trace_verbose then + logs.report("fileio","old and new script are the same") + end + elseif not find(newscript,scriptpath) then + if trace_verbose then + logs.report("fileio","new script should come from %s",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_verbose then + logs.report("fileio","invalid new script name") + end + else + local newdata = io.loaddata(newscript) + if newdata then + if trace_verbose then + logs.report("fileio","old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_verbose then + logs.report("fileio","unable to load new script") + end + end + end end end -input.moreinfo = [[ -more information about ConTeXt and the tools that come with it can be found at: -maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context -webpage : http://www.pragma-ade.nl / http://tex.aanhet.net -wiki : http://contextgarden.net -]] +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['data-tmf'] = { + version = 1.001, + comment = "companion to luat-lib.tex", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- loads *.tmf files in minimal tree roots (to be optimized and documented) + +function resolvers.check_environment(tree) + logs.simpleline() + os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) + os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) + os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) + os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) + logs.simpleline() + logs.simple("preset : TEXPATH => %s", os.getenv('TEXPATH')) + logs.simple("preset : TEXOS => %s", os.getenv('TEXOS')) + logs.simple("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) + logs.simple("preset : TMP => %s", os.getenv('TMP')) + logs.simple('') +end -function input.help(banner,message) - if not input.verbose then - input.verbose = true - -- input.report(banner,"\n") +function resolvers.load_environment(name) -- todo: key=value as well as lua + local f = io.open(name) + if f then + for line in f:lines() do + if line:find("^[%%%#]") then + -- skip comment + else + local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") + if how then + value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) + if how == "=" or how == "<<" then + os.setenv(key,value) + elseif how == "?" or how == "??" then + os.setenv(key,os.getenv(key) or value) + elseif how == "<" or how == "+=" then + if os.getenv(key) then + os.setenv(key,os.getenv(key) .. io.fileseparator .. value) + else + os.setenv(key,value) + end + elseif how == ">" or how == "=+" then + if os.getenv(key) then + os.setenv(key,value .. io.pathseparator .. os.getenv(key)) + else + os.setenv(key,value) + end + end + end + end + end + f:close() end - input.report(banner,"\n") - input.report("") - input.reportlines(message) - if input.moreinfo and input.moreinfo ~= "" then - input.report("") - input.reportlines(input.moreinfo) +end + +function resolvers.load_tree(tree) + if tree and tree ~= "" then + local setuptex = 'setuptex.tmf' + if lfs.attributes(tree, "mode") == "directory" then -- check if not nil + setuptex = tree .. "/" .. setuptex + else + setuptex = tree + end + if io.exists(setuptex) then + resolvers.check_environment(tree) + resolvers.load_environment(setuptex) + end end end -logs.set_level('error') -logs.set_method('tex') +end -- of closure + +do -- create closure to overcome 200 locals limit if not modules then modules = { } end modules ['luat-sta'] = { version = 1.001, @@ -7992,6 +9287,8 @@ if not modules then modules = { } end modules ['luat-sta'] = { license = "see context related readme files" } +-- this code is used in the updater + states = states or { } states.data = states.data or { } states.hash = states.hash or { } @@ -8018,27 +9315,32 @@ end function states.set_by_tag(tag,key,value,default,persistent) local d, h = states.data[tag], states.hash[tag] if d then - local dkey, hkey = key, key - local pre, post = key:match("(.+)%.([^%.]+)$") - if pre and post then - for k in pre:gmatch("[^%.]+") do - local dk = d[k] - if not dk then - dk = { } - d[k] = dk + if type(d) == "table" then + local dkey, hkey = key, key + local pre, post = key:match("(.+)%.([^%.]+)$") + if pre and post then + for k in pre:gmatch("[^%.]+") do + local dk = d[k] + if not dk then + dk = { } + d[k] = dk + end + d = dk end - d = dk + dkey, hkey = post, key end - dkey, hkey = post, key - end - if type(value) == nil then - value = value or default - elseif persistent then - value = value or d[dkey] or default - else - value = value or default + if type(value) == nil then + value = value or default + elseif persistent then + value = value or d[dkey] or default + else + value = value or default + end + d[dkey], h[hkey] = value, value + elseif type(d) == "string" then + -- weird + states.data[tag], states.hash[tag] = value, value end - d[dkey], h[hkey] = value, value end end @@ -8158,7 +9460,6 @@ end --~ }, --~ } - --~ states.save("teststate", "update") --~ states.load("teststate", "update") @@ -8169,6 +9470,8 @@ end --~ states.load("teststate", "update") --~ print(states.get_by_tag("update","rsync.server","unknown")) + +end -- of closure -- end library merge own = { } -- not local @@ -8178,27 +9481,43 @@ own.libs = { -- todo: check which ones are really needed 'l-lpeg.lua', 'l-table.lua', 'l-io.lua', - 'l-md5.lua', 'l-number.lua', 'l-set.lua', 'l-os.lua', 'l-file.lua', + 'l-md5.lua', 'l-dir.lua', 'l-boolean.lua', 'l-math.lua', - 'l-xml.lua', -- 'l-unicode.lua', - 'l-utils.lua', -- 'l-tex.lua', - 'luat-lib.lua', - 'luat-inp.lua', - 'luat-log.lua', --- 'luat-zip.lua', --- 'luat-tex.lua', --- 'luat-kps.lua', - 'luat-tmp.lua', - 'luat-log.lua', - 'luat-sta.lua', + 'l-utils.lua', +-- 'l-xml.lua', + 'lxml-tab.lua', + 'lxml-pth.lua', + 'lxml-ent.lua', + 'lxml-mis.lua', + 'trac-tra.lua', + 'luat-env.lua', + 'trac-inf.lua', + 'trac-log.lua', + 'data-res.lua', + 'data-tmp.lua', + 'data-pre.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-tex.lua', +-- 'data-bin.lua', + 'data-zip.lua', + 'data-crl.lua', +-- 'data-lua.lua', + 'data-kps.lua', -- so that we can replace kpsewhich + 'data-aux.lua', -- updater + 'data-tmf.lua', -- tree files + -- needed ? + 'luat-sta.lua', -- states } -- We need this hack till luatex is fixed. @@ -8236,11 +9555,11 @@ local function locate_libs() end end -if not input then +if not resolvers then locate_libs() end -if not input then +if not resolvers then print("") print("Mtxrun is unable to start up due to lack of libraries. You may") print("try to run 'lua mtxrun.lua --selfmerge' in the path where this") @@ -8249,216 +9568,12 @@ if not input then os.exit() end -input.instance = input.reset() -input.banner = 'MtxRun' -utils.report = input.report - -local instance = input.instance - - --- use os.env or environment when available - -function input.check_environment(tree) - input.report('') - os.setenv('TMP', os.getenv('TMP') or os.getenv('TEMP') or os.getenv('TMPDIR') or os.getenv('HOME')) - os.setenv('TEXOS', os.getenv('TEXOS') or ("texmf-" .. os.currentplatform())) - os.setenv('TEXPATH', (tree or "tex"):gsub("\/+$",'')) - os.setenv('TEXMFOS', os.getenv('TEXPATH') .. "/" .. os.getenv('TEXOS')) - input.report('') - input.report("preset : TEXPATH => %s", os.getenv('TEXPATH')) - input.report("preset : TEXOS => %s", os.getenv('TEXOS')) - input.report("preset : TEXMFOS => %s", os.getenv('TEXMFOS')) - input.report("preset : TMP => %s", os.getenv('TMP')) - input.report('') -end - -function input.load_environment(name) -- todo: key=value as well as lua - local f = io.open(name) - if f then - for line in f:lines() do - if line:find("^[%%%#]") then - -- skip comment - else - local key, how, value = line:match("^(.-)%s*([<=>%?]+)%s*(.*)%s*$") - if how then - value = value:gsub("%%(.-)%%", function(v) return os.getenv(v) or "" end) - if how == "=" or how == "<<" then - os.setenv(key,value) - elseif how == "?" or how == "??" then - os.setenv(key,os.getenv(key) or value) - elseif how == "<" or how == "+=" then - if os.getenv(key) then - os.setenv(key,os.getenv(key) .. io.fileseparator .. value) - else - os.setenv(key,value) - end - elseif how == ">" or how == "=+" then - if os.getenv(key) then - os.setenv(key,value .. io.pathseparator .. os.getenv(key)) - else - os.setenv(key,value) - end - end - end - end - end - f:close() - end -end - -function input.load_tree(tree) - if tree and tree ~= "" then - local setuptex = 'setuptex.tmf' - if lfs.attributes(tree, "mode") == "directory" then -- check if not nil - setuptex = tree .. "/" .. setuptex - else - setuptex = tree - end - if io.exists(setuptex) then - input.check_environment(tree) - input.load_environment(setuptex) - end - end -end - --- md5 extensions - --- maybe md.md5 md.md5hex md.md5HEX - -if not md5 then md5 = { } end - -if not md5.sum then - function md5.sum(k) - return string.rep("x",16) - end -end - -function md5.hexsum(k) - return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02x", string.byte(c)) end)) -end - -function md5.HEXsum(k) - return (string.gsub(md5.sum(k), ".", function(c) return string.format("%02X", string.byte(c)) end)) -end - --- file extensions +logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) -file.needs_updating_threshold = 1 +local instance = resolvers.reset() -function file.needs_updating(oldname,newname) -- size modification access change - local oldtime = lfs.attributes(oldname, modification) - local newtime = lfs.attributes(newname, modification) - if newtime >= oldtime then - return false - elseif oldtime - newtime < file.needs_updating_threshold then - return false - else - return true - end -end - -function file.checksum(name) - if md5 then - local data = io.loaddata(name) - if data then - return md5.HEXsum(data) - end - end - return nil -end - -function file.loadchecksum(name) - if md5 then - local data = io.loaddata(name .. ".md5") - return data and data:gsub("%s","") - end - return nil -end - -function file.savechecksum(name, checksum) - if not checksum then checksum = file.checksum(name) end - if checksum then - io.savedata(name .. ".md5",checksum) - return checksum - end - return nil -end - -os.arch = os.arch or function() - return os.resultof("uname -m") or "linux" -end - -function os.currentplatform(name, default) - local name = os.name or os.platform or name -- os.name is built in, os.platform is mine - if name then - if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" then - return "mswin" - elseif name == "linux" then - local architecture = os.arch() - if architecture:find("x86_64") then - return "linux-64" - elseif architecture:find("ppc") then - return "linux-ppc" - else - return "linux" - end - elseif name == "macosx" then - local architecture = os.arch() - if architecture:find("i386") then - return "osx-intel" - else - return "osx-ppc" - end - elseif name == "freebsd" then - return "freebsd" - end - end - return default or name -end - --- it starts here - -input.runners = { } -input.runners.applications = { } - -input.runners.applications.lua = "luatex --luaonly" -input.runners.applications.pl = "perl" -input.runners.applications.py = "python" -input.runners.applications.rb = "ruby" - -input.runners.suffixes = { - 'rb', 'lua', 'py', 'pl' -} - -input.runners.registered = { - texexec = { 'texexec.rb', true }, - texutil = { 'texutil.rb', true }, - texfont = { 'texfont.pl', true }, - texshow = { 'texshow.pl', false }, - - makempy = { 'makempy.pl', true }, - mptopdf = { 'mptopdf.pl', true }, - pstopdf = { 'pstopdf.rb', true }, - --- examplex = { 'examplex.rb', false }, - concheck = { 'concheck.rb', false }, - - runtools = { 'runtools.rb', true }, - textools = { 'textools.rb', true }, - tmftools = { 'tmftools.rb', true }, - ctxtools = { 'ctxtools.rb', true }, - rlxtools = { 'rlxtools.rb', true }, - pdftools = { 'pdftools.rb', true }, - mpstools = { 'mpstools.rb', true }, - exatools = { 'exatools.rb', true }, - xmltools = { 'xmltools.rb', true }, - luatools = { 'luatools.lua', true }, - mtxtools = { 'mtxtools.rb', true }, - - pdftrimwhite = { 'pdftrimwhite.pl', false } -} - -if not messages then messages = { } end +runners = runners or { } -- global +messages = messages or { } messages.help = [[ --script run an mtx script (--noquotes) @@ -8488,43 +9603,78 @@ messages.help = [[ --launch (--all) launch files like manuals, assumes os support --intern run script using built in libraries + +--usekpse use kpse as fallback (when no mkiv and cache installed, often slower) +--forcekpse force using kpse (handy when no mkiv and cache installed but less functionality) ]] -function input.runners.my_prepare_a() - input.resetconfig() - input.identify_cnf() - input.load_lua() - input.expand_variables() - input.load_cnf() - input.expand_variables() -end +runners.applications = { + ["lua"] = "luatex --luaonly", + ["luc"] = "luatex --luaonly", + ["pl"] = "perl", + ["py"] = "python", + ["rb"] = "ruby", +} -function input.runners.my_prepare_b() - input.runners.my_prepare_a() - input.load_hash() - input.automount() -end +runners.suffixes = { + 'rb', 'lua', 'py', 'pl' +} -function input.runners.prepare() +runners.registered = { + texexec = { 'texexec.rb', true }, -- context mkii runner (only tool not to be luafied) + texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) + texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files + texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma + texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied + -- texwork = { \texwork.pl', false }, -- perltk based editing environment, only used at pragma + + makempy = { 'makempy.pl', true }, + mptopdf = { 'mptopdf.pl', true }, + pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) + +-- examplex = { 'examplex.rb', false }, + concheck = { 'concheck.rb', false }, + + runtools = { 'runtools.rb', true }, + textools = { 'textools.rb', true }, + tmftools = { 'tmftools.rb', true }, + ctxtools = { 'ctxtools.rb', true }, + rlxtools = { 'rlxtools.rb', true }, + pdftools = { 'pdftools.rb', true }, + mpstools = { 'mpstools.rb', true }, +-- exatools = { 'exatools.rb', true }, + xmltools = { 'xmltools.rb', true }, +-- luatools = { 'luatools.lua', true }, + mtxtools = { 'mtxtools.rb', true }, + + pdftrimwhite = { 'pdftrimwhite.pl', false } +} + +runners.launchers = { + windows = { }, + unix = { } +} + +function runners.prepare() local checkname = environment.argument("ifchanged") if checkname and checkname ~= "" then local oldchecksum = file.loadchecksum(checkname) local newchecksum = file.checksum(checkname) if oldchecksum == newchecksum then - input.report("file '%s' is unchanged",checkname) + logs.simple("file '%s' is unchanged",checkname) return "skip" else - input.report("file '%s' is changed, processing started",checkname) + logs.simple("file '%s' is changed, processing started",checkname) end file.savechecksum(checkname) end local oldname, newname = string.split(environment.argument("iftouched") or "", ",") if oldname and newname and oldname ~= "" and newname ~= "" then if not file.needs_updating(oldname,newname) then - input.report("file '%s' and '%s' have same age",oldname,newname) + logs.simple("file '%s' and '%s' have same age",oldname,newname) return "skip" else - input.report("file '%s' is older than '%s'",oldname,newname) + logs.simple("file '%s' is older than '%s'",oldname,newname) end end local tree = environment.argument('tree') or "" @@ -8532,28 +9682,27 @@ function input.runners.prepare() tree = os.getenv('TEXMFSTART_TREE') or os.getenv('TEXMFSTARTTREE') or tree end if tree and tree ~= "" then - input.load_tree(tree) + resolvers.load_tree(tree) end local env = environment.argument('environment') or "" if env and env ~= "" then for _,e in pairs(string.split(env)) do -- maybe force suffix when not given - input.load_tree(e) + resolvers.load_tree(e) end end local runpath = environment.argument("path") - if runpath and not dir.chdir(runpath) then - input.report("unable to change to path '%s'",runpath) + if runpath and not lfs.chdir(runpath) then + logs.simple("unable to change to path '%s'",runpath) return "error" end return "run" end -function input.runners.execute_script(fullname,internal) - local instance = input.instance +function runners.execute_script(fullname,internal) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then - local state = input.runners.prepare() + local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8570,41 +9719,44 @@ function input.runners.execute_script(fullname,internal) return "" end ) name = name:gsub("^script:","") - if suffix == "" and input.runners.registered[name] and input.runners.registered[name][1] then - name = input.runners.registered[name][1] + if suffix == "" and runners.registered[name] and runners.registered[name][1] then + name = runners.registered[name][1] suffix = file.extname(name) end if suffix == "" then -- loop over known suffixes - for _,s in pairs(input.runners.suffixes) do - result = input.find_file(name .. "." .. s, 'texmfscripts') + for _,s in pairs(runners.suffixes) do + result = resolvers.find_file(name .. "." .. s, 'texmfscripts') if result ~= "" then break end end - elseif input.runners.applications[suffix] then - result = input.find_file(name, 'texmfscripts') + elseif runners.applications[suffix] then + result = resolvers.find_file(name, 'texmfscripts') else -- maybe look on path - result = input.find_file(name, 'other text files') + result = resolvers.find_file(name, 'other text files') end end if result and result ~= "" then + local before, after = environment.split_arguments(fullname) -- already done + environment.arguments_before, environment.arguments_after = before, after if internal then - local before, after = environment.split_arguments(fullname) arg = { } for _,v in pairs(after) do arg[#arg+1] = v end dofile(result) else - local binary = input.runners.applications[file.extname(result)] + local binary = runners.applications[file.extname(result)] if binary and binary ~= "" then result = binary .. " " .. result end - local before, after = environment.split_arguments(fullname) local command = result .. " " .. environment.reconstruct_commandline(after,noquote) - input.report("") - input.report("executing: %s",command) - input.report("\n \n") - io.flush() + if logs.verbose then + logs.simpleline() + logs.simple("executing: %s",command) + logs.simpleline() + logs.simpleline() + io.flush() + end local code = os.exec(command) -- maybe spawn return code == 0 end @@ -8614,10 +9766,10 @@ function input.runners.execute_script(fullname,internal) return false end -function input.runners.execute_program(fullname) +function runners.execute_program(fullname) local noquote = environment.argument("noquotes") if fullname and fullname ~= "" then - local state = input.runners.prepare() + local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then @@ -8627,9 +9779,10 @@ function input.runners.execute_program(fullname) environment.initialize_arguments(after) fullname = fullname:gsub("^bin:","") local command = fullname .. " " .. (environment.reconstruct_commandline(after or "",noquote) or "") - input.report("") - input.report("executing: %s",command) - input.report("\n \n") + logs.simpleline() + logs.simple("executing: %s",command) + logs.simpleline() + logs.simpleline() io.flush() local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn return code == 0 @@ -8638,8 +9791,13 @@ function input.runners.execute_program(fullname) return false end -function input.runners.handle_stubs(create) - local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer supported +-- the --usekpse flag will fallback on kpse + +local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' +local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' + +function runners.handle_stubs(create) + local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported local windows = environment.argument('windows') or environment.argument('mswin') or false local unix = environment.argument('unix') or environment.argument('linux') or false if not windows and not unix then @@ -8649,77 +9807,77 @@ function input.runners.handle_stubs(create) windows = true end end - for _,v in pairs(input.runners.registered) do + for _,v in pairs(runners.registered) do local name, doit = v[1], v[2] if doit then local base = string.gsub(file.basename(name), "%.(.-)$", "") if create then - -- direct local command = input.runners.applications[file.extname(name)] .. " " .. name - local command = "luatex --luaonly mtxrun.lua " .. name if windows then - io.savedata(base..".bat", {"@echo off", command.." %*"}, "\013\010") - input.report("windows stub for '%s' created",base) + io.savedata(file.join(stubpath,base..".bat"),string.format(windows_stub,name)) + logs.simple("windows stub for '%s' created",base) end if unix then - io.savedata(base, {"#!/bin/sh", command..' "$@"'}, "\010") - input.report("unix stub for '%s' created",base) + io.savedata(file.join(stubpath,base),string.format(unix_stub,name)) + logs.simple("unix stub for '%s' created",base) end else - if windows and (os.remove(base..'.bat') or os.remove(base..'.cmd')) then - input.report("windows stub for '%s' removed", base) + if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then + logs.simple("windows stub for '%s' removed", base) end - if unix and (os.remove(base) or os.remove(base..'.sh')) then - input.report("unix stub for '%s' removed",base) + if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then + logs.simple("unix stub for '%s' removed",base) end end end end end -function input.runners.resolve_string(filename) +function runners.resolve_string(filename) if filename and filename ~= "" then - input.runners.report_location(input.resolve(filename)) + runners.report_location(resolvers.resolve(filename)) end end -function input.runners.locate_file(filename) +function runners.locate_file(filename) + -- differs from texmfstart where locate appends .com .exe .bat ... todo if filename and filename ~= "" then - input.runners.report_location(input.find_given_file(filename)) + runners.report_location(resolvers.find_given_file(filename)) end end -function input.runners.locate_platform() - input.runners.report_location(os.currentplatform()) +function runners.locate_platform() + runners.report_location(os.currentplatform()) end -function input.runners.report_location(result) - if input.verbose then - input.report("") +function runners.report_location(result) + if logs.verbose then + logs.simpleline() if result and result ~= "" then - input.report(result) + logs.simple(result) else - input.report("not found") + logs.simple("not found") end else io.write(result) end end -function input.runners.edit_script(filename) -- we assume that vim is present on most systems +function runners.edit_script(filename) -- we assume that vim is present on most systems local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'vim' - local rest = input.resolve(filename) + local rest = resolvers.resolve(filename) if rest ~= "" then local command = editor .. " " .. rest - if input.verbose then - input.report("") - input.report("starting editor: %s",command) - input.report("\n \n") + if logs.verbose then + logs.simpleline() + logs.simple("starting editor: %s",command) + logs.simple_line() + logs.simple_line() end os.launch(command) end end -function input.runners.save_script_session(filename, list) +function runners.save_script_session(filename, list) local t = { } for _, key in ipairs(list) do t[key] = environment.arguments[key] @@ -8727,7 +9885,7 @@ function input.runners.save_script_session(filename, list) io.savedata(filename,table.serialize(t,true)) end -function input.runners.load_script_session(filename) +function runners.load_script_session(filename) if lfs.isfile(filename) then local t = io.loaddata(filename) if t then @@ -8740,15 +9898,10 @@ function input.runners.load_script_session(filename) end end -input.runners.launchers = { - windows = { }, - unix = { } -} - -function input.launch(str) +function resolvers.launch(str) -- maybe we also need to test on mtxrun.launcher.suffix environment -- variable or on windows consult the assoc and ftype vars and such - local launchers = input.runners.launchers[os.platform] if launchers then + local launchers = runners.launchers[os.platform] if launchers then local suffix = file.extname(str) if suffix then local runner = launchers[suffix] if runner then str = runner .. " " .. str @@ -8758,41 +9911,40 @@ function input.launch(str) os.launch(str) end -function input.runners.launch_file(filename) - local instance = input.instance +function runners.launch_file(filename) instance.allresults = true - input.verbose = true + logs.setverbose(true) local pattern = environment.arguments["pattern"] if not pattern or pattern == "" then pattern = filename end if not pattern or pattern == "" then - input.report("provide name or --pattern=") + logs.simple("provide name or --pattern=") else - local t = input.find_files(pattern) + local t = resolvers.find_files(pattern) if not t or #t == 0 then - t = input.aux.find_file("*/" .. pattern,true) + t = resolvers.find_files("*/" .. pattern) end if not t or #t == 0 then - t = input.aux.find_file("*/" .. pattern .. "*",true) + t = resolvers.find_files("*/" .. pattern .. "*") end if t and #t > 0 then if environment.arguments["all"] then for _, v in pairs(t) do - input.report("launching %s", v) - input.launch(v) + logs.simple("launching %s", v) + resolvers.launch(v) end else - input.report("launching %s", t[1]) - input.launch(t[1]) + logs.simple("launching %s", t[1]) + resolvers.launch(t[1]) end else - input.report("no match for %s", pattern) + logs.simple("no match for %s", pattern) end end end -function input.runners.find_mtx_script(filename) +function runners.find_mtx_script(filename) local function found(name) local path = file.dirname(name) if path and path ~= "" then @@ -8803,10 +9955,10 @@ function input.runners.find_mtx_script(filename) end end filename = file.addsuffix(filename,"lua") - local basename = file.stripsuffix(file.basename(filename)) + local basename = file.removesuffix(file.basename(filename)) local suffix = file.extname(filename) -- qualified path, raw name - local fullname = input.aux.qualified_path(filename) and io.exists(filename) and filename + local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename if fullname and fullname ~= "" then return fullname end @@ -8818,74 +9970,84 @@ function input.runners.find_mtx_script(filename) end -- context namespace, mtx-<filename> fullname = "mtx-" .. filename - fullname = found(fullname) or input.find_file(fullname) + fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename>s fullname = "mtx-" .. basename .. "s" .. "." .. suffix - fullname = found(fullname) or input.find_file(fullname) + fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, mtx-<filename minus trailing s> fullname = "mtx-" .. basename:gsub("s$","") .. "." .. suffix - fullname = found(fullname) or input.find_file(fullname) + fullname = found(fullname) or resolvers.find_file(fullname) if fullname and fullname ~= "" then return fullname end -- context namespace, just <filename> - fullname = input.find_file(fullname) + fullname = resolvers.find_file(filename) return fullname end -function input.runners.execute_ctx_script(filename,arguments) - local fullname = input.runners.find_mtx_script(filename) +function runners.execute_ctx_script(filename,arguments) + local fullname = runners.find_mtx_script(filename) or "" + -- retyr after generate but only if --autogenerate + if fullname == "" and environment.argument("autogenerate") then -- might become the default + instance.renewcache = true + logs.setverbose(true) + resolvers.load() + -- + fullname = runners.find_mtx_script(filename) or "" + end -- that should do it - if fullname and fullname ~= "" then - local state = input.runners.prepare() + if fullname ~= "" then + local state = runners.prepare() if state == 'error' then return false elseif state == 'skip' then return true elseif state == "run" then -- load and save ... kind of undocumented - arg = { } for _,v in pairs(arguments) do arg[#arg+1] = v end + arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end environment.initialize_arguments(arg) local loadname = environment.arguments['load'] if loadname then if type(loadname) ~= "string" then loadname = file.basename(fullname) end loadname = file.replacesuffix(loadname,"cfg") - input.runners.load_script_session(loadname) + runners.load_script_session(loadname) end filename = environment.files[1] - if input.verbose then - input.report("using script: %s\n",fullname) + if logs.verbose then + logs.simple("using script: %s\n",fullname) end dofile(fullname) local savename = environment.arguments['save'] - if savename and input.runners.save_list and not table.is_empty(input.runners.save_list or { }) then + if savename and runners.save_list and not table.is_empty(runners.save_list or { }) then if type(savename) ~= "string" then savename = file.basename(fullname) end savename = file.replacesuffix(savename,"cfg") - input.runners.save_script_session(savename, input.runners.save_list) + runners.save_script_session(savename, runners.save_list) end return true end else - input.verbose = true + logs.setverbose(true) filename = file.addsuffix(filename,"lua") if filename == "" then - input.report("unknown script, no name given") - elseif input.aux.qualified_path(filename) then - input.report("unknown script '%s'",filename) + logs.simple("unknown script, no name given") + elseif file.is_qualified_path(filename) then + logs.simple("unknown script '%s'",filename) else - input.report("unknown script '%s' or 'mtx-%s'",filename,filename) + logs.simple("unknown script '%s' or 'mtx-%s'",filename,filename) end return false end end -input.report("%s\n",banner) +function runners.timed(action) + statistics.timed(action) +end -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script @@ -8895,14 +10057,79 @@ local filename = environment.files[1] or "" local ok = true local before, after = environment.split_arguments(filename) +environment.arguments_before, environment.arguments_after = before, after environment.initialize_arguments(before) instance.engine = environment.argument("engine") or 'luatex' instance.progname = environment.argument("progname") or 'context' instance.lsrmode = environment.argument("lsr") or false -input.verbose = environment.argument("verbose") or false -input.runners.my_prepare_b() +-- maybe the unset has to go to this level + +if environment.argument("usekpse") or environment.argument("forcekpse") then + + os.setenv("engine","") + os.setenv("progname","") + + local remapper = { + otf = "opentype fonts", + ttf = "truetype fonts", + ttc = "truetype fonts", + pfb = "type1 fonts", + other = "other text files", + } + + local function kpse_initialized() + texconfig.kpse_init = true + local t = os.clock() + local k = kpse.original.new("luatex",instance.progname) + local dummy = k:find_file("mtxrun.lua") -- so that we're initialized + logs.simple("kpse fallback with progname '%s' initialized in %s seconds",instance.progname,os.clock()-t) + kpse_initialized = function() return k end + return k + end + + local find_file = resolvers.find_file + local show_path = resolvers.show_path + + if environment.argument("forcekpse") then + + function resolvers.find_file(name,kind) + return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + end + function resolvers.show_path(name) + return (kpse_initialized():show_path(name)) or "" + end + + elseif environment.argument("usekpse") then + + resolvers.load() + + function resolvers.find_file(name,kind) + local found = find_file(name,kind) or "" + if found ~= "" then + return found + else + return (kpse_initialized():find_file(resolvers.clean_path(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + end + end + function resolvers.show_path(name) + local found = show_path(name) or "" + if found ~= "" then + return found + else + return (kpse_initialized():show_path(name)) or "" + end + end + + end + +else + + resolvers.load() + +end + if environment.argument("selfmerge") then -- embed used libraries @@ -8911,47 +10138,47 @@ elseif environment.argument("selfclean") then -- remove embedded libraries utils.merger.selfclean(own.name) elseif environment.argument("selfupdate") then - input.verbose = true - input.update_script(own.name,"mtxrun") + logs.setverbose(true) + resolvers.update_script(own.name,"mtxrun") elseif environment.argument("ctxlua") or environment.argument("internal") then -- run a script by loading it (using libs) - ok = input.runners.execute_script(filename,true) -elseif environment.argument("script") then + ok = runners.execute_script(filename,true) +elseif environment.argument("script") or environment.argument("s") then -- run a script by loading it (using libs), pass args - ok = input.runners.execute_ctx_script(filename,after) + ok = runners.execute_ctx_script(filename,after) elseif environment.argument("execute") then -- execute script - ok = input.runners.execute_script(filename) + ok = runners.execute_script(filename) elseif environment.argument("direct") then -- equals bin: - ok = input.runners.execute_program(filename) + ok = runners.execute_program(filename) elseif environment.argument("edit") then -- edit file - input.runners.edit_script(filename) + runners.edit_script(filename) elseif environment.argument("launch") then - input.runners.launch_file(filename) + runners.launch_file(filename) elseif environment.argument("make") then -- make stubs - input.runners.handle_stubs(true) + runners.handle_stubs(true) elseif environment.argument("remove") then -- remove stub - input.runners.handle_stubs(false) + runners.handle_stubs(false) elseif environment.argument("resolve") then -- resolve string - input.runners.resolve_string(filename) + runners.resolve_string(filename) elseif environment.argument("locate") then -- locate file - input.runners.locate_file(filename) + runners.locate_file(filename) elseif environment.argument("platform")then -- locate platform - input.runners.locate_platform() + runners.locate_platform() elseif environment.argument("help") or filename=='help' or filename == "" then - input.help(banner,messages.help) + logs.help(messages.help) -- execute script elseif filename:find("^bin:") then - ok = input.runners.execute_program(filename) + ok = runners.execute_program(filename) else - ok = input.runners.execute_script(filename) + ok = runners.execute_script(filename) end if os.platform == "unix" then diff --git a/scripts/context/lua/x-ldx.lua b/scripts/context/lua/x-ldx.lua index 67d5f925c..af5c9c0c8 100644 --- a/scripts/context/lua/x-ldx.lua +++ b/scripts/context/lua/x-ldx.lua @@ -335,10 +335,10 @@ The next function wraps it all in one call: --ldx]]-- function ldx.convert(luaname,ldxname) - if not file.is_readable(luaname) then + if not file.isreadable(luaname) then luaname = luaname .. ".lua" end - if file.is_readable(luaname) then + if file.isreadable(luaname) then if not ldxname then ldxname = file.replacesuffix(luaname,"ldx") end |