diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/context/lua/luatools.lua | 845 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-context.lua | 3 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-server.lua | 83 | ||||
| -rw-r--r-- | scripts/context/lua/mtxrun.lua | 920 |
4 files changed, 1046 insertions, 805 deletions
diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua index b092c5050..89e5e0eb4 100644 --- a/scripts/context/lua/luatools.lua +++ b/scripts/context/lua/luatools.lua @@ -15,7 +15,7 @@ -- the future. As long as Luatex is under development the -- interfaces and names of functions may change. -banner = "version 1.1.1 - 2006+ - PRAGMA ADE / CONTEXT" +banner = "version 1.2.0 - 2006+ - PRAGMA ADE / CONTEXT" texlua = true -- For the sake of independence we optionally can merge the library @@ -374,7 +374,6 @@ local patterns_escapes = { ["]"] = "%]", } - function string:pattesc() return (self:gsub(".",patterns_escapes)) end @@ -1687,14 +1686,22 @@ end --~ print("../test/" .. " == " .. file.collapse_path("../test/")) --~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) +--~ function file.collapse_path(str) +--~ local ok, n = false, 0 +--~ while not ok do +--~ ok = true +--~ str, n = str:gsub("[^%./]+/%.%./", function(s) +--~ ok = false +--~ return "" +--~ end) +--~ end +--~ return (str:gsub("/%./","/")) +--~ end + function file.collapse_path(str) - local ok, n = false, 0 - while not ok do - ok = true - str, n = str:gsub("[^%./]+/%.%./", function(s) - ok = false - return "" - end) + local n = 1 + while n > 0 do + str, n = str:gsub("([^/%.]+/%.%./)","") end return (str:gsub("/%./","/")) end @@ -1762,7 +1769,7 @@ function url.hashed(str) path = s[3], query = s[4], fragment = s[5], - original=str + original = str } end @@ -2560,14 +2567,32 @@ os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") -- -- for k,v in pairs(arg) do print(k,v) end -if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then - arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil -end - -- environment if not environment then environment = { } end +environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" + +local ownpath = nil -- we could use a metatable here + +function environment.ownpath() + if not ownpath then + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,environment.ownbin) + if lfs.isfile(b..".exe") or lfs.isfile(b) then + ownpath = p + break + end + end + if not ownpath then ownpath = '.' end + end + return ownpath +end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + environment.arguments = { } environment.files = { } environment.sorted_argument_keys = nil @@ -2679,6 +2704,10 @@ end -- additional functionality becomes available. We will split this -- module in components when we're done with prototyping. +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow + -- This is the first code I wrote for LuaTeX, so it needs some cleanup. -- To be considered: hash key lowercase, first entry in table filename @@ -2689,9 +2718,6 @@ end -- Beware, loading and saving is overloaded in luat-tmp! --- todo: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) --- todo: check escaping in find etc, too much, too slow - if not versions then versions = { } end versions['luat-inp'] = 1.001 if not environment then environment = { } end if not file then file = { } end @@ -2710,6 +2736,8 @@ if not input.hashers then input.hashers = { } end -- load databases if not input.generators then input.generators = { } end -- generate databases if not input.filters then input.filters = { } end -- conversion filters +local format = string.format + input.locators.notfound = { nil } input.hashers.notfound = { nil } input.generators.notfound = { nil } @@ -2719,6 +2747,7 @@ input.banner = nil input.verbose = false input.debug = false input.cnfname = 'texmf.cnf' +input.luaname = 'texmfcnf.lua' input.lsrname = 'ls-R' input.luasuffix = '.tma' input.lucsuffix = '.tmc' @@ -2808,12 +2837,14 @@ function input.reset() instance.files = { } instance.remap = { } instance.configuration = { } + instance.setup = { } instance.order = { } instance.found = { } instance.foundintrees = { } instance.kpsevars = { } instance.hashes = { } instance.cnffiles = { } + instance.luafiles = { } instance.lists = { } instance.remember = true instance.diskcache = true @@ -2833,7 +2864,6 @@ function input.reset() instance.stoptime = 0 instance.validfile = function(path,name) return true end instance.data = { } -- only for loading - instance.sortdata = false instance.force_suffixes = true instance.dummy_path_expr = "^!*unset/*$" instance.fakepaths = { } @@ -2846,7 +2876,7 @@ function input.reset() end else -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF','SELFAUTOPARENT'}) do + for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do local e = os.getenv(v) if e then -- input.report("setting",v,"to",input.bare_variable(e)) @@ -2874,7 +2904,7 @@ function input.reset_hashes(instance) instance.found = { } end -function input.bare_variable(str) +function input.bare_variable(str) -- assumes str is a string -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) end @@ -2959,7 +2989,7 @@ do instance.stoptime = stoptime instance.loadtime = instance.loadtime + loadtime if report then - input.report('load time', string.format("%0.3f",loadtime)) + input.report('load time', format("%0.3f",loadtime)) end return loadtime end @@ -2970,7 +3000,7 @@ do end function input.elapsedtime(instance) - return string.format("%0.3f",(instance and instance.loadtime) or 0) + return format("%0.3f",(instance and instance.loadtime) or 0) end function input.report_loadtime(instance) @@ -2986,15 +3016,19 @@ function input.env(instance,key) end function input.osenv(instance,key) - if instance.environment[key] == nil then - local e = os.getenv(key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] if e == nil then - instance.environment[key] = "" -- false + -- value = "" -- false else - instance.environment[key] = input.bare_variable(e) + value = input.bare_variable(e) end + ie[key] = value end - return instance.environment[key] or "" + return value or "" end -- we follow a rather traditional approach: @@ -3005,66 +3039,103 @@ end -- for the moment we don't expect a configuration file in a zip function input.identify_cnf(instance) + -- we no longer support treepath and rootpath (was handy for testing); + -- also we now follow the stupid route: if not set then just assume *one* + -- cnf file under texmf (i.e. distribution) if #instance.cnffiles == 0 then - if instance.treepath ~= "" then - -- this is a special purpose branch, not really used - if instance.rootpath ~= "" then - local t = instance.treepath:splitchr(',') - for k,v in ipairs(t) do - t[k] = file.join(instance.rootpath,v) + if input.env(instance,'TEXMFCNF') == "" then + local ownpath = environment.ownpath() or "." + if ownpath then + -- beware, this is tricky on my own system because at that location I do have + -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess + local function locate(filename,list) + local ownroot = input.normalize_name(file.join(ownpath,"../..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + ownroot = input.normalize_name(file.join(ownpath,"..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + input.verbose = true + input.report("error", "unable to identify cnf file") + return + end + end + local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself + if not lfs.isfile(texmfcnf) then + texmfcnf = file.join(ownroot,"texmf/web2c",filename) + if not lfs.isfile(texmfcnf) then + input.verbose = true + input.report("error", "unable to locate",filename) + return + end + end + table.insert(list,texmfcnf) + local ie = instance.environment + if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end + if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end + end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + if #instance.luafiles == 0 and instance.cnffiles == 0 then + input.verbose = true + input.report("error", "unable to locate",filename) + os.exit() end - instance.treepath = table.concat(t,',') + -- here we also assume then TEXMF is set in the distribution, if this trickery is + -- used in the minimals, then users who don't use setuptex are on their own with + -- regards to extra trees + else + input.verbose = true + input.report("error", "unable to identify own path") + os.exit() end - local t = instance.treepath:splitchr(',') - instance.environment['TEXMF'] = input.bare_variable(instance.treepath) - instance.environment['TEXMFCNF'] = file.join(t[1] or '.','texmf/web2c') - end - if instance.rootpath ~= "" then - -- this assumes a single path, maybe do an expanded split here too - instance.environment['TEXMFCNF'] = file.join(instance.rootpath,'texmf/web2c') - instance.environment['SELFAUTOPARENT'] = instance.rootpath - end - if input.env(instance,'TEXMFCNF') ~= "" then + else local t = input.split_path(input.env(instance,'TEXMFCNF')) t = input.aux.expanded_path(instance,t) input.aux.expand_vars(instance,t) - for _,v in ipairs(t) do - table.insert(instance.cnffiles,file.join(v,input.cnfname)) - end - elseif input.env(instance,'SELFAUTOPARENT') == '.' then - table.insert(instance.cnffiles,file.join('.',input.cnfname)) - else - for _,v in ipairs({'texmf-local','texmf'}) do - table.insert(instance.cnffiles,file.join(input.env(instance,'SELFAUTOPARENT'),v,'web2c',input.cnfname)) + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) + end + end end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) end end end function input.load_cnf(instance) + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + input.aux.load_cnf(instance,fname) + end + end -- instance.cnffiles contain complete names now ! if #instance.cnffiles == 0 then input.report("no cnf files found (TEXMFCNF may not be set/known)") else instance.rootpath = instance.cnffiles[1] for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = fname:gsub("\\",'/') -- needed? + instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure if instance.lsrmode then - input.loadconfigdata(instance,instance.cnffiles) + loadoldconfigdata() elseif instance.diskcache and not instance.renewcache then - input.loadconfig(instance,instance.cnffiles) + input.loadoldconfig(instance,instance.cnffiles) if instance.loaderror then - input.loadconfigdata(instance,instance.cnffiles) - input.saveconfig(instance) + loadoldconfigdata() + input.saveoldconfig(instance) end else - input.loadconfigdata(instance,instance.cnffiles) + loadoldconfigdata() if instance.renewcache then - input.saveconfig(instance) + input.saveoldconfig(instance) end end input.aux.collapse_cnf_data(instance) @@ -3072,40 +3143,34 @@ function input.load_cnf(instance) input.checkconfigdata(instance) end -function input.loadconfigdata(instance) - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) +function input.load_lua(instance) + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + input.loadnewconfig(instance) + input.aux.collapse_cnf_data(instance) end + input.checkconfigdata(instance) end -if os.env then - function input.aux.collapse_cnf_data(instance) - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) - end - end - end - end - end -else - function input.aux.collapse_cnf_data(instance) - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - local e = os.getenv(k) - if e then - instance.environment[k] = input.bare_variable(e) - instance.variables[k] = instance.environment[k] - else - instance.variables[k] = input.bare_variable(v) - instance.kpsevars[k] = true - end +function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in pairs(c) do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = input.bare_variable(v) end end end @@ -3116,11 +3181,11 @@ function input.aux.load_cnf(instance,fname) fname = input.clean_path(fname) local lname = fname:gsub("%.%a+$",input.luasuffix) local f = io.open(lname) - if f then + if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_data(instance,dname,'configuration',file.basename(lname)) + input.aux.load_configuration(instance,dname,lname) instance.order[#instance.order+1] = instance.configuration[dname] end else @@ -3135,7 +3200,7 @@ function input.aux.load_cnf(instance,fname) end local data = instance.configuration[dname] while true do - line = f:read() + local line, n = f:read(), 0 if line then while true do -- join lines line, n = line:gsub("\\%s*$", "") @@ -3146,7 +3211,7 @@ function input.aux.load_cnf(instance,fname) end end if not line:find("^[%%#]") then - k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") if k and v and not data[k] then data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") instance.kpsevars[k] = true @@ -3220,6 +3285,7 @@ end function input.locatelists(instance) for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do + path = file.collapse_path(path) input.report("locating list of",path) input.locatedatabase(instance,input.normalize_name(path)) end @@ -3256,7 +3322,7 @@ function input.loadfiles(instance) end function input.hashers.tex(instance,tag,name) - input.aux.load_data(instance,tag,'files') + input.aux.load_files(instance,tag) end -- generators: @@ -3333,7 +3399,7 @@ do end end action() - input.report(string.format("%s files found on %s directories with %s uppercase remappings",n,m,r)) + input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) else local fullname = file.join(specification,input.lsrname) local path = '.' @@ -3386,7 +3452,7 @@ end -- is more convenient. function input.splitconfig(instance) - for i,c in ipairs(instance.order) do + for i,c in ipairs(instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -3420,14 +3486,7 @@ function input.join_path(str) return str end end ---~ function input.splitexpansions(instance) ---~ for k,v in pairs(instance.expansions) do ---~ local t = file.split_path(v) ---~ if #t > 1 then ---~ instance.expansions[k] = t ---~ end ---~ end ---~ end + function input.splitexpansions(instance) for k,v in pairs(instance.expansions) do local t, h = { }, { } @@ -3447,7 +3506,7 @@ end -- end of split/join code -function input.saveconfig(instance) +function input.saveoldconfig(instance) input.splitconfig(instance) input.aux.save_data(instance, 'configuration', nil) input.joinconfig(instance) @@ -3460,44 +3519,83 @@ input.configbanner = [[ -- not copyrighted. [HH & TH] ]] -function input.aux.save_data(instance, dataname, check) - for cachename, files in pairs(instance[dataname]) do - local name = file.join(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.configbanner) - f:write("\n") - f:write("if not texmf then texmf = { } end\n") - f:write("if not texmf.data then texmf.data = { } end\n") - f:write("\n") - f:write("texmf.data.type = '" .. dataname .. "'\n") - f:write("texmf.data.version = '" .. input.cacheversion .. "'\n") - f:write("texmf.data.date = '" .. os.date("%Y-%m-%d") .. "'\n") - f:write("texmf.data.time = '" .. os.date("%H:%M:%S") .. "'\n") - f:write('texmf.data.content = {\n') - local function dump(k,v) - if not check or check(v,k) then -- path, name - if type(v) == 'string' then - f:write("\t['" .. k .. "'] = '" .. v .. "',\n") - elseif #v == 1 then - f:write("\t['" .. k .. "'] = '" .. v[1] .. "',\n") - else - f:write("\t['" .. k .. "'] = {'" .. table.concat(v,"','").. "'},\n") - end +function input.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local concat = table.concat + local sorted = table.sortedkeys + local function dump(k,v,m) + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," + else + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," + end + end + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sorted(files)) do + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sorted(fk)) do + t[#t+1] = dump(kk,fk[kk],"\t\t") end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") end - if instance.sortdata then - for _, k in pairs(table.sortedkeys(files)) do - dump(k,files[k]) + end + else + for k, v in pairs(files) do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in pairs(v) do + t[#t+1] = dump(kk,vv,"\t\t") end + t[#t+1] = "\t}," else - for k, v in pairs(files) do - dump(k,v) + t[#t+1] = dump(k,v,"\t") + end + end + end + t[#t+1] = "}" + return concat(t,"\n") +end + +if not texmf then texmf = {} end -- no longer needed, at least not here + +function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(instance[dataname]) do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix + input.report("preparing " .. dataname .. " for", luaname) + for k, v in pairs(files) do + if not check or check(v,k) then -- path, name + if type(v) == "table" and #v == 1 then + files[k] = v[1] end + else + files[k] = nil -- false end - f:write('}\n') + end + local data = { + type = dataname, + root = cachename, + version = input.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local f = io.open(luaname,'w') + if f then + input.report("saving " .. dataname .. " in", luaname) + f:write(input.serialize(data)) f:close() input.report("compiling " .. dataname .. " to", lucname) if not utils.lua.compile(luaname,lucname) then @@ -3510,49 +3608,106 @@ function input.aux.save_data(instance, dataname, check) end end -function input.loadconfig(instance) - instance.configuration, instance.order, instance.loaderror = { }, { }, false - if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do - local dname = file.dirname(cnf) - input.aux.load_data(instance,dname,'configuration') - instance.order[#instance.order+1] = instance.configuration[dname] - if instance.loaderror then break end +function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == input.cacheversion then + input.report("loading",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = data.content + else + input.report("skipping",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = { } + instance.loaderror = true end + else + input.report("skipping",dataname,"for",pathname,"from",filename) end - input.joinconfig(instance) end -if not texmf then texmf = {} end -if not texmf.data then texmf.data = {} end +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } -function input.aux.load_data(instance,pathname,dataname,filename) - if not filename or (filename == "") then - filename = dataname .. input.lucsuffix - end - local blob = loadfile(file.join(pathname,filename)) - if not blob then - filename = dataname .. input.luasuffix - blob = loadfile(file.join(pathname,filename)) - end +function input.aux.load_texmfcnf(instance,dataname,pathname) + local filename = file.join(pathname,input.luaname) + local blob = loadfile(filename) if blob then - blob() - if (texmf.data.type == dataname) and (texmf.data.version == input.cacheversion) and texmf.data.content then - input.report("loading",dataname,"for",pathname,"from",filename) - instance[dataname][pathname] = texmf.data.content + local data = blob() + if data then + input.report("loading","configuration file",filename) + if true then + -- flatten to variable.progname + local t = { } + for k, v in pairs(data) do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in pairs(v) do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance[dataname][pathname] = t + else + instance[dataname][pathname] = data + end else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping","configuration file",filename) instance[dataname][pathname] = { } instance.loaderror = true end + else + input.report("skipping","configuration file",filename) + end +end + +function input.aux.load_configuration(instance,dname,lname) + input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +end +function input.aux.load_files(instance,tag) + input.aux.load_data(instance,tag,'files') +end + +function input.resetconfig(instance) + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function input.loadnewconfig(instance) + for _, cnf in ipairs(instance.luafiles) do + local dname = file.dirname(cnf) + input.aux.load_texmfcnf(instance,'setup',dname) + instance.order[#instance.order+1] = instance.setup[dname] + if instance.loaderror then break end + end +end + +function input.loadoldconfig(instance) + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + input.aux.load_configuration(instance,dname) + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end end - texmf.data.content = { } + input.joinconfig(instance) end function input.expand_variables(instance) instance.expansions = { } - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.engine end +--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath + if instance.engine ~= "" then instance.environment['engine'] = instance.engine end + if instance.progname ~= "" then instance.environment['progname'] = instance.progname end for k,v in pairs(instance.environment) do local a, b = k:match("^(%a+)%_(.*)%s*$") if a and b then @@ -3643,53 +3798,6 @@ function input.is_expansion(instance,name) return input.aux.is_entry(instance,instance.expansions,name) end -function input.aux.list(instance,list) - local pat = string.upper(instance.pattern or "","") - for _,key in pairs(table.sortedkeys(list)) do - if (instance.pattern=="") or string.find(key:upper(),pat) then - if instance.kpseonly then - if instance.kpsevars[key] then - print(key .. "=" .. input.aux.tabstr(list[key])) - end - elseif instance.kpsevars[key] then - print('K ' .. key .. "=" .. input.aux.tabstr(list[key])) - else - print('E ' .. key .. "=" .. input.aux.tabstr(list[key])) - end - end - end -end - -function input.list_variables(instance) - input.aux.list(instance,instance.variables) -end -function input.list_expansions(instance) - input.aux.list(instance,instance.expansions) -end - -function input.list_configurations(instance) - for _,key in pairs(table.sortedkeys(instance.kpsevars)) do - if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then - print(key.."\n") - for i,c in ipairs(instance.order) do - local str = c[key] - if str then - print("\t" .. i .. "\t\t" .. input.aux.tabstr(str)) - end - end - print() - end - end -end - -function input.aux.tabstr(str) - if type(str) == 'table' then - return table.concat(str," | ") - else - return str - end -end - function input.simplified_list(str) if type(str) == 'table' then return str -- troubles ; ipv , in texmf @@ -3713,23 +3821,6 @@ function input.unexpanded_path(instance,str) return file.join_path(input.unexpanded_path_list(instance,str)) end ---~ function input.expanded_path_list(instance,str) ---~ if not str then ---~ return { } ---~ elseif instance.savelists then ---~ -- engine+progname hash ---~ str = str:gsub("%$","") ---~ if not instance.lists[str] then -- cached ---~ local lst = input.split_path(input.expansion(instance,str)) ---~ instance.lists[str] = input.aux.expanded_path(instance,lst) ---~ end ---~ return instance.lists[str] ---~ else ---~ local lst = input.split_path(input.expansion(instance,str)) ---~ return input.aux.expanded_path(instance,lst) ---~ end ---~ end - do local done = { } @@ -4142,6 +4233,8 @@ do return original end + input.normalize_name = file.collapse_path + end function input.aux.register_in_trees(instance,name) @@ -4507,7 +4600,10 @@ end function input.load(instance) input.starttiming(instance) + input.resetconfig(instance) input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) input.load_cnf(instance) input.expand_variables(instance) input.load_hash(instance) @@ -4894,28 +4990,45 @@ caches.more = caches.more or "context" caches.direct = false -- true is faster but may need huge amounts of memory caches.trace = false caches.tree = false -caches.temp = caches.temp or os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("VARTEXMF") or os.getenv("TEXMFVAR") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil -caches.paths = caches.paths or { caches.temp } +caches.paths = caches.paths or nil caches.force = false input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true -if caches.temp and caches.temp ~= "" and lfs.attributes(caches.temp,"mode") ~= "directory" then - if caches.force or io.ask(string.format("Should I create the cache path %s?",caches.temp), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(caches.temp) +function caches.temp(instance) + local function checkpath(cachepath) + if not cachepath or cachepath == "" then + return nil + elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then + return cachepath + elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + return (lfs.attributes(cachepath,"mode") == "directory") and cachepath + else + return nil + end end -end -if not caches.temp or caches.temp == "" then - print("\nfatal error: there is no valid cache path defined\n") - os.exit() -elseif lfs.attributes(caches.temp,"mode") ~= "directory" then - print(string.format("\nfatal error: cache path %s is not a directory\n",caches.temp)) - os.exit() + local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") + cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + if not cachepath then + cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil + cachepath = checkpath(cachepath) + end + if not cachepath then + print("\nfatal error: there is no valid cache path defined\n") + os.exit() + elseif lfs.attributes(cachepath,"mode") ~= "directory" then + print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + function caches.temp(instance) + return cachepath + end + return cachepath end function caches.configpath(instance) return table.concat(instance.cnffiles,";") ---~ return input.expand_var(instance,"TEXMFCNF") end function caches.hashed(tree) @@ -4933,19 +5046,8 @@ end function caches.setpath(instance,...) if not caches.path then - if lfs and instance then - for _,v in pairs(caches.paths) do - for _,vv in pairs(input.expanded_path_list(instance,v)) do - if lfs.isdir(vv) then - caches.path = vv - break - end - end - if caches.path then break end - end - end if not caches.path then - caches.path = caches.temp + caches.path = caches.temp(instance) end caches.path = input.clean_path(caches.path) -- to be sure if lfs then @@ -4969,6 +5071,12 @@ function caches.setpath(instance,...) return caches.path end +function caches.definepath(instance,category,subcategory) + return function() + return caches.setpath(instance,category,subcategory) + end +end + function caches.setluanames(path,name) return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" end @@ -5034,19 +5142,35 @@ do -- local report end end + local allocated = { } + + -- tracing + function containers.define(category, subcategory, version, enabled) - if category and subcategory then - return { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(texmf.instance,category,subcategory), - } - else - return nil + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches.setpath(texmf.instance,category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end end end @@ -5103,129 +5227,35 @@ end -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. +local save_data = input.aux.save_data + +input.cachepath = nil + function input.aux.save_data(instance, dataname, check) - for cachename, files in pairs(instance[dataname]) do - local name + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + save_data(instance, dataname, check, function(cachename,dataname) if input.usecache then - name = file.join(caches.setpath(instance,"trees"),caches.hashed(cachename)) - else - name = file.join(cachename,dataname) - end - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " in", luaname) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false - end - end - local data = { - type = dataname, - root = cachename, - version = input.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = files, - } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - -- f:write(table.serialize(data,'return')) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) - os.remove(lucname) - end + return file.join(input.cachepath(),caches.hashed(cachename)) else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + return file.join(cachename,dataname) end - end + end) end -function input.serialize(files) - -- This version is somewhat optimized for the kind of - -- tables that we deal with, so it's much faster than - -- the generic serializer. This makes sense because - -- luatools and mtxtools are called frequently. Okay, - -- we pay a small price for properly tabbed tables. - local t = { } - local concat = table.concat - local sorted = table.sortedkeys - local function dump(k,v,m) - if type(v) == 'string' then - return m .. "['" .. k .. "']='" .. v .. "'," - elseif #v == 1 then - return m .. "['" .. k .. "']='" .. v[1] .. "'," - else - return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," - end - end - t[#t+1] = "return {" - if instance.sortdata then - for _, k in pairs(sorted(files)) do - local fk = files[k] - if type(fk) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do - t[#t+1] = dump(kk,fk[kk],"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,fk,"\t") - end - end - else - for k, v in pairs(files) do - if type(v) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do - t[#t+1] = dump(kk,vv,"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,v,"\t") - end - end - end - t[#t+1] = "}" - return concat(t,"\n") -end +local load_data = input.aux.load_data function input.aux.load_data(instance,pathname,dataname,filename) - local luaname, lucname, pname, fname - if input.usecache then - pname, fname = caches.setpath(instance,"trees"), caches.hashed(pathname) - filename = file.join(pname,fname) - else - if not filename or (filename == "") then - filename = dataname - end - pname, fname = pathname, filename - end - luaname = file.join(pname,fname) .. input.luasuffix - lucname = file.join(pname,fname) .. input.lucsuffix - local blob = loadfile(lucname) - if not blob then - blob = loadfile(luaname) - end - if blob then - local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) - instance[dataname][pathname] = data.content + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + load_data(instance,pathname,dataname,filename,function(dataname,filename) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(pathname)) else - input.report("skipping",dataname,"for",pathname,"from",filename) - instance[dataname][pathname] = { } - instance.loaderror = true + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) end - else - input.report("skipping",dataname,"for",pathname,"from",filename) - end + end) end -- we will make a better format, maybe something xml or just text or lua @@ -5853,7 +5883,6 @@ if texconfig and not texlua then if not texmf.instance then -- prevent a second loading - texmf.instance = input.reset() texmf.instance.progname = environment.progname or 'context' texmf.instance.engine = environment.engine or 'luatex' @@ -6297,7 +6326,10 @@ if environment.arguments["minimize"] then end function input.my_prepare_a(instance) + input.resetconfig(instance) input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) input.load_cnf(instance) input.expand_variables(instance) end @@ -6472,6 +6504,51 @@ function input.my_run_format(instance,name,data,more) end end +-- helpers for verbose lists + +input.listers = input.listers or { } + +local function tabstr(str) + if type(str) == 'table' then + return table.concat(str," | ") + else + return str + end +end + +local function list(instance,list) + local pat = string.upper(instance.pattern or "","") + for _,key in pairs(table.sortedkeys(list)) do + if instance.pattern == "" or string.find(key:upper(),pat) then + if instance.kpseonly then + if instance.kpsevars[key] then + print(format("%s=%s",key,tabstr(list[key]))) + end + else + print(format('%s %s=%s',(instance.kpsevars[key] and 'K') or 'E',key,tabstr(list[key]))) + end + end + end +end + +function input.listers.variables (instance) list(instance,instance.variables ) end +function input.listers.expansions(instance) list(instance,instance.expansions) end + +function input.listers.configurations(instance) + for _,key in pairs(table.sortedkeys(instance.kpsevars)) do + if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then + print(key.."\n") + for i,c in ipairs(instance.order) do + local str = c[key] + if str then + print(format("\t%s\t\t%s",i,input.aux.tabstr(str))) + end + end + print() + end + end +end + input.report(banner,"\n") local ok = true @@ -6545,13 +6622,13 @@ elseif environment.arguments["selfupdate"] then input.update_script(instance,own.name,"luatools") elseif environment.arguments["variables"] or environment.arguments["show-variables"] then input.my_prepare_a(instance) - input.list_variables(instance) + input.listers.variables(instance) elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then input.my_prepare_a(instance) - input.list_expansions(instance) + input.listers.expansions(instance) elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then input.my_prepare_a(instance) - input.list_configurations(instance) + input.listers.configurations(instance) elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then if not input.verbose then input.verbose = true diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index ae0ee1b8e..6c444d531 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -583,7 +583,8 @@ function scripts.context.run(ctxdata) -- end else - input.error("no format found with name " .. formatname) + input.verbose = true + input.report("error", "no format found with name " .. formatname) end end end diff --git a/scripts/context/lua/mtx-server.lua b/scripts/context/lua/mtx-server.lua new file mode 100644 index 000000000..293bc0c1c --- /dev/null +++ b/scripts/context/lua/mtx-server.lua @@ -0,0 +1,83 @@ +if not modules then modules = { } end modules ['mtx-server'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +-- The starting point was stripped down webserver.lua by Samuel +-- Saint-Pettersen (as downloaded on 21-5-2008) which only served +-- html and was not configureable. In due time I will extend the +-- next code. Eventually we may move code to l-server. + +scripts = scripts or { } +scripts.webserver = scripts.webserver or {} + +local socket = require("socket") + +local function message(str) + return string.format("<h1>%s</h1>",str) +end + +function scripts.webserver.run(configuration) + local server = assert(socket.bind("*", tonumber(configuration.port or 8080))) + while true do + local client = server:accept() + client:settimeout(configuration.timeout or 60) + local request, e = client:receive() + if e then + client:send(message("404 Not Found")) + else + -- GET /showcase.pdf HTTP/1.1 + local filename = request:match("GET (.+) HTTP/.*$") -- todo: more clever + -- filename = filename:gsub("%%(%d%d)",function(c) return string.char(tonumber(c,16)) end) + filename = socket.url.unescape(filename) + if filename == nil or filename == "" then + filename = configuration.index or "index.html" + end + -- todo chunked + local fullname = file.join(configuration.root,filename) + local data = io.loaddata(fullname) + if data and data ~= "" then + local result + client:send("HTTP/1.1 200 OK\r\n") + client:send("Connection: close\r\n") + if filename:find(".pdf$") then -- todo: special handler + client:send(string.format("Content-Length: %s\r\n",#data)) + client:send("Content-Type: application/pdf\r\n") + else + client:send("Content-Type: text/html\r\n") + end + client:send("\r\n") + client:send(data) + client:send("\r\n") + else + client:send(message("404 Not Found")) + end + end + client:close() + end +end + + +banner = banner .. " | webserver " + +messages.help = [[ +--start start server +--port port to listen to +--root server root +--index index file +]] + +if environment.argument("start") then + scripts.webserver.run { + port = environment.argument("port") or "8080", + root = environment.argument("root") or ".", -- "e:/websites/www.pragma-ade.com", + index = environment.argument("index") or "index.html", + } +else + input.help(banner,messages.help) +end diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 63605b42f..040214178 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -388,7 +388,6 @@ local patterns_escapes = { ["]"] = "%]", } - function string:pattesc() return (self:gsub(".",patterns_escapes)) end @@ -1701,14 +1700,22 @@ end --~ print("../test/" .. " == " .. file.collapse_path("../test/")) --~ print("a/a" .. " == " .. file.collapse_path("a/b/c/../../a")) +--~ function file.collapse_path(str) +--~ local ok, n = false, 0 +--~ while not ok do +--~ ok = true +--~ str, n = str:gsub("[^%./]+/%.%./", function(s) +--~ ok = false +--~ return "" +--~ end) +--~ end +--~ return (str:gsub("/%./","/")) +--~ end + function file.collapse_path(str) - local ok, n = false, 0 - while not ok do - ok = true - str, n = str:gsub("[^%./]+/%.%./", function(s) - ok = false - return "" - end) + local n = 1 + while n > 0 do + str, n = str:gsub("([^/%.]+/%.%./)","") end return (str:gsub("/%./","/")) end @@ -2307,7 +2314,7 @@ do local remove, nsremap, resolvens = table.remove, xml.xmlns, xml.resolvens - local stack, top, dt, at, xmlns, errorstr = {}, {}, {}, {}, {}, nil + local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} local mt = { __tostring = xml.text } @@ -2383,14 +2390,6 @@ do dt[#dt+1] = text end end - --~ local function add_special(what, spacing, text) - --~ if #spacing > 0 then - --~ dt[#dt+1] = spacing - --~ end - --~ top = stack[#stack] -- hm, left over 1 - --~ setmetatable(top, mt) -- hm, left over 2 - --~ dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } - --~ end local function add_special(what, spacing, text) if #spacing > 0 then dt[#dt+1] = spacing @@ -2450,12 +2449,20 @@ do local somecomment = C((1 - endcomment )^0) local somecdata = C((1 - endcdata )^0) + function entity(k,v) entities[k] = v end + local begindoctype = open * P("!DOCTYPE") local enddoctype = close - local publicdoctype = P("PUBLIC") * somespace * value * somespace * value * somespace * balanced^0 - local systemdoctype = P("SYSTEM") * somespace * value * somespace * balanced^0 - local simpledoctype = (1-close)^1 * balanced^0 - local somedoctype = C((somespace * P(publicdoctype + systemdoctype + simpledoctype) * optionalspace)^0) + local beginset = P("[") + local endset = P("]") + local doctypename = C((1-somespace)^0) + local elementdoctype = optionalspace * P("<!ELEMENT") * (1-close)^0 * close + local entitydoctype = optionalspace * P("<!ENTITY") * somespace * (doctypename * somespace * value)/entity * optionalspace * close + local publicdoctype = doctypename * somespace * P("PUBLIC") * somespace * value * somespace * value * somespace + local systemdoctype = doctypename * somespace * P("SYSTEM") * somespace * value * somespace + local definitiondoctype= doctypename * somespace * beginset * P(elementdoctype + entitydoctype)^0 * optionalspace * endset + local simpledoctype = (1-close)^1 -- * balanced^0 + local somedoctype = C((somespace * (publicdoctype + systemdoctype + definitiondoctype + simpledoctype) * optionalspace)^0) local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end @@ -2481,9 +2488,11 @@ do children = text + V("parent") + emptyelement + comment + cdata + instruction, } - function xml.convert(data, no_root, strip_cm_and_dt) + -- todo: xml.new + properties like entities and strip and such (store in root) + + function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result = {}, {}, {}, {}, nil, nil + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} stack[#stack+1] = top top.dt = { } dt = top.dt @@ -2502,7 +2511,7 @@ do result = stack[1] end if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={} } + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do @@ -2629,6 +2638,22 @@ do elseif not nocommands then local ec = e.command if ec ~= nil then -- we can have all kind of types + +if e.special then -- todo test for true/false + local etg, edt = e.tg, e.dt + local spc = specialconverter and specialconverter[etg] + if spc then +--~ print("SPECIAL",etg,table.serialize(specialconverter), spc) + local result = spc(edt[1]) + if result then + handle(result) + return + else + -- no need to handle any further + end + end +end + local xc = xml.command if xc then xc(e,ec) @@ -2678,12 +2703,18 @@ do end end end - if ern and xml.trace_remap then - if ats then - ats[#ats+1] = format("xmlns:remapped='%s'",ern) - else - ats = { format("xmlns:remapped='%s'",ern) } - end + if ern and xml.trace_remap and ern ~= ens then +--~ if ats then +--~ ats[#ats+1] = format("xmlns:remapped='%s'",ern) +--~ else +--~ ats = { format("xmlns:remapped='%s'",ern) } +--~ end +--~ if ats then +--~ ats[#ats+1] = format("remappedns='%s'",ens or '-') +--~ else +--~ ats = { format("remappedns='%s'",ens or '-') } +--~ end +ens = ern end if ens ~= "" then if edt and #edt > 0 then @@ -3261,10 +3292,10 @@ do end end functions.name = function(root,k,n) --- way too fuzzy + -- way too fuzzy local found if not k or not n then - local ns, tg = root.ns, root.tg + local ns, tg = root.rn or root.ns or "", root.tg if not tg then for i=1,#root do local e = root[i] @@ -3273,10 +3304,10 @@ do break end end - elseif ns == "" then - return tg - else + elseif ns ~= "" then return ns .. ":" .. tg + else + return tg end elseif n == 0 then local e = root[k] @@ -3296,6 +3327,7 @@ do end end else +--~ print(k,n,#root) for i=k+1,#root,1 do local e = root[i] if type(e) == "table" then @@ -3309,7 +3341,7 @@ do end end if found then - local ns, tg = found.ns, found.tg + local ns, tg = found.rn or found.ns or "", found.tg if ns ~= "" then return ns .. ":" .. tg else @@ -3709,7 +3741,7 @@ do end function xml.filters.tag(root,pattern,n) local tag = "" - xml.traverse(root, xml.lpath(pattern), function(r,d,k) + traverse(root, lpath(pattern), function(r,d,k) tag = xml.functions.tag(d,k,n and tonumber(n)) return true end) @@ -3717,7 +3749,7 @@ do end function xml.filters.name(root,pattern,n) local tag = "" - xml.traverse(root, xml.lpath(pattern), function(r,d,k) + traverse(root, lpath(pattern), function(r,d,k) tag = xml.functions.name(d,k,n and tonumber(n)) return true end) @@ -3748,8 +3780,13 @@ do local attribute_filter = xml.filters.attributes local default_filter = xml.filters.default + -- todo: also hash, could be gc'd + function xml.filter(root,pattern) local kind, a, b, c = parser:match(pattern) +--~ if xml.trace_lpath then +--~ print(pattern,kind,a,b,c) +--~ end if kind == 1 or kind == 3 then return (filters[b] or default_filter)(root,a,c) elseif kind == 2 then @@ -4292,7 +4329,7 @@ do if unicode and unicode.utf8 then xml.entities = xml.entities or { } -- xml.entities.handler == function function xml.entities.handler(e) - return format("[s]",e) + return format("[%s]",e) end local char = unicode.utf8.char @@ -4301,6 +4338,8 @@ do if unicode and unicode.utf8 then return char(tonumber(s,16)) end + local entities = xml.entities -- global entities + function utfize(root) local d = root.dt for k=1,#d do @@ -4318,8 +4357,6 @@ do if unicode and unicode.utf8 then xml.utfize = utfize - local entities = xml.entities - local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks if e:find("#x") then return char(tonumber(e:sub(3),16)) @@ -4334,20 +4371,24 @@ do if unicode and unicode.utf8 then end end - function xml.resolve_entities(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if dk:find("&.-;") then - d[k] = dk:gsub("&(.-);",resolve) + local function resolve_entities(root) + if not root.special or root.tg == "@rt@" then + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + if dk:find("&.-;") then + d[k] = dk:gsub("&(.-);",resolve) + end + else + resolve_entities(dk) end - else - utfize(dk) end end end + xml.resolve_entities = resolve_entities + function xml.utfize_text(str) if str:find("&#") then return (str:gsub("&#x(.-);",toutf)) @@ -4372,6 +4413,18 @@ do if unicode and unicode.utf8 then end end + -- experimental, this will be done differently + + function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in pairs(documententities) do + allentities[k] = v + end + end + end + end end -- xml.set_text_cleanup(xml.show_text_entities) @@ -4590,14 +4643,32 @@ os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") -- -- for k,v in pairs(arg) do print(k,v) end -if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then - arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil -end - -- environment if not environment then environment = { } end +environment.ownbin = environment.ownbin or arg[-2] or arg[-1] or arg[0] or "luatex" + +local ownpath = nil -- we could use a metatable here + +function environment.ownpath() + if not ownpath then + for p in string.gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local b = file.join(p,environment.ownbin) + if lfs.isfile(b..".exe") or lfs.isfile(b) then + ownpath = p + break + end + end + if not ownpath then ownpath = '.' end + end + return ownpath +end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + environment.arguments = { } environment.files = { } environment.sorted_argument_keys = nil @@ -4709,6 +4780,10 @@ end -- additional functionality becomes available. We will split this -- module in components when we're done with prototyping. +-- TODO: os.getenv -> os.env[] +-- TODO: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) +-- TODO: check escaping in find etc, too much, too slow + -- This is the first code I wrote for LuaTeX, so it needs some cleanup. -- To be considered: hash key lowercase, first entry in table filename @@ -4719,9 +4794,6 @@ end -- Beware, loading and saving is overloaded in luat-tmp! --- todo: instances.[hashes,cnffiles,configurations,522] -> ipairs (alles check, sneller) --- todo: check escaping in find etc, too much, too slow - if not versions then versions = { } end versions['luat-inp'] = 1.001 if not environment then environment = { } end if not file then file = { } end @@ -4740,6 +4812,8 @@ if not input.hashers then input.hashers = { } end -- load databases if not input.generators then input.generators = { } end -- generate databases if not input.filters then input.filters = { } end -- conversion filters +local format = string.format + input.locators.notfound = { nil } input.hashers.notfound = { nil } input.generators.notfound = { nil } @@ -4749,6 +4823,7 @@ input.banner = nil input.verbose = false input.debug = false input.cnfname = 'texmf.cnf' +input.luaname = 'texmfcnf.lua' input.lsrname = 'ls-R' input.luasuffix = '.tma' input.lucsuffix = '.tmc' @@ -4838,12 +4913,14 @@ function input.reset() instance.files = { } instance.remap = { } instance.configuration = { } + instance.setup = { } instance.order = { } instance.found = { } instance.foundintrees = { } instance.kpsevars = { } instance.hashes = { } instance.cnffiles = { } + instance.luafiles = { } instance.lists = { } instance.remember = true instance.diskcache = true @@ -4863,7 +4940,6 @@ function input.reset() instance.stoptime = 0 instance.validfile = function(path,name) return true end instance.data = { } -- only for loading - instance.sortdata = false instance.force_suffixes = true instance.dummy_path_expr = "^!*unset/*$" instance.fakepaths = { } @@ -4876,7 +4952,7 @@ function input.reset() end else -- we will access os.env frequently - for k,v in pairs({'HOME','TEXMF','TEXMFCNF','SELFAUTOPARENT'}) do + for k,v in pairs({'HOME','TEXMF','TEXMFCNF'}) do local e = os.getenv(v) if e then -- input.report("setting",v,"to",input.bare_variable(e)) @@ -4904,7 +4980,7 @@ function input.reset_hashes(instance) instance.found = { } end -function input.bare_variable(str) +function input.bare_variable(str) -- assumes str is a string -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) end @@ -4989,7 +5065,7 @@ do instance.stoptime = stoptime instance.loadtime = instance.loadtime + loadtime if report then - input.report('load time', string.format("%0.3f",loadtime)) + input.report('load time', format("%0.3f",loadtime)) end return loadtime end @@ -5000,7 +5076,7 @@ do end function input.elapsedtime(instance) - return string.format("%0.3f",(instance and instance.loadtime) or 0) + return format("%0.3f",(instance and instance.loadtime) or 0) end function input.report_loadtime(instance) @@ -5016,15 +5092,19 @@ function input.env(instance,key) end function input.osenv(instance,key) - if instance.environment[key] == nil then - local e = os.getenv(key) + local ie = instance.environment + local value = ie[key] + if value == nil then + -- local e = os.getenv(key) + local e = os.env[key] if e == nil then - instance.environment[key] = "" -- false + -- value = "" -- false else - instance.environment[key] = input.bare_variable(e) + value = input.bare_variable(e) end + ie[key] = value end - return instance.environment[key] or "" + return value or "" end -- we follow a rather traditional approach: @@ -5035,66 +5115,103 @@ end -- for the moment we don't expect a configuration file in a zip function input.identify_cnf(instance) + -- we no longer support treepath and rootpath (was handy for testing); + -- also we now follow the stupid route: if not set then just assume *one* + -- cnf file under texmf (i.e. distribution) if #instance.cnffiles == 0 then - if instance.treepath ~= "" then - -- this is a special purpose branch, not really used - if instance.rootpath ~= "" then - local t = instance.treepath:splitchr(',') - for k,v in ipairs(t) do - t[k] = file.join(instance.rootpath,v) + if input.env(instance,'TEXMFCNF') == "" then + local ownpath = environment.ownpath() or "." + if ownpath then + -- beware, this is tricky on my own system because at that location I do have + -- the raw tree that ends up in the zip; i.e. I cannot test this kind of mess + local function locate(filename,list) + local ownroot = input.normalize_name(file.join(ownpath,"../..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + ownroot = input.normalize_name(file.join(ownpath,"..")) + if not lfs.isdir(file.join(ownroot,"texmf")) then + input.verbose = true + input.report("error", "unable to identify cnf file") + return + end + end + local texmfcnf = file.join(ownroot,"texmf-local/web2c",filename) -- for minimals and myself + if not lfs.isfile(texmfcnf) then + texmfcnf = file.join(ownroot,"texmf/web2c",filename) + if not lfs.isfile(texmfcnf) then + input.verbose = true + input.report("error", "unable to locate",filename) + return + end + end + table.insert(list,texmfcnf) + local ie = instance.environment + if not ie['SELFAUTOPARENT'] then ie['SELFAUTOPARENT'] = ownroot end + if not ie['TEXMFCNF'] then ie['TEXMFCNF'] = file.dirname(texmfcnf) end end - instance.treepath = table.concat(t,',') + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) + if #instance.luafiles == 0 and instance.cnffiles == 0 then + input.verbose = true + input.report("error", "unable to locate",filename) + os.exit() + end + -- here we also assume then TEXMF is set in the distribution, if this trickery is + -- used in the minimals, then users who don't use setuptex are on their own with + -- regards to extra trees + else + input.verbose = true + input.report("error", "unable to identify own path") + os.exit() end - local t = instance.treepath:splitchr(',') - instance.environment['TEXMF'] = input.bare_variable(instance.treepath) - instance.environment['TEXMFCNF'] = file.join(t[1] or '.','texmf/web2c') - end - if instance.rootpath ~= "" then - -- this assumes a single path, maybe do an expanded split here too - instance.environment['TEXMFCNF'] = file.join(instance.rootpath,'texmf/web2c') - instance.environment['SELFAUTOPARENT'] = instance.rootpath - end - if input.env(instance,'TEXMFCNF') ~= "" then + else local t = input.split_path(input.env(instance,'TEXMFCNF')) t = input.aux.expanded_path(instance,t) input.aux.expand_vars(instance,t) - for _,v in ipairs(t) do - table.insert(instance.cnffiles,file.join(v,input.cnfname)) - end - elseif input.env(instance,'SELFAUTOPARENT') == '.' then - table.insert(instance.cnffiles,file.join('.',input.cnfname)) - else - for _,v in ipairs({'texmf-local','texmf'}) do - table.insert(instance.cnffiles,file.join(input.env(instance,'SELFAUTOPARENT'),v,'web2c',input.cnfname)) + local function locate(filename,list) + for _,v in ipairs(t) do + local texmfcnf = input.normalize_name(file.join(v,filename)) + if lfs.isfile(texmfcnf) then + table.insert(list,texmfcnf) + end + end end + locate(input.luaname,instance.luafiles) + locate(input.cnfname,instance.cnffiles) end end end function input.load_cnf(instance) + local function loadoldconfigdata() + for _, fname in ipairs(instance.cnffiles) do + input.aux.load_cnf(instance,fname) + end + end -- instance.cnffiles contain complete names now ! if #instance.cnffiles == 0 then input.report("no cnf files found (TEXMFCNF may not be set/known)") else instance.rootpath = instance.cnffiles[1] for k,fname in ipairs(instance.cnffiles) do - instance.cnffiles[k] = fname:gsub("\\",'/') -- needed? + instance.cnffiles[k] = input.normalize_name(fname:gsub("\\",'/')) end for i=1,3 do instance.rootpath = file.dirname(instance.rootpath) end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure if instance.lsrmode then - input.loadconfigdata(instance,instance.cnffiles) + loadoldconfigdata() elseif instance.diskcache and not instance.renewcache then - input.loadconfig(instance,instance.cnffiles) + input.loadoldconfig(instance,instance.cnffiles) if instance.loaderror then - input.loadconfigdata(instance,instance.cnffiles) - input.saveconfig(instance) + loadoldconfigdata() + input.saveoldconfig(instance) end else - input.loadconfigdata(instance,instance.cnffiles) + loadoldconfigdata() if instance.renewcache then - input.saveconfig(instance) + input.saveoldconfig(instance) end end input.aux.collapse_cnf_data(instance) @@ -5102,40 +5219,34 @@ function input.load_cnf(instance) input.checkconfigdata(instance) end -function input.loadconfigdata(instance) - for _, fname in ipairs(instance.cnffiles) do - input.aux.load_cnf(instance,fname) +function input.load_lua(instance) + if #instance.luafiles == 0 then + -- yet harmless + else + instance.rootpath = instance.luafiles[1] + for k,fname in ipairs(instance.luafiles) do + instance.luafiles[k] = input.normalize_name(fname:gsub("\\",'/')) + end + for i=1,3 do + instance.rootpath = file.dirname(instance.rootpath) + end + instance.rootpath = input.normalize_name(instance.rootpath) + instance.environment['SELFAUTOPARENT'] = instance.rootpath -- just to be sure + input.loadnewconfig(instance) + input.aux.collapse_cnf_data(instance) end + input.checkconfigdata(instance) end -if os.env then - function input.aux.collapse_cnf_data(instance) - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - if instance.environment[k] then - instance.variables[k] = instance.environment[k] - else - instance.kpsevars[k] = true - instance.variables[k] = input.bare_variable(v) - end - end - end - end - end -else - function input.aux.collapse_cnf_data(instance) - for _,c in ipairs(instance.order) do - for k,v in pairs(c) do - if not instance.variables[k] then - local e = os.getenv(k) - if e then - instance.environment[k] = input.bare_variable(e) - instance.variables[k] = instance.environment[k] - else - instance.variables[k] = input.bare_variable(v) - instance.kpsevars[k] = true - end +function input.aux.collapse_cnf_data(instance) -- potential optmization: pass start index (setup and configuration are shared) + for _,c in ipairs(instance.order) do + for k,v in pairs(c) do + if not instance.variables[k] then + if instance.environment[k] then + instance.variables[k] = instance.environment[k] + else + instance.kpsevars[k] = true + instance.variables[k] = input.bare_variable(v) end end end @@ -5146,11 +5257,11 @@ function input.aux.load_cnf(instance,fname) fname = input.clean_path(fname) local lname = fname:gsub("%.%a+$",input.luasuffix) local f = io.open(lname) - if f then + if f then -- this will go f:close() local dname = file.dirname(fname) if not instance.configuration[dname] then - input.aux.load_data(instance,dname,'configuration',file.basename(lname)) + input.aux.load_configuration(instance,dname,lname) instance.order[#instance.order+1] = instance.configuration[dname] end else @@ -5165,7 +5276,7 @@ function input.aux.load_cnf(instance,fname) end local data = instance.configuration[dname] while true do - line = f:read() + local line, n = f:read(), 0 if line then while true do -- join lines line, n = line:gsub("\\%s*$", "") @@ -5176,7 +5287,7 @@ function input.aux.load_cnf(instance,fname) end end if not line:find("^[%%#]") then - k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") + local k, v = (line:gsub("%s*%%.*$","")):match("%s*(.-)%s*=%s*(.-)%s*$") if k and v and not data[k] then data[k] = (v:gsub("[%%#].*",'')):gsub("~", "$HOME") instance.kpsevars[k] = true @@ -5250,6 +5361,7 @@ end function input.locatelists(instance) for _, path in pairs(input.simplified_list(input.expansion(instance,'TEXMF'))) do + path = file.collapse_path(path) input.report("locating list of",path) input.locatedatabase(instance,input.normalize_name(path)) end @@ -5286,7 +5398,7 @@ function input.loadfiles(instance) end function input.hashers.tex(instance,tag,name) - input.aux.load_data(instance,tag,'files') + input.aux.load_files(instance,tag) end -- generators: @@ -5363,7 +5475,7 @@ do end end action() - input.report(string.format("%s files found on %s directories with %s uppercase remappings",n,m,r)) + input.report(format("%s files found on %s directories with %s uppercase remappings",n,m,r)) else local fullname = file.join(specification,input.lsrname) local path = '.' @@ -5416,7 +5528,7 @@ end -- is more convenient. function input.splitconfig(instance) - for i,c in ipairs(instance.order) do + for i,c in ipairs(instance) do for k,v in pairs(c) do if type(v) == 'string' then local t = file.split_path(v) @@ -5450,14 +5562,7 @@ function input.join_path(str) return str end end ---~ function input.splitexpansions(instance) ---~ for k,v in pairs(instance.expansions) do ---~ local t = file.split_path(v) ---~ if #t > 1 then ---~ instance.expansions[k] = t ---~ end ---~ end ---~ end + function input.splitexpansions(instance) for k,v in pairs(instance.expansions) do local t, h = { }, { } @@ -5477,7 +5582,7 @@ end -- end of split/join code -function input.saveconfig(instance) +function input.saveoldconfig(instance) input.splitconfig(instance) input.aux.save_data(instance, 'configuration', nil) input.joinconfig(instance) @@ -5490,44 +5595,83 @@ input.configbanner = [[ -- not copyrighted. [HH & TH] ]] -function input.aux.save_data(instance, dataname, check) - for cachename, files in pairs(instance[dataname]) do - local name = file.join(cachename,dataname) - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - f:write(input.configbanner) - f:write("\n") - f:write("if not texmf then texmf = { } end\n") - f:write("if not texmf.data then texmf.data = { } end\n") - f:write("\n") - f:write("texmf.data.type = '" .. dataname .. "'\n") - f:write("texmf.data.version = '" .. input.cacheversion .. "'\n") - f:write("texmf.data.date = '" .. os.date("%Y-%m-%d") .. "'\n") - f:write("texmf.data.time = '" .. os.date("%H:%M:%S") .. "'\n") - f:write('texmf.data.content = {\n') - local function dump(k,v) - if not check or check(v,k) then -- path, name - if type(v) == 'string' then - f:write("\t['" .. k .. "'] = '" .. v .. "',\n") - elseif #v == 1 then - f:write("\t['" .. k .. "'] = '" .. v[1] .. "',\n") - else - f:write("\t['" .. k .. "'] = {'" .. table.concat(v,"','").. "'},\n") - end +function input.serialize(files) + -- This version is somewhat optimized for the kind of + -- tables that we deal with, so it's much faster than + -- the generic serializer. This makes sense because + -- luatools and mtxtools are called frequently. Okay, + -- we pay a small price for properly tabbed tables. + local t = { } + local concat = table.concat + local sorted = table.sortedkeys + local function dump(k,v,m) + if type(v) == 'string' then + return m .. "['" .. k .. "']='" .. v .. "'," + elseif #v == 1 then + return m .. "['" .. k .. "']='" .. v[1] .. "'," + else + return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," + end + end + t[#t+1] = "return {" + if instance.sortdata then + for _, k in pairs(sorted(files)) do + local fk = files[k] + if type(fk) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for _, kk in pairs(sorted(fk)) do + t[#t+1] = dump(kk,fk[kk],"\t\t") end + t[#t+1] = "\t}," + else + t[#t+1] = dump(k,fk,"\t") end - if instance.sortdata then - for _, k in pairs(table.sortedkeys(files)) do - dump(k,files[k]) + end + else + for k, v in pairs(files) do + if type(v) == 'table' then + t[#t+1] = "\t['" .. k .. "']={" + for kk,vv in pairs(v) do + t[#t+1] = dump(kk,vv,"\t\t") end + t[#t+1] = "\t}," else - for k, v in pairs(files) do - dump(k,v) + t[#t+1] = dump(k,v,"\t") + end + end + end + t[#t+1] = "}" + return concat(t,"\n") +end + +if not texmf then texmf = {} end -- no longer needed, at least not here + +function input.aux.save_data(instance, dataname, check, makename) -- untested without cache overload + for cachename, files in pairs(instance[dataname]) do + local name = (makename or file.join)(cachename,dataname) + local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix + input.report("preparing " .. dataname .. " for", luaname) + for k, v in pairs(files) do + if not check or check(v,k) then -- path, name + if type(v) == "table" and #v == 1 then + files[k] = v[1] end + else + files[k] = nil -- false end - f:write('}\n') + end + local data = { + type = dataname, + root = cachename, + version = input.cacheversion, + date = os.date("%Y-%m-%d"), + time = os.date("%H:%M:%S"), + content = files, + } + local f = io.open(luaname,'w') + if f then + input.report("saving " .. dataname .. " in", luaname) + f:write(input.serialize(data)) f:close() input.report("compiling " .. dataname .. " to", lucname) if not utils.lua.compile(luaname,lucname) then @@ -5540,49 +5684,106 @@ function input.aux.save_data(instance, dataname, check) end end -function input.loadconfig(instance) - instance.configuration, instance.order, instance.loaderror = { }, { }, false - if not instance.renewcache then - for _, cnf in ipairs(instance.cnffiles) do - local dname = file.dirname(cnf) - input.aux.load_data(instance,dname,'configuration') - instance.order[#instance.order+1] = instance.configuration[dname] - if instance.loaderror then break end +function input.aux.load_data(instance,pathname,dataname,filename,makename) -- untested without cache overload + filename = ((not filename or (filename == "")) and dataname) or filename + filename = (makename and makename(dataname,filename)) or file.join(pathname,filename) + local blob = loadfile(filename .. input.lucsuffix) or loadfile(filename .. input.luasuffix) + if blob then + local data = blob() + if data and data.content and data.type == dataname and data.version == input.cacheversion then + input.report("loading",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = data.content + else + input.report("skipping",dataname,"for",pathname,"from",filename) + instance[dataname][pathname] = { } + instance.loaderror = true end + else + input.report("skipping",dataname,"for",pathname,"from",filename) end - input.joinconfig(instance) end -if not texmf then texmf = {} end -if not texmf.data then texmf.data = {} end +-- some day i'll use the nested approach, but not yet (actually we even drop +-- engine/progname support since we have only luatex now) +-- +-- first texmfcnf.lua files are located, next the cached texmf.cnf files +-- +-- return { +-- TEXMFBOGUS = 'effe checken of dit werkt', +-- } -function input.aux.load_data(instance,pathname,dataname,filename) - if not filename or (filename == "") then - filename = dataname .. input.lucsuffix - end - local blob = loadfile(file.join(pathname,filename)) - if not blob then - filename = dataname .. input.luasuffix - blob = loadfile(file.join(pathname,filename)) - end +function input.aux.load_texmfcnf(instance,dataname,pathname) + local filename = file.join(pathname,input.luaname) + local blob = loadfile(filename) if blob then - blob() - if (texmf.data.type == dataname) and (texmf.data.version == input.cacheversion) and texmf.data.content then - input.report("loading",dataname,"for",pathname,"from",filename) - instance[dataname][pathname] = texmf.data.content + local data = blob() + if data then + input.report("loading","configuration file",filename) + if true then + -- flatten to variable.progname + local t = { } + for k, v in pairs(data) do -- v = progname + if type(v) == "string" then + t[k] = v + else + for kk, vv in pairs(v) do -- vv = variable + if type(vv) == "string" then + t[vv.."."..v] = kk + end + end + end + end + instance[dataname][pathname] = t + else + instance[dataname][pathname] = data + end else - input.report("skipping",dataname,"for",pathname,"from",filename) + input.report("skipping","configuration file",filename) instance[dataname][pathname] = { } instance.loaderror = true end + else + input.report("skipping","configuration file",filename) + end +end + +function input.aux.load_configuration(instance,dname,lname) + input.aux.load_data(instance,dname,'configuration',lname and file.basename(lname)) +end +function input.aux.load_files(instance,tag) + input.aux.load_data(instance,tag,'files') +end + +function input.resetconfig(instance) + instance.configuration, instance.setup, instance.order, instance.loaderror = { }, { }, { }, false +end + +function input.loadnewconfig(instance) + for _, cnf in ipairs(instance.luafiles) do + local dname = file.dirname(cnf) + input.aux.load_texmfcnf(instance,'setup',dname) + instance.order[#instance.order+1] = instance.setup[dname] + if instance.loaderror then break end + end +end + +function input.loadoldconfig(instance) + if not instance.renewcache then + for _, cnf in ipairs(instance.cnffiles) do + local dname = file.dirname(cnf) + input.aux.load_configuration(instance,dname) + instance.order[#instance.order+1] = instance.configuration[dname] + if instance.loaderror then break end + end end - texmf.data.content = { } + input.joinconfig(instance) end function input.expand_variables(instance) instance.expansions = { } - if instance.engine ~= "" then instance.environment['engine'] = instance.engine end - if instance.progname ~= "" then instance.environment['progname'] = instance.engine end +--~ instance.environment['SELFAUTOPARENT'] = instance.environment['SELFAUTOPARENT'] or instance.rootpath + if instance.engine ~= "" then instance.environment['engine'] = instance.engine end + if instance.progname ~= "" then instance.environment['progname'] = instance.progname end for k,v in pairs(instance.environment) do local a, b = k:match("^(%a+)%_(.*)%s*$") if a and b then @@ -5673,53 +5874,6 @@ function input.is_expansion(instance,name) return input.aux.is_entry(instance,instance.expansions,name) end -function input.aux.list(instance,list) - local pat = string.upper(instance.pattern or "","") - for _,key in pairs(table.sortedkeys(list)) do - if (instance.pattern=="") or string.find(key:upper(),pat) then - if instance.kpseonly then - if instance.kpsevars[key] then - print(key .. "=" .. input.aux.tabstr(list[key])) - end - elseif instance.kpsevars[key] then - print('K ' .. key .. "=" .. input.aux.tabstr(list[key])) - else - print('E ' .. key .. "=" .. input.aux.tabstr(list[key])) - end - end - end -end - -function input.list_variables(instance) - input.aux.list(instance,instance.variables) -end -function input.list_expansions(instance) - input.aux.list(instance,instance.expansions) -end - -function input.list_configurations(instance) - for _,key in pairs(table.sortedkeys(instance.kpsevars)) do - if not instance.pattern or (instance.pattern=="") or key:find(instance.pattern) then - print(key.."\n") - for i,c in ipairs(instance.order) do - local str = c[key] - if str then - print("\t" .. i .. "\t\t" .. input.aux.tabstr(str)) - end - end - print() - end - end -end - -function input.aux.tabstr(str) - if type(str) == 'table' then - return table.concat(str," | ") - else - return str - end -end - function input.simplified_list(str) if type(str) == 'table' then return str -- troubles ; ipv , in texmf @@ -5743,23 +5897,6 @@ function input.unexpanded_path(instance,str) return file.join_path(input.unexpanded_path_list(instance,str)) end ---~ function input.expanded_path_list(instance,str) ---~ if not str then ---~ return { } ---~ elseif instance.savelists then ---~ -- engine+progname hash ---~ str = str:gsub("%$","") ---~ if not instance.lists[str] then -- cached ---~ local lst = input.split_path(input.expansion(instance,str)) ---~ instance.lists[str] = input.aux.expanded_path(instance,lst) ---~ end ---~ return instance.lists[str] ---~ else ---~ local lst = input.split_path(input.expansion(instance,str)) ---~ return input.aux.expanded_path(instance,lst) ---~ end ---~ end - do local done = { } @@ -6172,6 +6309,8 @@ do return original end + input.normalize_name = file.collapse_path + end function input.aux.register_in_trees(instance,name) @@ -6537,7 +6676,10 @@ end function input.load(instance) input.starttiming(instance) + input.resetconfig(instance) input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) input.load_cnf(instance) input.expand_variables(instance) input.load_hash(instance) @@ -6924,28 +7066,45 @@ caches.more = caches.more or "context" caches.direct = false -- true is faster but may need huge amounts of memory caches.trace = false caches.tree = false -caches.temp = caches.temp or os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("VARTEXMF") or os.getenv("TEXMFVAR") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil -caches.paths = caches.paths or { caches.temp } +caches.paths = caches.paths or nil caches.force = false input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true -if caches.temp and caches.temp ~= "" and lfs.attributes(caches.temp,"mode") ~= "directory" then - if caches.force or io.ask(string.format("Should I create the cache path %s?",caches.temp), "no", { "yes", "no" }) == "yes" then - dir.mkdirs(caches.temp) +function caches.temp(instance) + local function checkpath(cachepath) + if not cachepath or cachepath == "" then + return nil + elseif lfs.attributes(cachepath,"mode") == "directory" then -- lfs.isdir(cachepath) then + return cachepath + elseif caches.force or io.ask(string.format("Should I create the cache path %s?",cachepath), "no", { "yes", "no" }) == "yes" then + dir.mkdirs(cachepath) + return (lfs.attributes(cachepath,"mode") == "directory") and cachepath + else + return nil + end end -end -if not caches.temp or caches.temp == "" then - print("\nfatal error: there is no valid cache path defined\n") - os.exit() -elseif lfs.attributes(caches.temp,"mode") ~= "directory" then - print(string.format("\nfatal error: cache path %s is not a directory\n",caches.temp)) - os.exit() + local cachepath = input.expanded_path_list(instance,"TEXMFCACHE") + cachepath = cachepath and #cachepath > 0 and checkpath(cachepath[1]) + if not cachepath then + cachepath = os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil + cachepath = checkpath(cachepath) + end + if not cachepath then + print("\nfatal error: there is no valid cache path defined\n") + os.exit() + elseif lfs.attributes(cachepath,"mode") ~= "directory" then + print(string.format("\nfatal error: cache path %s is not a directory\n",cachepath)) + os.exit() + end + function caches.temp(instance) + return cachepath + end + return cachepath end function caches.configpath(instance) return table.concat(instance.cnffiles,";") ---~ return input.expand_var(instance,"TEXMFCNF") end function caches.hashed(tree) @@ -6963,19 +7122,8 @@ end function caches.setpath(instance,...) if not caches.path then - if lfs and instance then - for _,v in pairs(caches.paths) do - for _,vv in pairs(input.expanded_path_list(instance,v)) do - if lfs.isdir(vv) then - caches.path = vv - break - end - end - if caches.path then break end - end - end if not caches.path then - caches.path = caches.temp + caches.path = caches.temp(instance) end caches.path = input.clean_path(caches.path) -- to be sure if lfs then @@ -6999,6 +7147,12 @@ function caches.setpath(instance,...) return caches.path end +function caches.definepath(instance,category,subcategory) + return function() + return caches.setpath(instance,category,subcategory) + end +end + function caches.setluanames(path,name) return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc" end @@ -7064,19 +7218,35 @@ do -- local report end end + local allocated = { } + + -- tracing + function containers.define(category, subcategory, version, enabled) - if category and subcategory then - return { - category = category, - subcategory = subcategory, - storage = { }, - enabled = enabled, - version = version or 1.000, - trace = false, - path = caches.setpath(texmf.instance,category,subcategory), - } - else - return nil + return function() + if category and subcategory then + local c = allocated[category] + if not c then + c = { } + allocated[category] = c + end + local s = c[subcategory] + if not s then + s = { + category = category, + subcategory = subcategory, + storage = { }, + enabled = enabled, + version = version or 1.000, + trace = false, + path = caches.setpath(texmf.instance,category,subcategory), + } + c[subcategory] = s + end + return s + else + return nil + end end end @@ -7133,129 +7303,35 @@ end -- since we want to use the cache instead of the tree, we will now -- reimplement the saver. +local save_data = input.aux.save_data + +input.cachepath = nil + function input.aux.save_data(instance, dataname, check) - for cachename, files in pairs(instance[dataname]) do - local name + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + save_data(instance, dataname, check, function(cachename,dataname) if input.usecache then - name = file.join(caches.setpath(instance,"trees"),caches.hashed(cachename)) - else - name = file.join(cachename,dataname) - end - local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix - input.report("preparing " .. dataname .. " in", luaname) - for k, v in pairs(files) do - if not check or check(v,k) then -- path, name - if type(v) == "table" and #v == 1 then - files[k] = v[1] - end - else - files[k] = nil -- false - end - end - local data = { - type = dataname, - root = cachename, - version = input.cacheversion, - date = os.date("%Y-%m-%d"), - time = os.date("%H:%M:%S"), - content = files, - } - local f = io.open(luaname,'w') - if f then - input.report("saving " .. dataname .. " in", luaname) - -- f:write(table.serialize(data,'return')) - f:write(input.serialize(data)) - f:close() - input.report("compiling " .. dataname .. " to", lucname) - if not utils.lua.compile(luaname,lucname) then - input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname) - os.remove(lucname) - end + return file.join(input.cachepath(),caches.hashed(cachename)) else - input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix) + return file.join(cachename,dataname) end - end + end) end -function input.serialize(files) - -- This version is somewhat optimized for the kind of - -- tables that we deal with, so it's much faster than - -- the generic serializer. This makes sense because - -- luatools and mtxtools are called frequently. Okay, - -- we pay a small price for properly tabbed tables. - local t = { } - local concat = table.concat - local sorted = table.sortedkeys - local function dump(k,v,m) - if type(v) == 'string' then - return m .. "['" .. k .. "']='" .. v .. "'," - elseif #v == 1 then - return m .. "['" .. k .. "']='" .. v[1] .. "'," - else - return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'}," - end - end - t[#t+1] = "return {" - if instance.sortdata then - for _, k in pairs(sorted(files)) do - local fk = files[k] - if type(fk) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for _, kk in pairs(sorted(fk)) do - t[#t+1] = dump(kk,fk[kk],"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,fk,"\t") - end - end - else - for k, v in pairs(files) do - if type(v) == 'table' then - t[#t+1] = "\t['" .. k .. "']={" - for kk,vv in pairs(v) do - t[#t+1] = dump(kk,vv,"\t\t") - end - t[#t+1] = "\t}," - else - t[#t+1] = dump(k,v,"\t") - end - end - end - t[#t+1] = "}" - return concat(t,"\n") -end +local load_data = input.aux.load_data function input.aux.load_data(instance,pathname,dataname,filename) - local luaname, lucname, pname, fname - if input.usecache then - pname, fname = caches.setpath(instance,"trees"), caches.hashed(pathname) - filename = file.join(pname,fname) - else - if not filename or (filename == "") then - filename = dataname - end - pname, fname = pathname, filename - end - luaname = file.join(pname,fname) .. input.luasuffix - lucname = file.join(pname,fname) .. input.lucsuffix - local blob = loadfile(lucname) - if not blob then - blob = loadfile(luaname) - end - if blob then - local data = blob() - if data and data.content and data.type == dataname and data.version == input.cacheversion then - input.report("loading",dataname,"for",pathname,"from",filename) - instance[dataname][pathname] = data.content + input.cachepath = input.cachepath or caches.definepath(instance,"trees") + load_data(instance,pathname,dataname,filename,function(dataname,filename) + if input.usecache then + return file.join(input.cachepath(),caches.hashed(pathname)) else - input.report("skipping",dataname,"for",pathname,"from",filename) - instance[dataname][pathname] = { } - instance.loaderror = true + if not filename or (filename == "") then + filename = dataname + end + return file.join(pathname,filename) end - else - input.report("skipping",dataname,"for",pathname,"from",filename) - end + end) end -- we will make a better format, maybe something xml or just text or lua @@ -8027,7 +8103,10 @@ messages.help = [[ ]] function input.runners.my_prepare_a(instance) + input.resetconfig(instance) input.identify_cnf(instance) + input.load_lua(instance) + input.expand_variables(instance) input.load_cnf(instance) input.expand_variables(instance) end @@ -8035,6 +8114,7 @@ end function input.runners.my_prepare_b(instance) input.runners.my_prepare_a(instance) input.load_hash(instance) + input.automount(instance) end function input.runners.prepare(instance) |