diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/context/lua/luatools.lua | 876 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-check.lua | 138 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-context.lua | 20 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-update.lua | 392 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-watch.lua | 194 | ||||
| -rw-r--r-- | scripts/context/lua/mtxrun.lua | 665 | ||||
| -rw-r--r-- | scripts/context/ruby/ctxtools.rb | 50 |
7 files changed, 1454 insertions, 881 deletions
diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua index b6a158f6e..24e3bda9d 100644 --- a/scripts/context/lua/luatools.lua +++ b/scripts/context/lua/luatools.lua @@ -363,6 +363,31 @@ function string:split_settings() -- no {} handling, see l-aux for lpeg variant end end +local patterns_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["+"] = "%+", + ["*"] = "%*", + ["%"] = "%%", + ["("] = "%)", + [")"] = "%)", + ["["] = "%[", + ["]"] = "%]", +} + + +function string:pattesc() + return (self:gsub(".",patterns_escapes)) +end + +function string:tohash() + local t = { } + for s in self:gmatch("([^, ]+)") do -- lpeg + t[s] = true + end + return t +end + -- filename : l-lpeg.lua -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -406,7 +431,6 @@ end - -- filename : l-table.lua -- comment : split off from luat-lib -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -488,7 +512,8 @@ end --~ return t --~ end -function table.merge(t, ...) +function table.merge(t, ...) -- first one is target + t = t or {} local lst = {...} for i=1,#lst do for k, v in pairs(lst[i]) do @@ -1017,6 +1042,14 @@ function table.tohash(t) return h end +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + function table.contains(t, v) if t then for i=1, #t do @@ -1048,6 +1081,22 @@ end --~ return table.serialize(a) == table.serialize(b) --~ end +function table.clone(t,p) -- t is optional or nil or table + if not p then + t, p = { }, t or { } + elseif not t then + t = { } + end + setmetatable(t, { __index = function(_,key) return p[key] end }) + return t +end + + +function table.hexed(t,seperator) + local tt = { } + for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end + return table.concat(tt,seperator or " ") +end -- filename : l-io.lua @@ -1119,38 +1168,10 @@ function io.noflines(f) return n end ---~ t, f, n = os.clock(), io.open("testbed/sample-utf16-bigendian-big.txt",'rb'), 0 ---~ for a in io.characters(f) do n = n + 1 end ---~ print(string.format("characters: %s, time: %s", n, os.clock()-t)) - do local sb = string.byte ---~ local nextchar = { ---~ [ 4] = function(f) ---~ return f:read(1), f:read(1), f:read(1), f:read(1) ---~ end, ---~ [ 2] = function(f) ---~ return f:read(1), f:read(1) ---~ end, ---~ [ 1] = function(f) ---~ return f:read(1) ---~ end, ---~ [-2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ return b, a ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ return d, c, b, a ---~ end ---~ } - local nextchar = { [ 4] = function(f) return f:read(1,1,1,1) @@ -1457,6 +1478,19 @@ end if not os.exec then os.exec = os.execute end if not os.spawn then os.spawn = os.execute end +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end +end + +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + function os.launch(str) if os.platform == "windows" then os.execute("start " .. str) -- os.spawn ? @@ -1476,19 +1510,15 @@ if not os.times then -- cstime = children system time function os.times() return { - utime = os.clock(), -- user - stime = 0, -- system - cutime = 0, -- children user - cstime = 0, -- children system + utime = os.gettimeofday(), -- user + stime = 0, -- system + cutime = 0, -- children user + cstime = 0, -- children system } end end -if os.gettimeofday then - os.clock = os.gettimeofday -else - os.gettimeofday = os.clock -end +os.gettimeofday = os.gettimeofday or os.clock do local startuptime = os.gettimeofday() @@ -1535,11 +1565,11 @@ if not versions then versions = { } end versions['l-file'] = 1.001 if not file then file = { } end function file.removesuffix(filename) - return filename:gsub("%.%a+$", "") + return filename:gsub("%.[%a%d]+$", "") end function file.addsuffix(filename, suffix) - if not filename:find("%.%a-$") then + if not filename:find("%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1547,7 +1577,11 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - return (filename:gsub("%.%a+$", "." .. suffix)) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return (filename:gsub("%.[%a%d]+$","."..suffix)) + end end function file.dirname(name) @@ -1566,12 +1600,36 @@ function file.extname(name) return name:match("^.+%.([^/\\]-)$") or "" end +function file.stripsuffix(name) + return (name:gsub("%.[%a%d]+$","")) +end + +--~ function file.join(...) +--~ local t = { ... } +--~ for i=1,#t do +--~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ end +--~ return table.concat(t,"/") +--~ end + +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.join(...) - local t = { ... } - for i=1,#t do - t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") + local pth = table.concat({...},"/") + pth = pth:gsub("\\","/") + local a, b = pth:match("^(.*://)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + a, b = pth:match("^(//)(.*)$") + if a and b then + return a .. b:gsub("//+","/") end - return table.concat(t,"/") + return (pth:gsub("//+","/")) end function file.is_writable(name) @@ -1650,6 +1708,111 @@ file.readdata = io.loaddata file.savedata = io.savedata +-- filename : l-url.lua +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +if not versions then versions = { } end versions['l-url'] = 1.001 +if not url then url = { } end + +-- from the spec (on the web): +-- +-- foo://example.com:8042/over/there?name=ferret#nose +-- \_/ \______________/\_________/ \_________/ \__/ +-- | | | | | +-- scheme authority path query fragment +-- | _____________________|__ +-- / \ / \ +-- urn:example:animal:ferret:nose + +do + + local function tochar(s) + return string.char(tonumber(s,16)) + end + + local colon, qmark, hash, slash, percent, endofstring = lpeg.P(":"), lpeg.P("?"), lpeg.P("#"), lpeg.P("/"), lpeg.P("%"), lpeg.P(-1) + + local hexdigit = lpeg.R("09","AF","af") + local escaped = percent * lpeg.C(hexdigit * hexdigit) / tochar + + local scheme = lpeg.Cs((escaped+(1-colon-slash-qmark-hash))^0) * colon + lpeg.Cc("") + local authority = slash * slash * lpeg.Cs((escaped+(1- slash-qmark-hash))^0) + lpeg.Cc("") + local path = slash * lpeg.Cs((escaped+(1- qmark-hash))^0) + lpeg.Cc("") + local query = qmark * lpeg.Cs((escaped+(1- hash))^0) + lpeg.Cc("") + local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0) + lpeg.Cc("") + + local parser = lpeg.Ct(scheme * authority * path * query * fragment) + + function url.split(str) + return (type(str) == "string" and parser:match(str)) or str + end + +end + +function url.hashed(str) + local s = url.split(str) + return { + scheme = (s[1] ~= "" and s[1]) or "file", + authority = s[2], + path = s[3], + query = s[4], + fragment = s[5], + original=str + } +end + +function url.filename(filename) + local t = url.hashed(filename) + return (t.scheme == "file" and t.path:gsub("^/([a-zA-Z])([:|])/)","%1:")) or filename +end + +function url.query(str) + if type(str) == "string" then + local t = { } + for k, v in str:gmatch("([^&=]*)=([^&=]*)") do + t[k] = v + end + return t + else + return str + end +end + +--~ print(url.filename("file:///c:/oeps.txt")) +--~ print(url.filename("c:/oeps.txt")) +--~ print(url.filename("file:///oeps.txt")) +--~ print(url.filename("file:///etc/test.txt")) +--~ print(url.filename("/oeps.txt")) + +-- from the spec on the web (sort of): +--~ +--~ function test(str) +--~ print(table.serialize(url.hashed(str))) +--~ end +---~ +--~ test("%56pass%20words") +--~ test("file:///c:/oeps.txt") +--~ test("file:///c|/oeps.txt") +--~ test("file:///etc/oeps.txt") +--~ test("file://./etc/oeps.txt") +--~ test("file:////etc/oeps.txt") +--~ test("ftp://ftp.is.co.za/rfc/rfc1808.txt") +--~ test("http://www.ietf.org/rfc/rfc2396.txt") +--~ test("ldap://[2001:db8::7]/c=GB?objectClass?one#what") +--~ test("mailto:John.Doe@example.com") +--~ test("news:comp.infosystems.www.servers.unix") +--~ test("tel:+1-816-555-1212") +--~ test("telnet://192.0.2.16:80/") +--~ test("urn:oasis:names:specification:docbook:dtd:xml:4.1.2") +--~ test("/etc/passwords") +--~ test("http://www.pragma-ade.com/spaced%20name") + +--~ test("zip:///oeps/oeps.zip#bla/bla.tex") +--~ test("zip:///oeps/oeps.zip?bla/bla.tex") + + -- filename : l-dir.lua -- comment : split off from luat-lib -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -2112,6 +2275,8 @@ function utils.lua.compile(luafile, lucfile) end end + + -- filename : luat-lib.lua -- comment : companion to luat-lib.tex -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -2135,19 +2300,13 @@ os.setlocale(nil,nil) -- useless feature and even dangerous in luatex if not io.fileseparator then if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", "windows" + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", "unix" + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" end end -if not os.platform then - if io.pathseparator == ";" then - os.platform = "windows" - else - os.platform = "unix" - end -end +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" -- arg normalization -- @@ -2346,17 +2505,22 @@ input.formats ['lua'] = 'LUAINPUTS' -- new input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } -- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata(instance) +function input.checkconfigdata(instance) -- not yet ok, no time for debugging now local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" - if not instance.environment[proname] and not instance.variables[proname] == "" and not instance.environment[varname] and not instance.variables[varname] == "" then - instance.variables[varname] = default + local p = instance.environment[proname] + local v = instance.environment[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? end end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") - fix("FONTFEATURES", ".;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") end -- backward compatible ones @@ -2457,6 +2621,11 @@ function input.reset() end +function input.reset_hashes(instance) + instance.lists = { } + instance.found = { } +end + function input.bare_variable(str) -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) @@ -2522,7 +2691,7 @@ input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRA -- loading the database files. do - local clock = os.clock + local clock = os.gettimeofday or os.clock function input.starttiming(instance) if instance then @@ -2796,6 +2965,7 @@ function input.aux.extend_texmf_var(instance,specification) -- crap instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" end input.expand_variables(instance) + input.reset_hashes(instance) end -- locators @@ -2811,28 +2981,6 @@ function input.locatedatabase(instance,specification) return input.methodhandler('locators', instance, specification) end ---~ poor mans solution, from before we had lfs.isdir ---~ ---~ function input.locators.tex(instance,specification) ---~ if specification and specification ~= '' then ---~ local files = { ---~ file.join(specification,'files'..input.lucsuffix), ---~ file.join(specification,'files'..input.luasuffix), ---~ file.join(specification,input.lsrname) ---~ } ---~ for _, filename in pairs(files) do ---~ local f = io.open(filename) ---~ if f then ---~ input.logger('! tex locator', specification..' found') ---~ input.aux.append_hash(instance,'file',specification,filename) ---~ f:close() ---~ return ---~ end ---~ end ---~ input.logger('? tex locator', specification..' not found') ---~ end ---~ end - function input.locators.tex(instance,specification) if specification and specification ~= '' and lfs.isdir(specification) then input.logger('! tex locator', specification..' found') @@ -3191,8 +3339,6 @@ function input.expand_variables(instance) for k,v in pairs(instance.expansions) do instance.expansions[k] = v:gsub("\\", '/') end - -- ########## - --~ input.splitexpansions(instance) -- better not, fuzzy end function input.aux.expand_vars(instance,lst) -- simple vars @@ -3350,15 +3496,12 @@ do end function input.register_extra_path(instance,paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep if paths and paths ~= "" then - local ep = instance.extra_paths - if not ep then - ep = { } - instance.extra_paths = ep - end - local n = #ep - if subpath and subpaths ~= "" then + if subpaths and subpaths ~= "" then for p in paths:gmatch("[^,]+") do + -- we gmatch each step again, not that fast, but used seldom for s in subpaths:gmatch("[^,]+") do local ps = p .. "/" .. s if not done[ps] then @@ -3375,10 +3518,24 @@ do end end end - if n < #ep then - instance.lists = { } + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end end end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end end end @@ -3633,7 +3790,7 @@ input.is_readable.tex = input.is_readable.file -- name/name function input.aux.collect_files(instance,names) - local filelist = nil + local filelist = { } for _, fname in pairs(names) do if fname then if input.trace > 2 then @@ -3665,15 +3822,20 @@ function input.aux.collect_files(instance,names) if blobfile then if type(blobfile) == 'string' then if not dname or blobfile:find(dname) then - if not filelist then filelist = { } end - -- input.logger('= collected', blobpath.." | "..blobfile.." | "..bname) - filelist[#filelist+1] = file.join(blobpath,blobfile,bname) + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + input.concatinators[hash.type](blobpath,blobfile,bname) -- result + } end else for _, vv in pairs(blobfile) do if not dname or vv:find(dname) then - if not filelist then filelist = { } end - filelist[#filelist+1] = file.join(blobpath,vv,bname) + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + input.concatinators[hash.type](blobpath,vv,bname) -- result + } end end end @@ -3684,7 +3846,11 @@ function input.aux.collect_files(instance,names) end end end - return filelist + if #filelist > 0 then + return filelist + else + return nil + end end function input.suffix_of_format(str) @@ -3703,54 +3869,30 @@ function input.suffixes_of_format(str) end end ---~ function input.aux.qualified_path(filename) -- make platform dependent / not good yet ---~ return ---~ filename:find("^%.+/") or ---~ filename:find("^/") or ---~ filename:find("^%a+%:") or ---~ filename:find("^%a+##") ---~ end - ---~ function input.normalize_name(original) ---~ -- internally we use type##spec##subspec ; this hackery slightly slows down searching ---~ local str = original or "" ---~ str = str:gsub("::", "##") -- :: -> ## ---~ str = str:gsub("^(%a+)://" ,"%1##") -- zip:// -> zip## ---~ str = str:gsub("(.+)##(.+)##/(.+)","%1##%2##%3") -- ##/spec -> ##spec ---~ if (input.trace>1) and (original ~= str) then ---~ input.logger('= normalizer',original.." -> "..str) ---~ end ---~ return str ---~ end +do -do -- called about 700 times for an empty doc (font initializations etc) + -- called about 700 times for an empty doc (font initializations etc) -- i need to weed the font files for redundant calls local letter = lpeg.R("az","AZ") - local separator = lpeg.P("##") + local separator = lpeg.P("://") - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local normalized = lpeg.Cs( - (letter^1*(lpeg.P("://")/"##") * (1-lpeg.P(false))^1) + - (lpeg.P("::")/"##" + (1-separator)^1*separator*(1-separator)^1*separator*(lpeg.P("/")/"") + 1)^0 - ) + local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + local rootbased = lpeg.P("/") + letter*lpeg.P(":") - -- ./name ../name /name c: zip## (todo: use url internally and get rid of ##) + -- ./name ../name /name c: :// function input.aux.qualified_path(filename) return qualified:match(filename) end + function input.aux.rootbased_path(filename) + return rootbased:match(filename) + end - -- zip:// -> zip## ; :: -> ## ; aa##bb##/cc -> aa##bb##cc function input.normalize_name(original) - local str = normalized:match(original or "") - if input.trace > 1 and original ~= str then - input.logger('= normalizer',original.." -> "..str) - end - return str + return original end -end --- split the next one up, better for jit +end function input.aux.register_in_trees(instance,name) if not name:find("^%.") then @@ -3758,11 +3900,13 @@ function input.aux.register_in_trees(instance,name) end end +-- split the next one up, better for jit + function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) local result = { } local stamp = nil - filename = input.normalize_name(filename) - filename = file.collapse_path(filename:gsub("\\","/")) + filename = input.normalize_name(filename) -- elsewhere + filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format @@ -3834,7 +3978,7 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec local typespec = input.variable_of_format(filetype) local pathlist = input.expanded_path_list(instance,typespec) if not pathlist or #pathlist == 0 then - -- no pathlist, access check only + -- no pathlist, access check only / todo == wildcard if input.trace > 2 then input.logger('? filename',filename) input.logger('? filetype',filetype or '?') @@ -3849,8 +3993,9 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end -- this is actually 'other text files' or 'any' or 'whatever' local filelist = input.aux.collect_files(instance,wantedfiles) - filename = filelist and filelist[1] - if filename then + local lf = filelist and filelist[1] + if fl then + filename = fl[3] result[#result+1] = filename done = true end @@ -3860,8 +4005,8 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec local doscan, recurse if input.trace > 2 then input.logger('? filename',filename) - if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - if filelist then input.logger('? file list',table.concat(filelist," | ")) end + -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end + -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end end -- a bit messy ... esp the doscan setting here for _, path in pairs(pathlist) do @@ -3874,16 +4019,18 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec -- compare list entries with permitted pattern pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') + pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless local expr = "^" .. pathname -- input.debug('?',expr) - for _, f in pairs(filelist) do + for _, fl in ipairs(filelist) do + local f = fl[2] if f:find(expr) then -- input.debug('T',' '..f) if input.trace > 2 then input.logger('= found in hash',f) end - result[#result+1] = f + --- todo, test for readable + result[#result+1] = fl[3] input.aux.register_in_trees(instance,f) -- for tracing used files done = true if not instance.allresults then break end @@ -3893,7 +4040,7 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end end if not done and doscan then - -- check if on disk / unchecked / does not work at all + -- check if on disk / unchecked / does not work at all / also zips if input.method_is_file(pathname) then -- ? local pname = pathname:gsub("%.%*$",'') if not pname:find("%*") then @@ -3970,10 +4117,7 @@ end if not input.concatinators then input.concatinators = { } end -function input.concatinators.tex(tag,path,name) - return tag .. '/' .. path .. '/' .. name -end - +input.concatinators.tex = file.join input.concatinators.file = input.concatinators.tex function input.find_files(instance,filename,filetype,mustexist) @@ -4175,15 +4319,6 @@ function input.aux.register_file(files, name, path) end end --- zip:: zip## zip:// --- zip::pathtozipfile::pathinzipfile (also: pathtozipfile/pathinzipfile) --- file::name --- tex::name --- kpse::name --- kpse::format::name --- parent::n::name --- parent::name (default 2) - if not input.finders then input.finders = { } end if not input.openers then input.openers = { } end if not input.loaders then input.loaders = { } end @@ -4193,30 +4328,37 @@ input.openers.notfound = { nil } input.loaders.notfound = { false, nil, 0 } function input.splitmethod(filename) - local method, specification = filename:match("^(.-)##(.+)$") - if method and specification then - return method, specification + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not filename:find("://") then + return { scheme="file", path = filename, original=filename } -- quick hack else - return 'tex', filename + return url.hashed(filename) end end function input.method_is_file(filename) - local method, specification = input.splitmethod(filename) - return method == 'tex' or method == 'file' + return input.splitmethod(filename).scheme == 'file' +end + +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do + s[#s+1] = k .. "=" .. v + end + return table.concat(s, sep or " | ") end function input.methodhandler(what, instance, filename, filetype) -- ... - local method, specification = input.splitmethod(filename) - if method and specification then -- redundant - if input[what][method] then - input.logger('= handler',filename.." -> "..what.." | "..method.." | "..specification) - return input[what][method](instance,specification,filetype) - else - return nil - end + local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if input[what][scheme] then + input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) + return input[what][scheme](instance,filename,filetype) -- todo: specification else - return input[what].tex(instance,filename,filetype) + return input[what].tex(instance,filename,filetype) -- todo: specification end end @@ -4250,6 +4392,8 @@ function input.texdatablob(instance, filename, filetype) return data or "" end +input.loadtexfile = input.texdatablob + function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! local fullname = input.findtexfile(texmf.instance, filename) if fullname and (fullname ~= "") then @@ -4828,7 +4972,7 @@ function input.aux.load_data(instance,pathname,dataname,filename) end end --- we will make a better format, maybe something xml or just text +-- we will make a better format, maybe something xml or just text or lua input.automounted = input.automounted or { } @@ -4957,80 +5101,117 @@ if not zip.supported then function zip.openarchive (...) return nil end -- needed ? function zip.closenarchive (...) end -- needed ? - function input.registerzipfile (...) end -- needed ? function input.usezipfile (...) end -- needed ? else - function input.locators.zip(instance,specification) - local name, spec = specification:match("^(.-)##(.-)$") - local f = io.open(name or specification) - if f then -- todo: reuse code - input.logger('! zip locator', specification..' found') - if name and spec then - input.aux.append_hash(instance,'zip',"zip##"..specification,name) - input.aux.extend_texmf_var(instance, "zip##"..specification) + -- zip:///oeps.zip?name=bla/bla.tex + -- zip:///oeps.zip?tree=tex/texmf-local + + local function validzip(str) + if not str:find("^zip://") then + return "zip:///" .. str + else + return str + end + end + + zip.archives = { } + zip.registeredfiles = { } + + function zip.openarchive(instance,name) + if not name or name == "" then + return nil + else + local arch = zip.archives[name] + if arch then + return arch else - input.aux.append_hash(instance,'zip',"zip##"..specification.."##",specification) - input.aux.extend_texmf_var(instance, "zip##"..specification.."##") + local full = input.find_file(instance,name) or "" + local arch = (full ~= "" and zip.open(full)) or false + zip.archives[name] = arch + return arch end - f:close() + end + end + + function zip.closearchive(instance,name) + if not name or name == "" and zip.archives[name] then + zip.close(zip.archives[name]) + zip.archives[name] = nil + end + end + + -- zip:///texmf.zip?tree=/tex/texmf + -- zip:///texmf.zip?tree=/tex/texmf-local + -- zip:///texmf-mine.zip?tree=/tex/texmf-projects + + function input.locators.zip(instance,specification) -- where is this used? startup zips (untested) + specification = input.splitmethod(specification) + local zipfile = specification.path + local zfile = zip.openarchive(instance,name) -- tricky, could be in to be initialized tree + if zfile then + input.logger('! zip locator', specification.original ..' found') else - input.logger('? zip locator', specification..' not found') + input.logger('? zip locator', specification.original ..' not found') end end function input.hashers.zip(instance,tag,name) input.report("loading zip file",name,"as",tag) - input.registerzipfile(instance,name,tag) + input.usezipfile(instance,tag .."?tree=" .. name) end function input.concatinators.zip(tag,path,name) - return tag .. path .. '/' .. name + if not path or path == "" then + return tag .. '?name=' .. name + else + return tag .. '?name=' .. path .. "/" .. name + end end function input.is_readable.zip(name) return true end - function input.finders.zip(instance,filename,filetype) - local archive, dataname = filename:match("^(.+)##/*(.+)$") - if archive and dataname then - local zfile = zip.openarchive(archive) - if not zfile then - archive = input.find_file(instance,archive,filetype) - zfile = zip.openarchive(archive) - end - if zfile then - input.logger('! zip finder',archive) - local dfile = zfile:open(dataname) - if dfile then - dfile = zfile:close() - input.logger('+ zip finder',filename) - return 'zip##' .. filename + function input.finders.zip(instance,specification,filetype) + specification = input.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(instance,specification.path) + if zfile then + input.logger('! zip finder',specification.path) + local dfile = zfile:open(q.name) + if dfile then + dfile = zfile:close() + input.logger('+ zip finder',q.name) + return specification.original + end + else + input.logger('? zip finder',specification.path) end - else - input.logger('? zip finder',archive) end end input.logger('- zip finder',filename) return unpack(input.finders.notfound) end - function input.openers.zip(instance,filename) - if filename and filename ~= "" then - local archive, dataname = filename:match("^(.-)##/*(.+)$") - if archive and dataname then - local zfile= zip.openarchive(archive) + function input.openers.zip(instance,specification) + local zipspecification = input.splitmethod(specification) + if zipspecification.path then + local q = url.query(zipspecification.query) + if q.name then + local zfile = zip.openarchive(instance,zipspecification.path) if zfile then - input.logger('+ zip starter',archive) - local dfile = zfile:open(dataname) + input.logger('+ zip starter',zipspecification.path) + local dfile = zfile:open(q.name) if dfile then - input.show_open(filename) - return input.openers.text_opener(filename,dfile,'zip') + input.show_open(specification) + return input.openers.text_opener(specification,dfile,'zip') end else - input.logger('- zip starter',archive) + input.logger('- zip starter',zipspecification.path) end end end @@ -5038,15 +5219,15 @@ else return unpack(input.openers.notfound) end - function input.loaders.zip(instance, filename) -- we could use input.openers.zip - if filename and filename ~= "" then - input.logger('= zip loader',filename) - local archive, dataname = filename:match("^(.+)##/*(.+)$") - if archive and dataname then - local zfile = zip.openarchive(archive) + function input.loaders.zip(instance,specification) + specification = input.splitmethod(specification) + if specification.path then + local q = url.query(specification.query) + if q.name then + local zfile = zip.openarchive(instance,specification.path) if zfile then - input.logger('= zip starter',archive) - local dfile = zfile:open(dataname) + input.logger('+ zip starter',specification.path) + local dfile = zfile:open(q.name) if dfile then input.show_load(filename) input.logger('+ zip loader',filename) @@ -5055,105 +5236,67 @@ else return true, s, #s end else - input.logger('- zip starter',archive) + input.logger('- zip starter',specification.path) end end end input.logger('- zip loader',filename) - return unpack(input.loaders.notfound) - end - - zip.archives = { } - zip.registeredfiles = { } - - function zip.openarchive(name) - if name and name ~= "" and not zip.archives[name] then - zip.archives[name] = zip.open(name) - end - return zip.archives[name] - end - - function zip.closearchive(name) - if zip.archives[name] then - zip.close(archives[name]) - zip.archives[name] = nil - end + return unpack(input.openers.notfound) end - -- aparte register maken voor user (register tex / zip), runtime tree register - -- todo: alleen url syntax toestaan - -- user function: also handle zip::name::path + -- zip:///somefile.zip + -- zip:///somefile.zip?tree=texmf-local -> mount - function input.usezipfile(instance,zipname) -- todo zip:// - zipname = input.normalize_name(zipname) - if not zipname:find("^zip##") then - zipname = "zip##"..zipname - end - input.logger('! zip user','file '..zipname) - if not zipname:find("^zip##(.+)##(.-)$") then - zipname = zipname .. "##" -- dummy spec - end - local tag = zipname - local name = zipname:match("zip##(.+)##.-") - input.aux.prepend_hash(instance,'zip',tag,name) - input.aux.extend_texmf_var(instance, tag) - input.registerzipfile(instance,name,tag) - end - - function input.registerzipfile(instance,zipname,tag) - if not zip.registeredfiles[zipname] then - input.starttiming(instance) - local z = zip.open(zipname) - if not z then - zipname = input.find_file(instance,zipname) - z = zip.open(zipname) - end + function input.usezipfile(instance,zipname) + zipname = validzip(zipname) + input.logger('! zip use','file '..zipname) + local specification = input.splitmethod(zipname) + local zipfile = specification.path + if zipfile and not zip.registeredfiles[zipname] then + local tree = url.query(specification.query).tree or "" + input.logger('! zip register','file '..zipname) + local z = zip.openarchive(instance,zipfile) if z then input.logger("= zipfile","registering "..zipname) + input.starttiming(instance) + input.aux.prepend_hash(instance,'zip',zipname,zipfile) + input.aux.extend_texmf_var(instance,zipname) -- resets hashes too zip.registeredfiles[zipname] = z - input.aux.register_zip_file(instance,zipname,tag) + instance.files[zipname] = input.aux.register_zip_file(z,tree or "") + input.stoptiming(instance) else input.logger("? zipfile","unknown "..zipname) end - input.stoptiming(instance) + else + input.logger('! zip register','no file '..zipname) end end - function input.aux.register_zip_file(instance,zipname,tagname) - if zip.registeredfiles[zipname] then - if not tagname:find("^zip##") then - tagname = "zip##" .. tagname - end - local path, name, n = nil, nil, 0 - if not instance.files[tagname] then - instance.files[tagname] = { } - end - local files, filter = instance.files[tagname], "" - local subtree = tagname:match("^zip##.+##(.+)$") - if subtree then - filter = "^"..subtree.."/(.+)/(.-)$" - else - filter = "^(.+)/(.-)$" - end - input.logger('= zip filter',filter) - -- we can consider putting a files.luc in the file - local register = input.aux.register_file - for i, _ in zip.registeredfiles[zipname]:files() do - path, name = i.filename:match(filter) - if path then - if name and name ~= '' then - register(files, name, path) - n = n + 1 - else - -- directory - end - else - register(files, i.filename, '') + function input.aux.register_zip_file(z,tree) + local files, filter = { }, "" + if tree == "" then + filter = "^(.+)/(.-)$" + else + filter = "^"..tree.."/(.+)/(.-)$" + end + input.logger('= zip filter',filter) + local register, n = input.aux.register_file, 0 + for i in z:files() do + local path, name = i.filename:match(filter) + if path then + if name and name ~= '' then + register(files, name, path) n = n + 1 + else + -- directory end + else + register(files, i.filename, '') + n = n + 1 end - input.report(n, 'entries in', zipname) end + input.report('= zip entries',n) + return files end end @@ -5348,6 +5491,8 @@ if texconfig and not texlua then do texio.write_nl(s .. b .. "\n") end + -- this will become: ctx.install_statistics(fnc() return ..,.. end) etc + function ctx.show_statistics() local function ws(...) ctx.writestatus("mkiv lua stats",string.format(...)) @@ -5383,6 +5528,12 @@ if texconfig and not texlua then do if languages then ws("language load time - %s seconds (n=%s)", input.loadtime(languages), languages.hyphenation.n()) end + if figures then + ws("graphics processing time - %s seconds (n=%s) (including tex)", input.loadtime(figures), figures.n or "?") + end + if metapost then + ws("metapost processing time - %s seconds (+ loading: %s seconds)", input.loadtime(metapost), input.loadtime(mplib)) + end if status.luastate_bytes then ws("current memory usage - %s bytes", status.luastate_bytes) end @@ -5583,7 +5734,7 @@ if texconfig and not texlua then 'hash_extra', 'max_strings', 'pool_free', 'pool_size', 'string_vacancies', 'obj_tab_size', 'pdf_mem_size', 'dest_names_size', 'nest_size', 'param_size', 'save_size', 'stack_size', - 'trie_size', 'hyph_size', + 'trie_size', 'hyph_size', 'max_in_open', 'ocp_stack_size', 'ocp_list_size', 'ocp_buf_size' } @@ -5610,6 +5761,7 @@ if texconfig and not texlua then end texconfig.max_print_line = 100000 + texconfig.max_in_open = 127 end @@ -5637,114 +5789,6 @@ function cs.testcase(b) end end --- This is not the most ideal place, but it will do. Maybe we need to move --- attributes to node-att.lua. - -if node then - - nodes = nodes or { } - - do - - -- just for testing - - local reserved = { } - - function nodes.register(n) - reserved[#reserved+1] = n - end - - function nodes.cleanup_reserved(nofboxes) -- todo - local nr, free = #reserved, node.free - for i=1,nr do - free(reserved[i]) - end - local nl, tb, flush = 0, tex.box, node.flush_list - if nofboxes then - for i=1,nofboxes do - local l = tb[i] - if l then - -- flush(l) - tb[i] = nil - nl = nl + 1 - end - end - end - reserved = { } - return nr, nl, nofboxes - end - - end - - do - - local pdfliteral = node.new("whatsit",8) pdfliteral.next, pdfliteral.prev = nil, nil pdfliteral.mode = 1 - local disc = node.new("disc") disc.next, disc.prev = nil, nil - local kern = node.new("kern",1) kern.next, kern.prev = nil, nil - local penalty = node.new("penalty") penalty.next, penalty.prev = nil, nil - local glue = node.new("glue") glue.next, glue.prev = nil, nil - local glue_spec = node.new("glue_spec") glue_spec.next, glue_spec.prev = nil, nil - - nodes.register(pdfliteral) - nodes.register(disc) - nodes.register(kern) - nodes.register(penalty) - nodes.register(glue) - nodes.register(glue_spec) - - local copy = node.copy - - function nodes.penalty(p) - local n = copy(penalty) - n.penalty = p - return n - end - function nodes.kern(k) - local n = copy(kern) - n.kern = k - return n - end - function nodes.glue(width,stretch,shrink) - local n = copy(glue) - local s = copy(glue_spec) - s.width, s.stretch, s.shrink = width, stretch, shrink - n.spec = s - return n - end - function nodes.glue_spec(width,stretch,shrink) - local s = copy(glue_spec) - s.width, s.stretch, s.shrink = width, stretch, shrink - return s - end - - function nodes.disc() - return copy(disc) - end - - function nodes.pdfliteral(str) - local t = copy(pdfliteral) - t.data = str - return t - end - - end - -end - -if tex then - - function tex.node_mem_status() - -- todo: lpeg - local s = status.node_mem_usage - local t = { } - for n, tag in s:gmatch("(%d+) ([a-z_]+)") do - t[tag] = n - end - return t - end - -end - if not modules then modules = { } end modules ['luat-kps'] = { version = 1.001, @@ -5869,6 +5913,7 @@ own.libs = { -- todo: check which ones are really needed 'l-os.lua', 'l-md5.lua', 'l-file.lua', + 'l-url.lua', 'l-dir.lua', 'l-boolean.lua', 'l-unicode.lua', @@ -5935,7 +5980,8 @@ utils.report = input.report input.defaultlibs = { -- not all are needed 'l-string.lua', 'l-lpeg.lua', 'l-table.lua', 'l-boolean.lua', 'l-number.lua', 'l-set.lua', 'l-unicode.lua', - 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-dir.lua', 'l-utils.lua', 'l-tex.lua', + 'l-md5.lua', 'l-os.lua', 'l-io.lua', 'l-file.lua', 'l-url.lua', 'l-dir.lua', 'l-utils.lua', 'l-tex.lua', +'luat-env.lua', 'luat-lib.lua', 'luat-inp.lua', 'luat-tmp.lua', 'luat-zip.lua', 'luat-tex.lua' } diff --git a/scripts/context/lua/mtx-check.lua b/scripts/context/lua/mtx-check.lua new file mode 100644 index 000000000..dd8a71264 --- /dev/null +++ b/scripts/context/lua/mtx-check.lua @@ -0,0 +1,138 @@ +if not modules then modules = { } end modules ['mtx-check'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.checker = scripts.checker or { } + +local validator = { } + +do + + validator.n = 1 + validator.errors = { } + validator.trace = false + validator.direct = false + + validator.printer = print + validator.tracer = print + + local message = function(position, kind) + local ve = validator.errors + ve[#ve+1] = { kind, position, validator.n } + if validator.direct then + validator.printer(string.format("%s error at position %s (line %s)", kind, position, validator.n)) + end + end + local progress = function(position, data, kind) + if validator.trace then + validator.tracer(string.format("%s at position %s: %s", kind, position, data or "")) + end + end + + local P, S, V, C, CP, CC = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cc + + local i_m, d_m = P("$"), P("$$") + local l_s, r_s = P("["), P("]") + local l_g, r_g = P("{"), P("}") + + local esc = P("\\") + local cr = P("\r") + local lf = P("\n") + local crlf = P("\r\n") + local space = S(" \t\f\v") + local newline = crlf + cr + lf + + local line = newline / function() validator.n = validator.n + 1 end + + local grammar = P { "tokens", + ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), + ["grouped"] = CP() * C(l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g) * CC("group") / progress, + ["setup"] = CP() * C(l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s) * CC("setup") / progress, + ["display"] = CP() * C(d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m) * CC("display") / progress, + ["inline"] = CP() * C(i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m) * CC("inline") / progress, + ["errors"] = (V("gerror") + V("serror") + V("derror") + V("ierror")) * true, + ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, + ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, + ["derror"] = CP() * d_m * CC("display math error") / message, + ["ierror"] = CP() * i_m * CC("inline math error") / message, + } + + local grammar = P { "tokens", + ["tokens"] = (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + V("errors") + 1)^0, + ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), + ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, + ["setup"] = l_s * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, + ["display"] = d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m, + ["inline"] = i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m, + ["errors"] = (V("gerror")+ V("serror") + V("derror") + V("ierror")), + ["gerror"] = CP() * (l_g + r_g) * CC("grouping") / message, + ["serror"] = CP() * (l_s + r_g) * CC("setup error") / message, + ["derror"] = CP() * d_m * CC("display math error") / message, + ["ierror"] = CP() * i_m * CC("inline math error") / message, + } + + function validator.check(str) + validator.n = 1 + validator.errors = { } + grammar:match(str) + end + +end + +--~ str = [[ +--~ a{oeps {oe\{\}ps} } +--~ test { oeps \} \[\] oeps \setupxxx[oeps=bla]} +--~ test $$ \hbox{$ oeps \} \[\] oeps $} $$ +--~ {$x\$xx$ $ +--~ ]] +--~ str = string.rep(str,10) + +function scripts.checker.check(filename) + local str = io.loaddata(filename) + if str then + validator.check(str) + if #validator.errors > 0 then + for k, v in ipairs(validator.errors) do + local kind, position, line = v[1], v[2], v[3] + local data = str:sub(position-30,position+30) + data = data:gsub("(.)", { + ["\n"] = " <lf> ", + ["\r"] = " <cr> ", + ["\t"] = " <tab> ", + }) + data = data:gsub("^ *","") + print(string.format("% 5i %s %s", line,string.rpadd(kind,10," "),data)) + end + else + print("no error") + end + else + print("no file") + end +end + + +banner = banner .. " | tex check tools " + +messages.help = [[ +--convert check tex file for errors +]] + +input.verbose = true + +if environment.argument("check") then + scripts.checker.check(environment.files[1]) +elseif environment.argument("help") then + input.help(banner,messages.help) +elseif environment.files[1] then + scripts.checker.check(environment.files[1]) +end + diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 3abd270aa..e0aa7d086 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -549,6 +549,10 @@ function scripts.context.run(ctxdata) filename = makestub("\\xmlprocess{%s}",filename) end -- + if environment.argument("autopdf") then + os.spawn(string.format('pdfclose --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + end + -- local command = "luatex --fmt=" .. string.quote(formatfile) .. " --lua=" .. string.quote(scriptfile) .. " " .. string.quote(filename) local oldhash, newhash = scripts.context.multipass.hashfiles(jobname), { } scripts.context.multipass.makeoptionfile(jobname,ctxdata) @@ -570,6 +574,13 @@ function scripts.context.run(ctxdata) end end end + -- + -- todo: result + -- + if environment.argument("autopdf") then + os.spawn(string.format('pdfopen --file "%s" 2>&1', file.replacesuffix(filename,"pdf"))) + end + -- end else input.error("no format found with name " .. formatname) @@ -578,8 +589,8 @@ function scripts.context.run(ctxdata) end function scripts.context.make() - -- hack, should also be a shared function - for _, name in ipairs( { "cont-en", "cont-nl", "mptopdf" } ) do + local list = (environment.files[1] and environment.files) or { "cont-en", "cont-nl", "mptopdf" } + for _, name in ipairs(list) do local command = "luatools --make --compile " .. name input.report("running command: " .. command) os.spawn(command) @@ -588,7 +599,7 @@ end function scripts.context.generate() -- hack, should also be a shared function - local command = "luatools --generate " .. name + local command = "luatools --generate " input.report("running command: " .. command) os.spawn(command) end @@ -607,8 +618,11 @@ messages.help = [[ --make create context formats formats --generate generate file database etc. --ctx=name use ctx file +--autopdf open pdf file afterwards ]] +input.verbose = true + input.starttiming(scripts.context) if environment.argument("run") then diff --git a/scripts/context/lua/mtx-update.lua b/scripts/context/lua/mtx-update.lua index cfe553731..008ad68e3 100644 --- a/scripts/context/lua/mtx-update.lua +++ b/scripts/context/lua/mtx-update.lua @@ -19,75 +19,105 @@ scripts.update = scripts.update or { } minimals = minimals or { } minimals.config = minimals.config or { } -scripts.update.collections = { +os.setenv("CYGWIN","nontsec") + +scripts.update.formats = { + "cont-en", + "cont-nl", + "cont-cz", + "cont-de", + "cont-fa", + "cont-it", + "cont-ro", + "cont-uk", + "metafun", + "mptopdf", + "plain" +} + +scripts.update.repositories = { + "current", + "experimental" +} + +scripts.update.versions = { + "current", + "latest" +} + +scripts.update.engines = { ["luatex"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, - { "fonts/new/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/current/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/%s/", "texmf-%s" }, - { "bin/context/%s/", "texmf-%s" }, - { "bin/metapost/%s/", "texmf-%s" }, - { "bin/luatex/%s/", "texmf-%s" }, - { "bin/man/", "texmf-%s" } + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "fonts/new/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } }, ["xetex"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, - { "base/xetex/", "texmf" }, - { "fonts/new/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/current/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/%s/", "texmf-%s" }, - { "bin/context/%s/", "texmf-%s" }, - { "bin/metapost/%s/", "texmf-%s" }, - { "bin/xetex/%s/", "texmf-%s" }, - { "bin/man/", "texmf-%s" } + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "base/xetex/", "texmf" }, + { "fonts/new/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/xetex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } }, ["pdftex"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, - { "fonts/old/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/current/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/%s/", "texmf-%s" }, - { "bin/context/%s/", "texmf-%s" }, - { "bin/metapost/%s/", "texmf-%s" }, - { "bin/pdftex/%s/", "texmf-%s" }, - { "bin/man/", "texmf-%s" } + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "fonts/old/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/pdftex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } }, ["all"] = { - { "base/tex/", "texmf" }, - { "base/metapost/", "texmf" }, - { "base/xetex/", "texmf" }, - { "fonts/old/", "texmf" }, - { "fonts/new/", "texmf" }, - { "fonts/common/", "texmf" }, - { "fonts/other/", "texmf" }, - { "context/current/", "texmf-context" }, - { "context/img/", "texmf-context" }, - { "misc/setuptex/", "." }, - { "misc/web2c", "texmf" }, - { "bin/common/%s/", "texmf-%s" }, - { "bin/context/%s/", "texmf-%s" }, - { "bin/metapost/%s/", "texmf-%s" }, - { "bin/luatex/%s/", "texmf-%s" }, - { "bin/xetex/%s/", "texmf-%s" }, - { "bin/pdftex/%s/", "texmf-%s" }, - { "bin/man/", "texmf-%s" } + { "base/tex/", "texmf" }, + { "base/metapost/", "texmf" }, + { "base/xetex/", "texmf" }, + { "fonts/old/", "texmf" }, + { "fonts/new/", "texmf" }, + { "fonts/common/", "texmf" }, + { "fonts/other/", "texmf" }, + { "context/<version>/", "texmf-context" }, + { "context/img/", "texmf-context" }, + { "context/config/", "texmf-context" }, + { "misc/setuptex/", "." }, + { "misc/web2c", "texmf" }, + { "bin/common/<platform>/", "texmf-<platform>" }, + { "bin/context/<platform>/", "texmf-<platform>" }, + { "bin/metapost/<platform>/", "texmf-<platform>" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, + { "bin/xetex/<platform>/", "texmf-<platform>" }, + { "bin/pdftex/<platform>/", "texmf-<platform>" }, + { "bin/man/", "texmf-<platform>" } }, } @@ -104,73 +134,59 @@ scripts.update.platforms = { ["osx-ppc"] = "osx-ppc", } -scripts.update.rsyncflagspath = "-rpztlv --stats --delete" -scripts.update.rsyncflagsroot = "-rpztlv --stats" - -function scripts.update.prepare() - local texroot = environment.argument("texroot") or "tex" - local engines = environment.argument("engine") - if engines then - engines = engines:split(",") - else - engines = minimals.config.engines or { "all" } - end - local platforms = environment.argument("platform") - if platforms then - platforms = platforms:split(",") - else - platforms = minimals.config.platform or { os.currentplatform() } - end - return texroot, engines, platforms -end - function scripts.update.run(str) - if environment.argument("dryrun") then - logs.report("run", str) - else + logs.report("run", str) + if environment.argument("force") then -- important, otherwise formats fly to a weird place -- (texlua sets luatex as the engine, we need to reset that or to fix texexec :) os.setenv("engine",nil) - os.spawn(str) + os.execute(str) + end +end + +function scripts.update.fullpath(path) + if input.aux.rootbased_path(path) then + return path + else + return lfs.currentdir() .. "/" .. path end end function scripts.update.synchronize() - local texroot, engines, platforms = scripts.update.prepare() - local dryrun = environment.argument("dryrun") - os.setenv("CYGWIN","nontsec") - local rsyncbin = environment.argument("rsync") or "rsync" - local url = environment.argument("url") or "contextgarden.net::" + logs.report("update","start") + local texroot = scripts.update.fullpath(states.get("paths.root")) + local engines = states.get('engines') + local platforms = states.get('platforms') + local repositories = states.get('repositories') + local bin = states.get("rsync.program") + local url = states.get("rsync.server") + local version = states.get("context.version") + local force = environment.argument("force") if not url:find("::$") then url = url .. "::" end local ok = lfs.attributes(texroot,"mode") == "directory" - if not ok and not dryrun then + if not ok and force then dir.mkdirs(texroot) ok = lfs.attributes(texroot,"mode") == "directory" end - if ok or dryrun then - if not dryrun then + if ok or not force then + if force then dir.mkdirs(string.format("%s/%s", texroot, "texmf-cache")) end - local fetched = { } - local individual = { } - local context = environment.argument("context") - for _, engine in ipairs(engines) do - local collections = scripts.update.collections[engine] + local fetched, individual = { }, { } + for engine, _ in pairs(engines) do + local collections = scripts.update.engines[engine] if collections then for _, collection in ipairs(collections) do - for _, platform in ipairs(platforms) do + for platform, _ in pairs(platforms) do platform = scripts.update.platforms[platform] if platform then - local archive = string.format(collection[1], platform) - local destination = string.format("%s/%s", texroot, string.format(collection[2], platform)) + local archive = collection[1]:gsub("<platform>", platform) + local destination = string.format("%s/%s", texroot, collection[2]:gsub("<platform>", platform)) destination = destination:gsub("\\","/") + archive = archive:gsub("<version>",version) if platform == "windows" or platform == "mswin" then destination = destination:gsub("([a-zA-Z]):/", "/cygdrive/%1/") end - -- if one uses experimental, context=... has no effect - if context and not environment.argument("experimental") then - archive = archive:gsub("/current/", "/" .. context .. "/") - end individual[#individual+1] = { archive, destination } end end @@ -178,20 +194,17 @@ function scripts.update.synchronize() end end local combined = { } - local distributions = { "current" } - -- we need to fetch files from both "current" and "experimental" branch - if environment.argument("experimental") then - distributions = { "experimental", "current" } - end - for _, d in pairs(distributions) do - for _, v in pairs(individual) do - local archive, destination = v[1], v[2] - local cd = combined[destination] - if not cd then - cd = { } - combined[destination] = cd + for _, repository in ipairs(scripts.update.repositories) do + if repositories[repository] then + for _, v in pairs(individual) do + local archive, destination = v[1], v[2] + local cd = combined[destination] + if not cd then + cd = { } + combined[destination] = cd + end + cd[#cd+1] = string.format("%s/%s/%s",states.get('rsync.module'),repository,archive) end - cd[#cd+1] = 'minimals/' .. d .. '/' .. archive end end if input.verbose then @@ -204,10 +217,11 @@ function scripts.update.synchronize() end for destination, archive in pairs(combined) do local archives, command = table.concat(archive," "), "" - if not environment.argument("delete") or destination:find("%.$") then - command = string.format("%s %s %s'%s' %s", rsyncbin, scripts.update.rsyncflagsroot, url, archives, destination) + local normalflags, deleteflags = states.get("rsync.flags.normal"), states.get("rsync.flags.delete") + if true then -- environment.argument("keep") or destination:find("%.$") then + command = string.format("%s %s %s'%s' %s", bin, normalflags, url, archives, destination) else - command = string.format("%s %s %s'%s' %s", rsyncbin, scripts.update.rsyncflagspath, url, archives, destination) + command = string.format("%s %s %s %s'%s' %s", bin, normalflags, deleteflags, url, archives, destination) end logs.report("mtx update", string.format("running command: %s",command)) if not fetched[command] then @@ -218,51 +232,141 @@ function scripts.update.synchronize() else logs.report("mtx update", string.format("no valid texroot: %s",texroot)) end - if environment.argument("make") then - scripts.update.make() + if not force then + logs.report("update", "use --force to really update") + end + logs.report("update","done") +end + +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end end + return h end + function scripts.update.make() - local texroot, engines, platforms = scripts.update.prepare() + logs.report("make","start") + local force = environment.argument("force") + local texroot = scripts.update.fullpath(states.get("paths.root")) + local engines = states.get('engines') + local platforms = states.get('platforms') + local formats = states.get('formats') input.load_tree(texroot) scripts.update.run("mktexlsr") scripts.update.run("luatools --generate") - engines = (engines[1] and engines[1] == "all" and { "pdftex", "xetex", "luatex" }) or engines - for _, engine in ipairs(engines) do - scripts.update.run(string.format("texexec --make --all --fast --%s",engine)) + local formatlist = table.concat(table.fromhash(formats), " ") + if formatlist ~= "" then + for engine in pairs(engines) do + -- todo: just handle make here or in mtxrun --script context --make +--~ os.execute("set") + scripts.update.run(string.format("texexec --make --all --fast --%s %s",engine,formatlist)) + end + end + if not force then + logs.report("make", "use --force to really make") end + logs.report("make","done") end banner = banner .. " | download tools " -input.runners.save_list = { - "update", "engine", "platform", "url", "rsync", "texroot", "dryrun", "make", "delete", "context" -} - messages.help = [[ ---update update minimal tree ---engine tex engine (luatex, pdftex, xetex) ---platform platform (windows, linux, linux-64, osx-intel, osx-ppc) ---url repository url (rsync://contextgarden.net/minimals) ---rsync rsync binary (rsync) +--platform=string platform (windows, linux, linux-64, osx-intel, osx-ppc) +--server=string repository url (rsync://contextgarden.net) +--module=string repository url (minimals) +--repository=string specify version (current, experimental) +--context=string specify version (current, latest, yyyy.mm.dd) +--rsync=string rsync binary (rsync) --texroot installation directory (not guessed for the moment) ---dryrun just show what will be done +--engine tex engine (luatex, pdftex, xetex) +--force instead of a dryrun, do the real thing +--update update minimal tree --make also make formats and generate file databases ---delete delete unused files ---context=string specify version (current, experimental, yyyy.mm.dd) +--keep don't delete unused or obsolete files ]] input.verbose = true +scripts.savestate = true + +if scripts.savestate then + + states.load("status-of-update.lua") + + -- tag, value, default, persistent + + input.starttiming(states) + + states.set("info.version",0.1) -- ok + states.set("info.count",(states.get("info.count") or 0) + 1,1,false) -- ok + states.set("info.comment","this file contains the settings of the last 'mtxrun --script update ' run",false) -- ok + states.set("info.date",os.date("!%Y-%m-%d %H:%M:%S")) -- ok + + states.set("rsync.program", environment.argument("rsync"), "rsync", true) -- ok + states.set("rsync.server", environment.argument("server"), "contextgarden.net::", true) -- ok + states.set("rsync.module", environment.argument("module"), "minimals", true) -- ok + states.set("rsync.flags.normal", environment.argument("flags"), "-rpztlv --stats", true) -- ok + states.set("rsync.flags.delete", nil, "--delete", true) -- ok + + states.set("paths.root", environment.argument("texroot"), "tex", true) -- ok + + states.set("context.version", environment.argument("context"), "current", true) -- ok + + local valid = table.tohash(scripts.update.repositories) + for r in string.gmatch(environment.argument("repository") or "current","([^, ]+)") do + if valid[r] then states.set("repositories." .. r, true) end + end + local valid = scripts.update.engines + for r in string.gmatch(environment.argument("engine") or "all","([^, ]+)") do + if r == "all" then + for k, v in pairs(valid) do + if k ~= "all" then + states.set("engines." .. k, true) + end + end + elseif valid[r] then + states.set("engines." .. r, true) + end + end + local valid = scripts.update.platforms + for r in string.gmatch(environment.argument("platform") or os.currentplatform(),"([^, ]+)") do + if valid[r] then states.set("platforms." .. r, true) end + end + + local valid = table.tohash(scripts.update.formats) + for r in string.gmatch(environment.argument("formats") or "","([^, ]+)") do + if valid[r] then states.set("formats." .. r, true) end + end + + states.set("formats.cont-en", true) + states.set("formats.cont-nl", true) + states.set("formats.metafun", true) + + -- modules + + logs.report("state","loaded") + +end + if environment.argument("update") then - logs.report("update","start") scripts.update.synchronize() - logs.report("update","done") + if environment.argument("make") then + scripts.update.make() + end elseif environment.argument("make") then - logs.report("make","start") scripts.update.make() - logs.report("make","done") else input.help(banner,messages.help) end + +if scripts.savestate then + input.stoptiming(states) + states.set("info.runtime",tonumber(input.elapsedtime(states))) + if environment.argument("force") then + states.save() + logs.report("state","saved") + end +end diff --git a/scripts/context/lua/mtx-watch.lua b/scripts/context/lua/mtx-watch.lua index 96f6f7eb2..f9e81da42 100644 --- a/scripts/context/lua/mtx-watch.lua +++ b/scripts/context/lua/mtx-watch.lua @@ -11,119 +11,123 @@ texmf.instance = instance -- we need to get rid of this / maybe current instance scripts = scripts or { } scripts.watch = scripts.watch or { } -function scripts.watch.watch() - local delay = environment.argument("delay") or 5 - local logpath = environment.argument("logpath") or "" - local pipe = environment.argument("pipe") or false - if #environment.files > 0 then - for _, path in ipairs(environment.files) do - logs.report("watch", "watching path ".. path) - end - local function glob(files,path) - for name in lfs.dir(path) do - if name:find("^%.") then - -- skip . and .. - else - name = path .. "/" .. name - local a = lfs.attributes(name) - if not a then - -- weird - elseif a.mode == "directory" then - if name:find("graphics$") or name:find("figures$") or name:find("resources$") then - -- skip these too - else - glob(files,name) +do + + function scripts.watch.watch() + local delay = environment.argument("delay") or 5 + local logpath = environment.argument("logpath") or "" + local pipe = environment.argument("pipe") or false + if #environment.files > 0 then + for _, path in ipairs(environment.files) do + logs.report("watch", "watching path ".. path) + end + local function glob(files,path) + for name in lfs.dir(path) do + if name:find("^%.") then + -- skip . and .. + else + name = path .. "/" .. name + local a = lfs.attributes(name) + if not a then + -- weird + elseif a.mode == "directory" then + if name:find("graphics$") or name:find("figures$") or name:find("resources$") then + -- skip these too + else + glob(files,name) + end + elseif name:find(".%luj$") then + files[name] = a.change or a.ctime or a.modification or a.mtime end - elseif name:find(".%luj$") then - files[name] = a.change or a.ctime or a.modification or a.mtime end end end - end - local n = 0 - local function process() - local done = false - for _, path in ipairs(environment.files) do - lfs.chdir(path) - local files = { } - glob(files,path) - table.sort(files) -- what gets sorted here - for name, time in pairs(files) do - --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) - local ok, joblog = pcall(dofile,name) - if ok and joblog then - if joblog.status == "processing" then - logs.report("watch",string.format("aborted job, %s added to queue",name)) - joblog.status = "queued" - io.savedata(name, table.serialize(joblog,true)) - elseif joblog.status == "queued" then - local command = joblog.command - if command then - local replacements = { - inputpath = (joblog.paths and joblog.paths.input ) or ".", - outputpath = (joblog.paths and joblog.paths.output) or ".", - filename = joblog.filename or "", - } - command = command:gsub("%%(.-)%%", replacements) - if command ~= "" then - joblog.status = "processing" - joblog.runtime = os.time() -- os.clock() - io.savedata(name, table.serialize(joblog,true)) - logs.report("watch",string.format("running: %s", command)) - local newpath = file.dirname(name) - io.flush() - local result = "" - if newpath ~= "" and newpath ~= "." then - local oldpath = lfs.currentdir() - lfs.chdir(newpath) - if pipe then result = os.resultof(command) else result = os.spawn(command) end - lfs.chdir(oldpath) + local function process() + local done = false + for _, path in ipairs(environment.files) do + lfs.chdir(path) + local files = { } + glob(files,path) + table.sort(files) -- what gets sorted here + for name, time in pairs(files) do + --~ local ok, joblog = xpcall(function() return dofile(name) end, function() end ) + local ok, joblog = pcall(dofile,name) + if ok and joblog then + if joblog.status == "processing" then + logs.report("watch",string.format("aborted job, %s added to queue",name)) + joblog.status = "queued" + io.savedata(name, table.serialize(joblog,true)) + elseif joblog.status == "queued" then + local command = joblog.command + if command then + local replacements = { + inputpath = (joblog.paths and joblog.paths.input ) or ".", + outputpath = (joblog.paths and joblog.paths.output) or ".", + filename = joblog.filename or "", + } + command = command:gsub("%%(.-)%%", replacements) + if command ~= "" then + joblog.status = "processing" + joblog.runtime = os.time() -- os.clock() + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch",string.format("running: %s", command)) + local newpath = file.dirname(name) + io.flush() + local result = "" + if newpath ~= "" and newpath ~= "." then + local oldpath = lfs.currentdir() + lfs.chdir(newpath) + if pipe then result = os.resultof(command) else result = os.spawn(command) end + lfs.chdir(oldpath) + else + if pipe then result = os.resultof(command) else result = os.spawn(command) end + end + logs.report("watch",string.format("return value: %s", result)) + done = true + local path, base = replacements.outputpath, file.basename(replacements.filename) + joblog.runtime = os.time() - joblog.runtime -- os.clock() - joblog.runtime + joblog.result = file.replacesuffix(file.join(path,base),"pdf") + joblog.size = lfs.attributes(joblog.result,"size") + joblog.status = "finished" else - if pipe then result = os.resultof(command) else result = os.spawn(command) end + joblog.status = "invalid command" end - logs.report("watch",string.format("return value: %s", result)) - done = true - local path, base = replacements.outputpath, file.basename(replacements.filename) - joblog.runtime = os.time() - joblog.runtime -- os.clock() - joblog.runtime - joblog.result = file.replacesuffix(file.join(path,base),"pdf") - joblog.size = lfs.attributes(joblog.result,"size") - joblog.status = "finished" else - joblog.status = "invalid command" + joblog.status = "no command" end - else - joblog.status = "no command" - end - -- pcall, when error sleep + again - io.savedata(name, table.serialize(joblog,true)) - if logpath ~= "" then - local name = string.format("%s/%s%04i%09i.lua", logpath, os.time(), math.floor((os.clock()*100)%1000), math.random(99999999)) + -- pcall, when error sleep + again io.savedata(name, table.serialize(joblog,true)) - logs.report("watch", "saving joblog ".. name) + if logpath ~= "" then + local name = string.format("%s/%s%04i%09i.lua", logpath, os.time(), math.floor((os.clock()*100)%1000), math.random(99999999)) + io.savedata(name, table.serialize(joblog,true)) + logs.report("watch", "saving joblog ".. name) + end end end end end end - end - local function wait() - io.flush() - if not done then - n = n + 1 - if n >= 10 then - logs.report("watch", "still sleeping " .. os.clock()) - n = 0 + local n, start = 0, os.clock() + local function wait() + io.flush() + if not done then + n = n + 1 + if n >= 10 then + logs.report("watch", string.format("run time: %i seconds, memory usage: %0.3g MB", os.clock() - start, (status.luastate_bytes/1024)/1000)) + n = 0 + end + os.sleep(delay) end - os.sleep(delay) end + while true do + pcall(process) + pcall(wait) + end + else + logs.report("watch", "no paths to watch") end - while true do - pcall(process) - pcall(wait) - end - else - logs.report("watch", "no paths to watch") end + end function scripts.watch.collect_logs(path) -- clean 'm up too diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index d0c7469e0..a3cbb35e0 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -377,6 +377,31 @@ function string:split_settings() -- no {} handling, see l-aux for lpeg variant end end +local patterns_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["+"] = "%+", + ["*"] = "%*", + ["%"] = "%%", + ["("] = "%)", + [")"] = "%)", + ["["] = "%[", + ["]"] = "%]", +} + + +function string:pattesc() + return (self:gsub(".",patterns_escapes)) +end + +function string:tohash() + local t = { } + for s in self:gmatch("([^, ]+)") do -- lpeg + t[s] = true + end + return t +end + -- filename : l-lpeg.lua -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -420,7 +445,6 @@ end - -- filename : l-table.lua -- comment : split off from luat-lib -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL @@ -502,7 +526,8 @@ end --~ return t --~ end -function table.merge(t, ...) +function table.merge(t, ...) -- first one is target + t = t or {} local lst = {...} for i=1,#lst do for k, v in pairs(lst[i]) do @@ -1031,6 +1056,14 @@ function table.tohash(t) return h end +function table.fromhash(t) + local h = { } + for k, v in pairs(t) do -- no ipairs here + if v then h[#h+1] = k end + end + return h +end + function table.contains(t, v) if t then for i=1, #t do @@ -1062,6 +1095,22 @@ end --~ return table.serialize(a) == table.serialize(b) --~ end +function table.clone(t,p) -- t is optional or nil or table + if not p then + t, p = { }, t or { } + elseif not t then + t = { } + end + setmetatable(t, { __index = function(_,key) return p[key] end }) + return t +end + + +function table.hexed(t,seperator) + local tt = { } + for i=1,#t do tt[i] = string.format("0x%04X",t[i]) end + return table.concat(tt,seperator or " ") +end -- filename : l-io.lua @@ -1133,38 +1182,10 @@ function io.noflines(f) return n end ---~ t, f, n = os.clock(), io.open("testbed/sample-utf16-bigendian-big.txt",'rb'), 0 ---~ for a in io.characters(f) do n = n + 1 end ---~ print(string.format("characters: %s, time: %s", n, os.clock()-t)) - do local sb = string.byte ---~ local nextchar = { ---~ [ 4] = function(f) ---~ return f:read(1), f:read(1), f:read(1), f:read(1) ---~ end, ---~ [ 2] = function(f) ---~ return f:read(1), f:read(1) ---~ end, ---~ [ 1] = function(f) ---~ return f:read(1) ---~ end, ---~ [-2] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ return b, a ---~ end, ---~ [-4] = function(f) ---~ local a = f:read(1) ---~ local b = f:read(1) ---~ local c = f:read(1) ---~ local d = f:read(1) ---~ return d, c, b, a ---~ end ---~ } - local nextchar = { [ 4] = function(f) return f:read(1,1,1,1) @@ -1488,18 +1509,27 @@ function os.resultof(command) return io.popen(command,"r"):read("*all") end -if not os.exec then -- still not ok - os.exec = os.execute -end -if not os.spawn then -- still not ok - os.spawn = os.execute +if not os.exec then os.exec = os.execute end +if not os.spawn then os.spawn = os.execute end + +--~ os.type : windows | unix (new, we already guessed os.platform) +--~ os.name : windows | msdos | linux | macosx | solaris | .. | generic (new) + +if not io.fileseparator then + if string.find(os.getenv("PATH"),";") then + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" + else + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" + end end +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" + function os.launch(str) if os.platform == "windows" then - os.spawn("start " .. str) + os.execute("start " .. str) -- os.spawn ? else - os.spawn(str .. " &") + os.execute(str .. " &") -- os.spawn ? end end @@ -1514,19 +1544,15 @@ if not os.times then -- cstime = children system time function os.times() return { - utime = os.clock(), -- user - stime = 0, -- system - cutime = 0, -- children user - cstime = 0, -- children system + utime = os.gettimeofday(), -- user + stime = 0, -- system + cutime = 0, -- children user + cstime = 0, -- children system } end end -if os.gettimeofday then - os.clock = os.gettimeofday -else - os.gettimeofday = os.clock -end +os.gettimeofday = os.gettimeofday or os.clock do local startuptime = os.gettimeofday() @@ -1553,11 +1579,11 @@ if not versions then versions = { } end versions['l-file'] = 1.001 if not file then file = { } end function file.removesuffix(filename) - return filename:gsub("%.%a+$", "") + return filename:gsub("%.[%a%d]+$", "") end function file.addsuffix(filename, suffix) - if not filename:find("%.%a-$") then + if not filename:find("%.[%a%d]+$") then return filename .. "." .. suffix else return filename @@ -1565,7 +1591,11 @@ function file.addsuffix(filename, suffix) end function file.replacesuffix(filename, suffix) - return (filename:gsub("%.%a+$", "." .. suffix)) + if not filename:find("%.[%a%d]+$") then + return filename .. "." .. suffix + else + return (filename:gsub("%.[%a%d]+$","."..suffix)) + end end function file.dirname(name) @@ -1584,12 +1614,36 @@ function file.extname(name) return name:match("^.+%.([^/\\]-)$") or "" end +function file.stripsuffix(name) + return (name:gsub("%.[%a%d]+$","")) +end + +--~ function file.join(...) +--~ local t = { ... } +--~ for i=1,#t do +--~ t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") +--~ end +--~ return table.concat(t,"/") +--~ end + +--~ print(file.join("x/","/y")) +--~ print(file.join("http://","/y")) +--~ print(file.join("http://a","/y")) +--~ print(file.join("http:///a","/y")) +--~ print(file.join("//nas-1","/y")) + function file.join(...) - local t = { ... } - for i=1,#t do - t[i] = (t[i]:gsub("\\","/")):gsub("/+$","") + local pth = table.concat({...},"/") + pth = pth:gsub("\\","/") + local a, b = pth:match("^(.*://)(.*)$") + if a and b then + return a .. b:gsub("//+","/") + end + a, b = pth:match("^(//)(.*)$") + if a and b then + return a .. b:gsub("//+","/") end - return table.concat(t,"/") + return (pth:gsub("//+","/")) end function file.is_writable(name) @@ -1888,6 +1942,8 @@ xml.trace_lpath = false xml.trace_print = false xml.trace_remap = false +-- todo: some things per xml file, liek namespace remapping + --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by a <l n='url'/>. The following function associates a namespace prefix with a @@ -2506,7 +2562,7 @@ function xml.text(root) return (root and xml.tostring(root)) or "" end -function xml.content(root) +function xml.content(root) -- bugged return (root and root.dt and xml.tostring(root.dt)) or "" end @@ -2837,14 +2893,13 @@ do local t = {...} for i=1,#t do if s == t[i] then return true end end return false end - function xml.traverse(root,pattern,handle,reverse,index,parent,wildcard) + local function traverse(root,pattern,handle,reverse,index,parent,wildcard) if not root then -- error return false elseif pattern == false then -- root handle(root,root.dt,root.ri) return false elseif pattern == true then -- wildcard - local traverse = xml.traverse local rootdt = root.dt if rootdt then local start, stop, step = 1, #rootdt, 1 @@ -2875,7 +2930,7 @@ do elseif command == 11 then -- parent local ep = root.__p__ or parent if index < #pattern then - if not xml.traverse(ep,pattern,handle,reverse,index+1,root) then return false end + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end elseif handle(root,rootdt,k) then return false end @@ -2890,12 +2945,11 @@ do if command == 11 then -- parent local ep = root.__p__ or parent if index < #pattern then - if not xml.traverse(ep,pattern,handle,reverse,index+1,root) then return false end + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end elseif handle(root,rootdt,k) then return false end else - local traverse = xml.traverse local rootdt = root.dt local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 if command == 30 then @@ -2989,6 +3043,7 @@ do if index == #pattern then if handle(root,rootdt,root.ri or k) then return false end if wildcard and multiple then +--~ if wildcard or multiple then if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end else @@ -3049,6 +3104,8 @@ do return true end + xml.traverse = traverse + end --[[ldx-- @@ -3119,7 +3176,7 @@ do traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') return dt and dt[dk], rt, dt, dk end - function xml.filters.count(root, pattern,everything) + function xml.filters.count(root,pattern,everything) local n = 0 traverse(root, lpath(pattern), function(r,d,t) if everything or type(d[t]) == "table" then @@ -3189,13 +3246,15 @@ do local rt, dt, dk traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - return (ekat and ekat[arguments]) or "" + return (ekat and (ekat[arguments] or ekat[arguments:gsub("^([\"\'])(.*)%1$","%2")])) or "" end - function xml.filters.text(root,pattern,arguments) + function xml.filters.text(root,pattern,arguments) -- ?? why index local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) if dtk then local dtkdt = dtk.dt - if #dtkdt == 1 and type(dtkdt[1]) == "string" then + if not dtkdt then + return "", rt, dt, dk + elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then return dtkdt[1], rt, dt, dk else return xml.tostring(dtkdt), rt, dt, dk @@ -3260,7 +3319,7 @@ do </typing> <p>Which will print all the titles in the document. The iterator variant takes - 1.5 times the runtime of the function variant which si due to the overhead in + 1.5 times the runtime of the function variant which is due to the overhead in creating the wrapper. So, instead of:</p> <typing> @@ -3279,6 +3338,10 @@ do return coroutine.wrap(function() traverse(root, lpath(pattern), coroutine.yield, reverse) end) end + function xml.elements_only(root,pattern,reverse) + return coroutine.wrap(function() traverse(root, lpath(pattern), function(r,d,k) coroutine.yield(d[k]) end, reverse) end) + end + function xml.each_element(root, pattern, handle, reverse) local ok traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) @@ -3424,10 +3487,20 @@ do end end - function xml.include(xmldata,pattern,attribute,recursive,findfile) + local function load_data(name) -- == io.loaddata + local f, data = io.open(name), "" + if f then + data = f:read("*all",'b') -- 'b' ? + f:close() + end + return data + end + + function xml.include(xmldata,pattern,attribute,recursive,loaddata) -- parse="text" (default: xml), encoding="" (todo) - pattern = pattern or 'include' -- attribute = attribute or 'href' + pattern = pattern or 'include' + loaddata = loaddata or load_data local function include(r,d,k) local ek, name = d[k], nil if not attribute or attribute == "" then @@ -3442,29 +3515,21 @@ do end end end - if name then - name = (findfile and findfile(name)) or name - if name ~= "" then - local f = io.open(name) - if f then - if ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(f:read("*all")) - else - local xi = xml.load(f) - if recursive then - xml.include(xi,pattern,attribute,recursive,findfile) - end - xml.assign(d,k,xi) - end - f:close() - else - xml.empty(d,k) - end - else + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + xml.empty(d,k) + elseif ek.at["parse"] == "text" then -- for the moment hard coded + d[k] = xml.escaped(data) + else + local xi = xml.convert(data) + if not xi then xml.empty(d,k) + else + if recursive then + xml.include(xi,pattern,attribute,recursive,loaddata) + end + xml.assign(d,k,xi) end - else - xml.empty(d,k) end end xml.each_element(xmldata, pattern, include) @@ -3948,19 +4013,13 @@ os.setlocale(nil,nil) -- useless feature and even dangerous in luatex if not io.fileseparator then if string.find(os.getenv("PATH"),";") then - io.fileseparator, io.pathseparator, os.platform = "\\", ";", "windows" + io.fileseparator, io.pathseparator, os.platform = "\\", ";", os.type or "windows" else - io.fileseparator, io.pathseparator, os.platform = "/" , ":", "unix" + io.fileseparator, io.pathseparator, os.platform = "/" , ":", os.type or "unix" end end -if not os.platform then - if io.pathseparator == ";" then - os.platform = "windows" - else - os.platform = "unix" - end -end +os.platform = os.platform or os.type or (io.pathseparator == ";" and "windows") or "unix" -- arg normalization -- @@ -4159,17 +4218,22 @@ input.formats ['lua'] = 'LUAINPUTS' -- new input.suffixes['lua'] = { 'lua', 'luc', 'tma', 'tmc' } -- here we catch a few new thingies (todo: add these paths to context.tmf) +-- +-- FONTFEATURES = .;$TEXMF/fonts/fea// +-- FONTCIDMAPS = .;$TEXMF/fonts/cid// -function input.checkconfigdata(instance) +function input.checkconfigdata(instance) -- not yet ok, no time for debugging now local function fix(varname,default) local proname = varname .. "." .. instance.progname or "crap" - if not instance.environment[proname] and not instance.variables[proname] == "" and not instance.environment[varname] and not instance.variables[varname] == "" then - instance.variables[varname] = default + local p = instance.environment[proname] + local v = instance.environment[varname] + if not ((p and p ~= "") or (v and v ~= "")) then + instance.variables[varname] = default -- or environment? end end fix("LUAINPUTS" , ".;$TEXINPUTS;$TEXMFSCRIPTS") - fix("FONTFEATURES", ".;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") - fix("FONTCIDMAPS" , ".;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTFEATURES", ".;$TEXMF/fonts/fea//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") + fix("FONTCIDMAPS" , ".;$TEXMF/fonts/cid//;$OPENTYPEFONTS;$TTFONTS;$T1FONTS;$AFMFONTS") end -- backward compatible ones @@ -4270,6 +4334,11 @@ function input.reset() end +function input.reset_hashes(instance) + instance.lists = { } + instance.found = { } +end + function input.bare_variable(str) -- return string.gsub(string.gsub(string.gsub(str,"%s+$",""),'^"(.+)"$',"%1"),"^'(.+)'$","%1") return (str:gsub("\s*([\"\']?)(.+)%1\s*", "%2")) @@ -4335,7 +4404,7 @@ input.settrace(tonumber(os.getenv("MTX.INPUT.TRACE") or os.getenv("MTX_INPUT_TRA -- loading the database files. do - local clock = os.clock + local clock = os.gettimeofday or os.clock function input.starttiming(instance) if instance then @@ -4609,6 +4678,7 @@ function input.aux.extend_texmf_var(instance,specification) -- crap instance.variables['TEXMF'] = "{" .. instance.variables['TEXMF'] .. "}" end input.expand_variables(instance) + input.reset_hashes(instance) end -- locators @@ -4624,28 +4694,6 @@ function input.locatedatabase(instance,specification) return input.methodhandler('locators', instance, specification) end ---~ poor mans solution, from before we had lfs.isdir ---~ ---~ function input.locators.tex(instance,specification) ---~ if specification and specification ~= '' then ---~ local files = { ---~ file.join(specification,'files'..input.lucsuffix), ---~ file.join(specification,'files'..input.luasuffix), ---~ file.join(specification,input.lsrname) ---~ } ---~ for _, filename in pairs(files) do ---~ local f = io.open(filename) ---~ if f then ---~ input.logger('! tex locator', specification..' found') ---~ input.aux.append_hash(instance,'file',specification,filename) ---~ f:close() ---~ return ---~ end ---~ end ---~ input.logger('? tex locator', specification..' not found') ---~ end ---~ end - function input.locators.tex(instance,specification) if specification and specification ~= '' and lfs.isdir(specification) then input.logger('! tex locator', specification..' found') @@ -5004,8 +5052,6 @@ function input.expand_variables(instance) for k,v in pairs(instance.expansions) do instance.expansions[k] = v:gsub("\\", '/') end - -- ########## - --~ input.splitexpansions(instance) -- better not, fuzzy end function input.aux.expand_vars(instance,lst) -- simple vars @@ -5163,15 +5209,12 @@ do end function input.register_extra_path(instance,paths,subpaths) + local ep = instance.extra_paths or { } + local n = #ep if paths and paths ~= "" then - local ep = instance.extra_paths - if not ep then - ep = { } - instance.extra_paths = ep - end - local n = #ep - if subpath and subpaths ~= "" then + if subpaths and subpaths ~= "" then for p in paths:gmatch("[^,]+") do + -- we gmatch each step again, not that fast, but used seldom for s in subpaths:gmatch("[^,]+") do local ps = p .. "/" .. s if not done[ps] then @@ -5188,10 +5231,24 @@ do end end end - if n < #ep then - instance.lists = { } + elseif subpaths and subpaths ~= "" then + for i=1,n do + -- we gmatch each step again, not that fast, but used seldom + for s in subpaths:gmatch("[^,]+") do + local ps = ep[i] .. "/" .. s + if not done[ps] then + ep[#ep+1] = input.clean_path(ps) + done[ps] = true + end + end end end + if #ep > 0 then + instance.extra_paths = ep -- register paths + end + if #ep > n then + instance.lists = { } -- erase the cache + end end end @@ -5446,7 +5503,7 @@ input.is_readable.tex = input.is_readable.file -- name/name function input.aux.collect_files(instance,names) - local filelist = nil + local filelist = { } for _, fname in pairs(names) do if fname then if input.trace > 2 then @@ -5478,15 +5535,20 @@ function input.aux.collect_files(instance,names) if blobfile then if type(blobfile) == 'string' then if not dname or blobfile:find(dname) then - if not filelist then filelist = { } end - -- input.logger('= collected', blobpath.." | "..blobfile.." | "..bname) - filelist[#filelist+1] = file.join(blobpath,blobfile,bname) + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,blobfile,bname), -- search + input.concatinators[hash.type](blobpath,blobfile,bname) -- result + } end else for _, vv in pairs(blobfile) do if not dname or vv:find(dname) then - if not filelist then filelist = { } end - filelist[#filelist+1] = file.join(blobpath,vv,bname) + filelist[#filelist+1] = { + hash.type, + file.join(blobpath,vv,bname), -- search + input.concatinators[hash.type](blobpath,vv,bname) -- result + } end end end @@ -5497,7 +5559,11 @@ function input.aux.collect_files(instance,names) end end end - return filelist + if #filelist > 0 then + return filelist + else + return nil + end end function input.suffix_of_format(str) @@ -5516,54 +5582,30 @@ function input.suffixes_of_format(str) end end ---~ function input.aux.qualified_path(filename) -- make platform dependent / not good yet ---~ return ---~ filename:find("^%.+/") or ---~ filename:find("^/") or ---~ filename:find("^%a+%:") or ---~ filename:find("^%a+##") ---~ end - ---~ function input.normalize_name(original) ---~ -- internally we use type##spec##subspec ; this hackery slightly slows down searching ---~ local str = original or "" ---~ str = str:gsub("::", "##") -- :: -> ## ---~ str = str:gsub("^(%a+)://" ,"%1##") -- zip:// -> zip## ---~ str = str:gsub("(.+)##(.+)##/(.+)","%1##%2##%3") -- ##/spec -> ##spec ---~ if (input.trace>1) and (original ~= str) then ---~ input.logger('= normalizer',original.." -> "..str) ---~ end ---~ return str ---~ end +do -do -- called about 700 times for an empty doc (font initializations etc) + -- called about 700 times for an empty doc (font initializations etc) -- i need to weed the font files for redundant calls local letter = lpeg.R("az","AZ") - local separator = lpeg.P("##") + local separator = lpeg.P("://") - local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator - local normalized = lpeg.Cs( - (letter^1*(lpeg.P("://")/"##") * (1-lpeg.P(false))^1) + - (lpeg.P("::")/"##" + (1-separator)^1*separator*(1-separator)^1*separator*(lpeg.P("/")/"") + 1)^0 - ) + local qualified = lpeg.P(".")^0 * lpeg.P("/") + letter*lpeg.P(":") + letter^1*separator + local rootbased = lpeg.P("/") + letter*lpeg.P(":") - -- ./name ../name /name c: zip## (todo: use url internally and get rid of ##) + -- ./name ../name /name c: :// function input.aux.qualified_path(filename) return qualified:match(filename) end + function input.aux.rootbased_path(filename) + return rootbased:match(filename) + end - -- zip:// -> zip## ; :: -> ## ; aa##bb##/cc -> aa##bb##cc function input.normalize_name(original) - local str = normalized:match(original or "") - if input.trace > 1 and original ~= str then - input.logger('= normalizer',original.." -> "..str) - end - return str + return original end -end --- split the next one up, better for jit +end function input.aux.register_in_trees(instance,name) if not name:find("^%.") then @@ -5571,11 +5613,13 @@ function input.aux.register_in_trees(instance,name) end end +-- split the next one up, better for jit + function input.aux.find_file(instance,filename) -- todo : plugin (scanners, checkers etc) local result = { } local stamp = nil - filename = input.normalize_name(filename) - filename = file.collapse_path(filename:gsub("\\","/")) + filename = input.normalize_name(filename) -- elsewhere + filename = file.collapse_path(filename:gsub("\\","/")) -- elsewhere -- speed up / beware: format problem if instance.remember then stamp = filename .. "--" .. instance.engine .. "--" .. instance.progname .. "--" .. instance.format @@ -5647,7 +5691,7 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec local typespec = input.variable_of_format(filetype) local pathlist = input.expanded_path_list(instance,typespec) if not pathlist or #pathlist == 0 then - -- no pathlist, access check only + -- no pathlist, access check only / todo == wildcard if input.trace > 2 then input.logger('? filename',filename) input.logger('? filetype',filetype or '?') @@ -5662,8 +5706,9 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end -- this is actually 'other text files' or 'any' or 'whatever' local filelist = input.aux.collect_files(instance,wantedfiles) - filename = filelist and filelist[1] - if filename then + local lf = filelist and filelist[1] + if fl then + filename = fl[3] result[#result+1] = filename done = true end @@ -5673,8 +5718,8 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec local doscan, recurse if input.trace > 2 then input.logger('? filename',filename) - if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end - if filelist then input.logger('? file list',table.concat(filelist," | ")) end + -- if pathlist then input.logger('? path list',table.concat(pathlist," | ")) end + -- if filelist then input.logger('? file list',table.concat(filelist," | ")) end end -- a bit messy ... esp the doscan setting here for _, path in pairs(pathlist) do @@ -5687,16 +5732,18 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec -- compare list entries with permitted pattern pathname = pathname:gsub("([%-%.])","%%%1") -- this also influences pathname = pathname:gsub("/+$", '/.*') -- later usage of pathname - pathname = pathname:gsub("//", '/.-/') + pathname = pathname:gsub("//", '/.-/') -- not ok for /// but harmless local expr = "^" .. pathname -- input.debug('?',expr) - for _, f in pairs(filelist) do + for _, fl in ipairs(filelist) do + local f = fl[2] if f:find(expr) then -- input.debug('T',' '..f) if input.trace > 2 then input.logger('= found in hash',f) end - result[#result+1] = f + --- todo, test for readable + result[#result+1] = fl[3] input.aux.register_in_trees(instance,f) -- for tracing used files done = true if not instance.allresults then break end @@ -5706,7 +5753,7 @@ function input.aux.find_file(instance,filename) -- todo : plugin (scanners, chec end end if not done and doscan then - -- check if on disk / unchecked / does not work at all + -- check if on disk / unchecked / does not work at all / also zips if input.method_is_file(pathname) then -- ? local pname = pathname:gsub("%.%*$",'') if not pname:find("%*") then @@ -5783,10 +5830,7 @@ end if not input.concatinators then input.concatinators = { } end -function input.concatinators.tex(tag,path,name) - return tag .. '/' .. path .. '/' .. name -end - +input.concatinators.tex = file.join input.concatinators.file = input.concatinators.tex function input.find_files(instance,filename,filetype,mustexist) @@ -5988,15 +6032,6 @@ function input.aux.register_file(files, name, path) end end --- zip:: zip## zip:// --- zip::pathtozipfile::pathinzipfile (also: pathtozipfile/pathinzipfile) --- file::name --- tex::name --- kpse::name --- kpse::format::name --- parent::n::name --- parent::name (default 2) - if not input.finders then input.finders = { } end if not input.openers then input.openers = { } end if not input.loaders then input.loaders = { } end @@ -6006,30 +6041,37 @@ input.openers.notfound = { nil } input.loaders.notfound = { false, nil, 0 } function input.splitmethod(filename) - local method, specification = filename:match("^(.-)##(.+)$") - if method and specification then - return method, specification + if not filename then + return { } -- safeguard + elseif type(filename) == "table" then + return filename -- already split + elseif not filename:find("://") then + return { scheme="file", path = filename, original=filename } -- quick hack else - return 'tex', filename + return url.hashed(filename) end end function input.method_is_file(filename) - local method, specification = input.splitmethod(filename) - return method == 'tex' or method == 'file' + return input.splitmethod(filename).scheme == 'file' +end + +function table.sequenced(t,sep) -- temp here + local s = { } + for k, v in pairs(t) do + s[#s+1] = k .. "=" .. v + end + return table.concat(s, sep or " | ") end function input.methodhandler(what, instance, filename, filetype) -- ... - local method, specification = input.splitmethod(filename) - if method and specification then -- redundant - if input[what][method] then - input.logger('= handler',filename.." -> "..what.." | "..method.." | "..specification) - return input[what][method](instance,specification,filetype) - else - return nil - end + local specification = (type(filename) == "string" and input.splitmethod(filename)) or filename -- no or { }, let it bomb + local scheme = specification.scheme + if input[what][scheme] then + input.logger('= handler',specification.original .." -> " .. what .. " -> " .. table.sequenced(specification)) + return input[what][scheme](instance,filename,filetype) -- todo: specification else - return input[what].tex(instance,filename,filetype) + return input[what].tex(instance,filename,filetype) -- todo: specification end end @@ -6063,6 +6105,8 @@ function input.texdatablob(instance, filename, filetype) return data or "" end +input.loadtexfile = input.texdatablob + function input.openfile(filename) -- brrr texmf.instance here / todo ! ! ! ! ! local fullname = input.findtexfile(texmf.instance, filename) if fullname and (fullname ~= "") then @@ -6641,7 +6685,7 @@ function input.aux.load_data(instance,pathname,dataname,filename) end end --- we will make a better format, maybe something xml or just text +-- we will make a better format, maybe something xml or just text or lua input.automounted = input.automounted or { } @@ -6879,6 +6923,191 @@ logs.set_level('error') logs.set_method('tex') +if not modules then modules = { } end modules ['luat-sta'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +states = states or { } +states.data = states.data or { } +states.hash = states.hash or { } +states.tag = states.tag or "" +states.filename = states.filename or "" + +function states.save(filename,tag) + tag = tag or states.tag + filename = file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n" .. + "-- state tag : " .. tag .. "\n\n" .. + table.serialize(states.data[tag or states.tag] or {},true) + ) +end + +function states.load(filename,tag) + states.filename = filename + states.tag = tag or "whatever" + states.filename = file.addsuffix(states.filename,'lus') + states.data[states.tag], states.hash[states.tag] = (io.exists(filename) and dofile(filename)) or { }, { } +end + +function states.set_by_tag(tag,key,value,default,persistent) + local d, h = states.data[tag], states.hash[tag] + if d then + local dkey, hkey = key, key + local pre, post = key:match("(.+)%.([^%.]+)$") + if pre and post then + for k in pre:gmatch("[^%.]+") do + local dk = d[k] + if not dk then + dk = { } + d[k] = dk + end + d = dk + end + dkey, hkey = post, key + end + if type(value) == nil then + value = value or default + elseif persistent then + value = value or d[dkey] or default + else + value = value or default + end + d[dkey], h[hkey] = value, value + end +end + +function states.get_by_tag(tag,key,default) + local h = states.hash[tag] + if h and h[key] then + return h[key] + else + local d = states.data[tag] + if d then + for k in key:gmatch("[^%.]+") do + local dk = d[k] + if dk then + d = dk + else + return default + end + end + return d or default + end + end +end + +function states.set(key,value,default,persistent) + states.set_by_tag(states.tag,key,value,default,persistent) +end + +function states.get(key,default) + return states.get_by_tag(states.tag,key,default) +end + +--~ states.data.update = { +--~ ["version"] = { +--~ ["major"] = 0, +--~ ["minor"] = 1, +--~ }, +--~ ["rsync"] = { +--~ ["server"] = "contextgarden.net", +--~ ["module"] = "minimals", +--~ ["repository"] = "current", +--~ ["flags"] = "-rpztlv --stats", +--~ }, +--~ ["tasks"] = { +--~ ["update"] = true, +--~ ["make"] = true, +--~ ["delete"] = false, +--~ }, +--~ ["platform"] = { +--~ ["host"] = true, +--~ ["other"] = { +--~ ["mswin"] = false, +--~ ["linux"] = false, +--~ ["linux-64"] = false, +--~ ["osx-intel"] = false, +--~ ["osx-ppc"] = false, +--~ ["sun"] = false, +--~ }, +--~ }, +--~ ["context"] = { +--~ ["available"] = {"current", "beta", "alpha", "experimental"}, +--~ ["selected"] = "current", +--~ }, +--~ ["formats"] = { +--~ ["cont-en"] = true, +--~ ["cont-nl"] = true, +--~ ["cont-de"] = false, +--~ ["cont-cz"] = false, +--~ ["cont-fr"] = false, +--~ ["cont-ro"] = false, +--~ }, +--~ ["engine"] = { +--~ ["pdftex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["pdftex"] = true, +--~ }, +--~ }, +--~ ["luatex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ }, +--~ }, +--~ ["xetex"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["xetex"] = false, +--~ }, +--~ }, +--~ ["metapost"] = { +--~ ["install"] = true, +--~ ["formats"] = { +--~ ["mpost"] = true, +--~ ["metafun"] = true, +--~ }, +--~ }, +--~ }, +--~ ["fonts"] = { +--~ }, +--~ ["doc"] = { +--~ }, +--~ ["modules"] = { +--~ ["f-urwgaramond"] = false, +--~ ["f-urwgothic"] = false, +--~ ["t-bnf"] = false, +--~ ["t-chromato"] = false, +--~ ["t-cmscbf"] = false, +--~ ["t-cmttbf"] = false, +--~ ["t-construction-plan"] = false, +--~ ["t-degrade"] = false, +--~ ["t-french"] = false, +--~ ["t-lettrine"] = false, +--~ ["t-lilypond"] = false, +--~ ["t-mathsets"] = false, +--~ ["t-tikz"] = false, +--~ ["t-typearea"] = false, +--~ ["t-vim"] = false, +--~ }, +--~ } + + +--~ states.save("teststate", "update") +--~ states.load("teststate", "update") + +--~ print(states.get_by_tag("update","rsync.server","unknown")) +--~ states.set_by_tag("update","rsync.server","oeps") +--~ print(states.get_by_tag("update","rsync.server","unknown")) +--~ states.save("teststate", "update") +--~ states.load("teststate", "update") +--~ print(states.get_by_tag("update","rsync.server","unknown")) + + -- end library merge own = { } @@ -6906,6 +7135,7 @@ own.libs = { -- todo: check which ones are really needed -- 'luat-kps.lua', 'luat-tmp.lua', 'luat-log.lua', + 'luat-sta.lua', } -- We need this hack till luatex is fixed. @@ -7185,6 +7415,7 @@ input.runners.registered = { if not messages then messages = { } end messages.help = [[ +--script run an mtx script --execute run a script or program --resolve resolve prefixed arguments --ctxlua run internally (using preloaded libs) diff --git a/scripts/context/ruby/ctxtools.rb b/scripts/context/ruby/ctxtools.rb index 4998b21b9..de64679ac 100644 --- a/scripts/context/ruby/ctxtools.rb +++ b/scripts/context/ruby/ctxtools.rb @@ -44,7 +44,7 @@ # it cannot do that (it tries to hyphenate as if the "ffi" was a # character), and the result is wrong hyphenation. -banner = ['CtxTools', 'version 1.3.3', '2004/2006', 'PRAGMA ADE/POD'] +banner = ['CtxTools', 'version 1.3.5', '2004/2008', 'PRAGMA ADE'] $: << File.expand_path(File.dirname($0)) ; $: << File.join($:.last,'lib') ; $:.uniq! @@ -866,6 +866,7 @@ class Language @filenames = filenames @remapping = Array.new @demapping = Array.new + @cloning = Array.new @unicode = Hash.new @encoding = encoding @data = '' @@ -895,6 +896,9 @@ class Language def demap(from, to) @demapping.push([from,to]) end + def clone(from, to) + @cloning.push([from,to]) + end def load(filenames=@filenames) found = false @@ -957,6 +961,22 @@ class Language @remapping[$1.to_i][1] end report(" nothing remapped") unless done + @cloning.each_index do |i| + c = 0 + f, s = @cloning[i][0], @cloning[i][1] + str = "#{f}|#{s}" + str.gsub!(/([\[\]])/) do "\\" + "#{$1}" end + reg = /(#{str})/ + content.gsub!(/(\S*(#{str})\S*)/) do + a, b = $1, $1 + a.gsub!(reg, f) + b.gsub!(reg, s) + c = c + 1 + "#{a} #{b}" + end + report("#{c.to_s.rjust(5)} times #{f} cloned to #{s}") + n += c + end report("") content.to_s end @@ -1300,6 +1320,14 @@ class Language remap(/\\a\s*/, "[aeligature]") remap(/\\o\s*/, "[oeligature]") when 'agr' then + # bug fix + remap("a2|", "[greekalphaiotasub]") + remap("h2|", "[greeketaiotasub]") + remap("w2|", "[greekomegaiotasub]") + remap(">2r1<2r", "[2ῤ1ῥ]") + remap(">a2n1wdu'", "[ἀ2ν1ωδύ]") + remap(">e3s2ou'", "[ἐ3σ2ού]") + # main conversion remap(/\<\'a\|/, "[greekalphaiotasubdasiatonos]") # remap(/\<\'a\|/, "[greekdasiatonos][greekAlpha][greekiota]") remap(/\>\'a\|/, "[greekalphaiotasubpsilitonos]") @@ -1399,7 +1427,7 @@ class Language remap(/\<\'u/, "[greekupsilondasiatonos]") remap(/\>\'u/, "[greekupsilonpsilitonos]") remap(/\<\`u/, "[greekupsilondasiavaria]") - remap(/\>\'u/, "[greekupsilonpsilivaria]") + remap(/\>\`u/, "[greekupsilonpsilivaria]") remap(/\<\~u/, "[greekupsilondasiaperispomeni]") remap(/\>\~u/, "[greekupsilonpsiliperispomeni]") remap(/\"\'u/, "[greekupsilondialytikatonos]") @@ -1433,14 +1461,15 @@ class Language remap(/\"\'/, "[greekdialytikatonos]") remap(/\"\`/, "[greekdialytikavaria]") remap(/\"\~/, "[greekdialytikaperispomeni]") - remap(/\</, "[dasia]") - remap(/\>/, "[psili]") - remap(/\'/, "[oxia]") + remap(/\</, "[greekdasia]") + remap(/\>/, "[greekpsili]") + remap(/\d.{0,2}''/, "") + remap(/\'/, "[greekoxia]") remap(/\`/, "[greekvaria]") remap(/\~/, "[perispomeni]") - remap(/\"/, "[dialytika]") + remap(/\"/, "[greekdialytika]") # unknown - remap(/\|/, "[greekIotadialytika]") + # remap(/\|/, "[greekIotadialytika]") # next remap(/A/, "[greekAlpha]") remap(/B/, "[greekBeta]") @@ -1491,6 +1520,13 @@ class Language remap(/x/, "[greekxi]") remap(/y/, "[greekpsi]") remap(/z/, "[greekzeta]") + clone("[greekalphatonos]", "[greekalphaoxia]") + clone("[greekepsilontonos]", "[greekepsilonoxia]") + clone("[greeketatonos]", "[greeketaoxia]") + clone("[greekiotatonos]", "[greekiotaoxia]") + clone("[greekomicrontonos]", "[greekomicronoxia]") + clone("[greekupsilontonos]", "[greekupsilonoxia]") + clone("[greekomegatonos]", "[greekomegaoxia]") when 'ru' then remap(/\xC1/, "[cyrillica]") remap(/\xC2/, "[cyrillicb]") |