From c3fb9b7d537059ec737147e65049e54585a9f777 Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Fri, 15 Jan 2010 15:49:00 +0100 Subject: beta 2010.01.15 15:49 --- tex/context/base/buff-ini.lua | 23 +++- tex/context/base/buff-ver.mkiv | 26 +++-- tex/context/base/core-fil.mkiv | 75 ++++++------- tex/context/base/data-sch.lua | 142 ++++++++++++++++++++++++ tex/context/base/l-url.lua | 36 ++++++- tex/context/base/luat-lib.mkiv | 3 +- tex/context/base/lxml-tex.lua | 1 - tex/context/base/supp-fil.lua | 162 +++++++++++++++++++++++++--- tex/generic/context/luatex-fonts-merged.lua | 2 +- 9 files changed, 400 insertions(+), 70 deletions(-) create mode 100644 tex/context/base/data-sch.lua (limited to 'tex') diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua index faac4ecfb..4822e8b3c 100644 --- a/tex/context/base/buff-ini.lua +++ b/tex/context/base/buff-ini.lua @@ -166,7 +166,18 @@ function buffers.loaddata(filename) -- this one might go away if not str then ok, str, n = resolvers.loaders.tex(file.addsuffix(filename,'tex')) end - return str or "" +end + +function buffers.loaddata(filename) -- this one might go away + local foundname = resolvers.findtexfile(filename) or "" + if foundname == "" then + foundname = resolvers.findtexfile(file.addsuffix(filename,'tex')) or "" + end + if foundname == "" then + return "" + else + return resolvers.loadtexfile(foundname) + end end function buffers.typefile(name,realign) -- still somewhat messy, since name can be be suffixless @@ -411,8 +422,14 @@ end function hooks.flush_line(str,nesting) - str = gsub(str," *[\n\r]+ *"," ") ; -- semi colon needed - (currenthandler.flush_line or default.flush_line)(str,nesting) + local fl = currenthandler.flush_line + if fl then + str = gsub(str," *[\n\r]+ *"," ") ; -- semi colon needed + fl(str,nesting) + else + -- gsub done later + default.flush_line(str,nesting) + end end function hooks.flush_inline(str,nesting) diff --git a/tex/context/base/buff-ver.mkiv b/tex/context/base/buff-ver.mkiv index 8b9c584f1..80816d0f7 100644 --- a/tex/context/base/buff-ver.mkiv +++ b/tex/context/base/buff-ver.mkiv @@ -138,17 +138,28 @@ \newtoks \everyinitializeverbatim +\def\ignorebeginofpretty [#1]{} +\def\ignoreendofpretty {} + +\def\doverbatimbop{\bgroup\beginofpretty} +\def\doverbatimeop{\endofpretty\egroup} +\def\doverbatimsop{\endofpretty\egroup\bgroup\beginofpretty} + +\let\noverbatimbop\ignorebeginofpretty +\let\noverbatimeop\ignoreendofpretty +\let\noverbatimsop\ignorebeginofpretty + \def\doinitializeverbatim {\ctxlua{buffers.visualizers.reset()}% \def\obs{\obeyedspace}% \ctxlua{buffers.doifelsevisualizer("\prettyidentifier")} {\ctxlua{buffers.setvisualizer("\prettyidentifier")}% - \def\bop{\bgroup\beginofpretty}% - \def\eop{\endofpretty\egroup}% - \def\sop{\endofpretty\egroup\bgroup\beginofpretty}}% - {\let\bop\donothing - \let\eop\donothing - \let\sop\donothing}% + \let\bop\doverbatimbop + \let\eop\doverbatimeop + \let\sop\doverbatimsop}% + {\let\bop\noverbatimbop + \let\eop\noverbatimeop + \let\sop\noverbatimsop}% \relax\the\everyinitializeverbatim\relax} \appendtoks @@ -772,9 +783,6 @@ \def\normalstopverbatimcolor {\stopcolor} \def\normalverbatimcolor {\getvalue{\typingparameter\c!color}}% command ! -\def\ignorebeginofpretty [#1]{} -\def\ignoreendofpretty {} - \def\setupnormalprettyverbatim {\edef\prettypalet{\prettyidentifier\prettypalet}% \let\beginofpretty \normalbeginofpretty diff --git a/tex/context/base/core-fil.mkiv b/tex/context/base/core-fil.mkiv index 1c7e701c7..2f77a0021 100644 --- a/tex/context/base/core-fil.mkiv +++ b/tex/context/base/core-fil.mkiv @@ -114,44 +114,47 @@ %D their dedicated manuals. We use \type {\next} so that we %D can \type {\end} in modules. -\newconditional\moduleisloaded - -\def\dododousemodules#1#2% no \unprotect/\protect when loading, - {\relax % since we need to use ? ! unprotected - \ifconditional\moduleisloaded % sometimes (see xtag-map) - \let\next\relax % or: \expandafter\gobbleoneargument - \else - \makeshortfilename[#1\truefilename{#2}]% beware: *- is not part of syn - \doifelseflagged\shortfilename - {\showmessage\m!systems7{#2 (line \number\inputlineno)}% - \settrue\moduleisloaded - \let\next\relax} - {\doglobal\setflag\shortfilename - \def\next - {\startreadingfile - \readsysfile\shortfilename - {\showmessage\m!systems5{#2}\settrue\moduleisloaded} - {\readsysfile{\shortfilename.\mksuffix} % new - {\showmessage\m!systems5{#2 (\mksuffix)}\settrue\moduleisloaded} - \donothing}% - \stopreadingfile}}% - \fi - \next} +% \newconditional\moduleisloaded + +% \def\dododousemodules#1#2% no \unprotect/\protect when loading, +% {\relax % since we need to use ? ! unprotected +% \ifconditional\moduleisloaded % sometimes (see xtag-map) +% \let\next\relax % or: \expandafter\gobbleoneargument +% \else +% \makeshortfilename[#1\truefilename{#2}]% beware: *- is not part of syn +% \doifelseflagged\shortfilename +% {\showmessage\m!systems7{#2 (line \number\inputlineno)}% +% \settrue\moduleisloaded +% \let\next\relax} +% {\doglobal\setflag\shortfilename +% \def\next +% {\startreadingfile +% \readsysfile\shortfilename +% {\showmessage\m!systems5{#2}\settrue\moduleisloaded} +% {\readsysfile{\shortfilename.\mksuffix} % new +% {\showmessage\m!systems5{#2 (\mksuffix)}\settrue\moduleisloaded} +% \donothing}% +% \stopreadingfile}}% +% \fi +% \next} + +% \def\dodousemodules#1#2% +% {\setfalse\moduleisloaded +% \doifelsenothing{#1} +% {\dododousemodules\f!moduleprefix {#2}% +% \dododousemodules\f!privateprefix{#2}% +% \dododousemodules\f!styleprefix {#2}% +% \dododousemodules\f!xstyleprefix {#2}% +% \dododousemodules\f!thirdprefix {#2}% +% \dododousemodules\empty {#2}}% new, fall back on raw name +% {\dododousemodules{#1-}{#2}}% +% \ifconditional\moduleisloaded\else +% \showmessage\m!systems6{#2}% +% \appendtoks\showmessage\m!systems6{#2}\to\everynotabene +% \fi} \def\dodousemodules#1#2% - {\setfalse\moduleisloaded - \doifelsenothing{#1} - {\dododousemodules\f!moduleprefix {#2}% - \dododousemodules\f!privateprefix{#2}% - \dododousemodules\f!styleprefix {#2}% - \dododousemodules\f!xstyleprefix {#2}% - \dododousemodules\f!thirdprefix {#2}% - \dododousemodules\empty {#2}}% new, fall back on raw name - {\dododousemodules{#1-}{#2}}% - \ifconditional\moduleisloaded\else - \showmessage\m!systems6{#2}% - \appendtoks\showmessage\m!systems6{#2}\to\everynotabene - \fi} + {\ctxlua{support.usemodules("#1","#2","\truefilename{#2}")}} \def\usemodules {\dotripleempty\dousemodules} diff --git a/tex/context/base/data-sch.lua b/tex/context/base/data-sch.lua new file mode 100644 index 000000000..e1aaf5eda --- /dev/null +++ b/tex/context/base/data-sch.lua @@ -0,0 +1,142 @@ +if not modules then modules = { } end modules ['data-sch'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local http = require("socket.http") +local ltn12 = require("ltn12") + +local gsub, concat, format = string.gsub, table.concat, string.format + +local trace_schemes = false trackers.register("resolvers.schemes",function(v) trace_schemes = v end) + +schemes = schemes or { } + +schemes.cached = { } +schemes.cachepath = caches.definepath("schemes") +schemes.threshold = 24 * 60 * 60 + +directives.register("schemes.threshold", function(v) schemes.threshold = tonumber(v) or schemes.threshold end) + +local cached, loaded, reused = schemes.cached, { }, { } + +local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders + +function schemes.curl(name,cachename) + local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://" + os.spawn(command) +end + +function schemes.fetch(protocol,name,handler) + local cachename = schemes.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-") + cachename = gsub(cachename,"[\\]", "/") -- cleanup + if not cached[name] then + statistics.starttiming(schemes) + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > schemes.threshold) then + cached[name] = cachename + if handler then + if trace_schemes then + logs.report("schemes","fetching '%s', protocol '%s', method 'built-in'",name,protocol) + end + io.flush() + handler(protocol,name,cachename) + else + if trace_schemes then + logs.report("schemes","fetching '%s', protocol '%s', method 'curl'",name,protocol) + end + io.flush() + schemes.curl(name,cachename) + end + end + if io.exists(cachename) then + cached[name] = cachename + if trace_schemes then + logs.report("schemes","using cached '%s', protocol '%s', cachename '%s'",name,protocol,cachename) + end + else + cached[name] = "" + if trace_schemes then + logs.report("schemes","using missing '%s', protocol '%s'",name,protocol) + end + end + loaded[protocol] = loaded[protocol] + 1 + statistics.stoptiming(schemes) + else + if trace_schemes then + logs.report("schemes","reusing '%s', protocol '%s'",name,protocol) + end + reused[protocol] = reused[protocol] + 1 + end + return cached[name] +end + +function finders.schemes(protocol,filename,handler) + local foundname = schemes.fetch(protocol,filename,handler) + return finders.generic(protocol,foundname,filetype) +end + +function openers.schemes(protocol,filename) + return openers.generic(protocol,filename) +end + +function loaders.schemes(protocol,filename) + return loaders.generic(protocol,filename) +end + +-- could be metatable + +function schemes.install(protocol,handler) + loaded [protocol] = 0 + reused [protocol] = 0 + finders[protocol] = function (filename,filetype) return finders.schemes(protocol,filename,handler) end + openers[protocol] = function (filename) return openers.schemes(protocol,filename) end + loaders[protocol] = function (filename) return loaders.schemes(protocol,filename) end +end + +local function http_handler(protocol,name,cachename) + local tempname = cachename .. ".tmp" + local f = io.open(tempname,"wb") + local status, message = http.request { + url = name, + sink = ltn12.sink.file(f) + } + if not status then + os.remove(tempname) + else + os.remove(cachename) + os.rename(tempname,cachename) + end +end + +schemes.install('http',http_handler) +schemes.install('https') +schemes.install('ftp') + +statistics.register("scheme handling time", function() + local l, r = { }, { } + for k, v in table.sortedpairs(loaded) do + if v > 0 then + l[#l+1] = k .. ":" .. v + end + end + for k, v in table.sortedpairs(reused) do + if v > 0 then + r[#r+1] = k .. ":" .. v + end + end + local n = #l + #r + if n > 0 then + l = (#l > 0 and concat(l)) or "none" + r = (#r > 0 and concat(r)) or "none" + return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", + statistics.elapsedtime(schemes), n, schemes.threshold, l, r) + else + return nil + end +end) + +--~ trace_schemes = true +--~ print(schemes.fetch("http","http://www.pragma-ade.com/show-man.pdf",http_handler)) diff --git a/tex/context/base/l-url.lua b/tex/context/base/l-url.lua index 5290df2fc..e3e6f8130 100644 --- a/tex/context/base/l-url.lua +++ b/tex/context/base/l-url.lua @@ -42,22 +42,48 @@ local fragment = hash * lpeg.Cs((escaped+(1- endofstring))^0 local parser = lpeg.Ct(scheme * authority * path * query * fragment) +-- todo: reconsider Ct as we can as well have five return values (saves a table) +-- so we can have two parsers, one with and one without + function url.split(str) return (type(str) == "string" and lpegmatch(parser,str)) or str end +-- todo: cache them + function url.hashed(str) local s = url.split(str) + local somescheme = s[1] ~= "" return { - scheme = (s[1] ~= "" and s[1]) or "file", + scheme = (somescheme and s[1]) or "file", authority = s[2], - path = s[3], - query = s[4], - fragment = s[5], - original = str + path = s[3], + query = s[4], + fragment = s[5], + original = str, + noscheme = not somescheme, } end +function url.hasscheme(str) + return url.split(str)[1] ~= "" +end + +function url.addscheme(str,scheme) + return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str) +end + +function url.construct(hash) + local fullurl = hash.sheme .. "://".. hash.authority .. hash.path + if hash.query then + fullurl = fullurl .. "?".. hash.query + end + if hash.fragment then + fullurl = fullurl .. "?".. hash.fragment + end + return fullurl +end + function url.filename(filename) local t = url.hashed(filename) return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename diff --git a/tex/context/base/luat-lib.mkiv b/tex/context/base/luat-lib.mkiv index 53a2b2379..91ddec0aa 100644 --- a/tex/context/base/luat-lib.mkiv +++ b/tex/context/base/luat-lib.mkiv @@ -27,7 +27,8 @@ \registerctxluafile{data-tex} {1.001} \registerctxluafile{data-bin} {1.001} \registerctxluafile{data-zip} {1.001} -\registerctxluafile{data-crl} {1.001} +%registerctxluafile{data-crl} {1.001} +\registerctxluafile{data-sch} {1.001} \registerctxluafile{data-tre} {1.001} \registerctxluafile{data-lua} {1.001} \registerctxluafile{data-ctx} {1.001} diff --git a/tex/context/base/lxml-tex.lua b/tex/context/base/lxml-tex.lua index a2a295614..6e16388dc 100644 --- a/tex/context/base/lxml-tex.lua +++ b/tex/context/base/lxml-tex.lua @@ -1364,7 +1364,6 @@ statistics.register("lxml lpath profile", function() end end) - -- misc function lxml.nonspace(id,pattern) -- slow, todo loop diff --git a/tex/context/base/supp-fil.lua b/tex/context/base/supp-fil.lua index 23b70782a..974e39143 100644 --- a/tex/context/base/supp-fil.lua +++ b/tex/context/base/supp-fil.lua @@ -12,8 +12,10 @@ if not modules then modules = { } end modules ['supp-fil'] = { at the side.

--ldx]]-- -local find, gsub, match, format = string.find, string.gsub, string.match, string.format -local texsprint, ctxcatcodes = tex.sprint, tex.ctxcatcodes +local find, gsub, match, format, concat = string.find, string.gsub, string.match, string.format, table.concat +local texsprint, texwrite, ctxcatcodes = tex.sprint, tex.write, tex.ctxcatcodes + +local trace_modules = false trackers.register("modules.loading", function(v) trace_modules = v end) support = support or { } environment = environment or { } @@ -25,7 +27,7 @@ function support.checkfilename(str) -- "/whatever..." "c:..." "http://..." end function support.thesanitizedfilename(str) - tex.write((gsub(str,"\\","/"))) + texwrite((gsub(str,"\\","/"))) end function support.splitfilename(fullname) @@ -83,7 +85,7 @@ function support.doiffileexistelse(name) end function support.lastexistingfile() - tex.sprint(ctxcatcodes,lastexistingfile) + texsprint(ctxcatcodes,lastexistingfile) end -- more, we can cache matches @@ -94,7 +96,7 @@ local found = { } -- can best be done in the resolver itself -- todo: tracing -local function readfile(specification,backtrack,treetoo) +local function readfilename(specification,backtrack,treetoo) local fnd = found[specification] if not fnd then local splitspec = resolvers.splitmethod(specification) @@ -113,19 +115,22 @@ local function readfile(specification,backtrack,treetoo) end end if not fnd and treetoo then - fnd = resolvers.find_file(filename) +--~ fnd = resolvers.find_file(filename) + fnd = resolvers.findtexfile(filename) end found[specification] = fnd end return fnd or "" end -function finders.job(filename) return readfile(filename,nil,false) end -- current path, no backtracking -function finders.loc(filename) return readfile(filename,2, false) end -- current path, backtracking -function finders.sys(filename) return readfile(filename,nil,true ) end -- current path, obeys tex search -function finders.fix(filename) return readfile(filename,2, false) end -- specified path, backtracking -function finders.set(filename) return readfile(filename,nil,false) end -- specified path, no backtracking -function finders.any(filename) return readfile(filename,2, true ) end -- loc job sys +support.readfilename = readfilename + +function finders.job(filename) return readfilename(filename,nil,false) end -- current path, no backtracking +function finders.loc(filename) return readfilename(filename,2, false) end -- current path, backtracking +function finders.sys(filename) return readfilename(filename,nil,true ) end -- current path, obeys tex search +function finders.fix(filename) return readfilename(filename,2, false) end -- specified path, backtracking +function finders.set(filename) return readfilename(filename,nil,false) end -- specified path, no backtracking +function finders.any(filename) return readfilename(filename,2, true ) end -- loc job sys openers.job = openers.generic loaders.job = loaders.generic openers.loc = openers.generic loaders.loc = loaders.generic @@ -134,7 +139,136 @@ openers.fix = openers.generic loaders.fix = loaders.generic openers.set = openers.generic loaders.set = loaders.generic openers.any = openers.generic loaders.any = loaders.generic -function support.doreadfile(protocol,path,name) - local specification = ((path == "") and format("%s:///%s",protocol,name)) or format("%s:///%s/%s",protocol,path,name) +function support.doreadfile(protocol,path,name) -- better do a split and then pass table + local specification + if url.hasscheme(name) then + specification = name + else + specification = ((path == "") and format("%s:///%s",protocol,name)) or format("%s:///%s/%s",protocol,path,name) + end texsprint(ctxcatcodes,resolvers.findtexfile(specification)) end + +-- modules can only have a tex or mkiv suffix or can have a specified one + +local prefixes = { "m", "p", "s", "x", "t" } +local suffixes = { "tex", "mkiv" } +local modstatus = { } + +local function usemodule(name,hassheme) + local foundname + if hasscheme then + -- no auto suffix as http will return a home page or error page + -- so we only add one if missing + local fullname = file.addsuffix(name,"tex") + if trace_modules then + logs.report("modules","checking scheme driven file '%s'",fullname) + end + foundname = resolvers.findtexfile(fullname) or "" + elseif file.extname(name) ~= "" then + if trace_modules then + logs.report("modules","checking suffix driven file '%s'",name) + end + foundname = support.readfilename(name,false,true) or "" + else + for i=1,#suffixes do + local fullname = file.addsuffix(name,suffixes[i]) + if trace_modules then + logs.report("modules","checking suffix driven file '%s'",fullname) + end + foundname = support.readfilename(fullname,false,true) or "" + if foundname ~= "" then + break + end + end + end + if foundname ~= "" then + if trace_modules then + logs.report("modules","loading '%s'",foundname) + end + context.startreadingfile() + context.input(foundname) + context.stopreadingfile() + return true + else + return false + end +end + +function support.usemodules(prefix,askedname,truename) + local status = modstatus[truename] + if status == 0 then + -- not found + elseif status == 1 then + status = status + 1 + else + if trace_modules then + logs.report("modules","locating '%s'",truename) + end + local hasscheme = url.hasscheme(truename) + if hasscheme then + -- no prefix and suffix done + if usemodule(truename,true) then + status = 1 + else + status = 0 + end + elseif prefix and prefix ~= "" then + if usemodule(prefix .. "-" .. truename) then + status = 1 + else + status = 0 + end + else + for i=1,#prefixes do + -- todo: reconstruct name i.e. basename + if usemodule(prefixes[i] .. "-" .. truename) then + status = 1 + break + end + end + if not status and usemodule(truename) then + status = 1 + else + status = 0 + end + end + end + if status == 0 then + if trace_modules then + logs.report("modules","skipping '%s' (not found)",truename) + else + context.showmessage("systems",6,askedname) + end + elseif status == 1 then + if not trace_modules then + context.showmessage("systems",5,askedname) + end + else + if trace_modules then + logs.report("modules","skipping '%s' (already loaded)",truename) + else + context.showmessage("systems",7,askedname) + end + end + modstatus[truename] = status +end + +statistics.register("loaded tex modules", function() + if next(modstatus) then + local t, f = { }, { } + for k, v in table.sortedpairs(modstatus) do + k = file.basename(k) + if v == 0 then + f[#f+1] = k + else + t[#t+1] = k + end + end + local ts = (#t>0 and format(" (%s)",concat(t," "))) or "" + local fs = (#f>0 and format(" (%s)",concat(f," "))) or "" + return format("%s requested, %s found%s, %s missing%s",#t+#f,#t,ts,#f,fs) + else + return nil + end +end) diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua index 5248cfd85..01927ba5a 100644 --- a/tex/generic/context/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/texmf/tex/generic/context/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/texmf/tex/generic/context/luatex-fonts.lua --- merge date : 01/14/10 18:30:13 +-- merge date : 01/15/10 15:53:50 do -- begin closure to overcome local limits and interference -- cgit v1.2.3