beta 2010.01.15 15:49

9 files changed, 400 insertions, 70 deletions

diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua
index faac4ecfb..4822e8b3c 100644
--- a/tex/context/base/buff-ini.lua
+++ b/tex/context/base/buff-ini.lua
@@ -166,7 +166,18 @@ function buffers.loaddata(filename) -- this one might go away
     if not str then
         ok, str, n = resolvers.loaders.tex(file.addsuffix(filename,'tex'))
     end
-    return str or ""
+end
+
+function buffers.loaddata(filename) -- this one might go away
+    local foundname = resolvers.findtexfile(filename) or ""
+    if foundname == ""  then
+        foundname = resolvers.findtexfile(file.addsuffix(filename,'tex')) or ""
+    end
+    if foundname == "" then
+        return ""
+    else
+        return resolvers.loadtexfile(foundname)
+    end
 end
 
 function buffers.typefile(name,realign) -- still somewhat messy, since name can be be suffixless
@@ -411,8 +422,14 @@ end
 
 
 function hooks.flush_line(str,nesting)
-    str = gsub(str," *[\n\r]+ *"," ") ; -- semi colon needed
-    (currenthandler.flush_line or default.flush_line)(str,nesting)
+    local fl = currenthandler.flush_line
+    if fl then
+        str = gsub(str," *[\n\r]+ *"," ") ; -- semi colon needed
+        fl(str,nesting)
+    else
+        -- gsub done later
+        default.flush_line(str,nesting)
+    end
 end
 
 function hooks.flush_inline(str,nesting)
diff --git a/tex/context/base/buff-ver.mkiv b/tex/context/base/buff-ver.mkiv
index 8b9c584f1..80816d0f7 100644
--- a/tex/context/base/buff-ver.mkiv
+++ b/tex/context/base/buff-ver.mkiv
@@ -138,17 +138,28 @@
 
 \newtoks \everyinitializeverbatim
 
+\def\ignorebeginofpretty [#1]{}
+\def\ignoreendofpretty       {}
+
+\def\doverbatimbop{\bgroup\beginofpretty}
+\def\doverbatimeop{\endofpretty\egroup}
+\def\doverbatimsop{\endofpretty\egroup\bgroup\beginofpretty}
+
+\let\noverbatimbop\ignorebeginofpretty
+\let\noverbatimeop\ignoreendofpretty
+\let\noverbatimsop\ignorebeginofpretty
+
 \def\doinitializeverbatim
   {\ctxlua{buffers.visualizers.reset()}%
    \def\obs{\obeyedspace}%
    \ctxlua{buffers.doifelsevisualizer("\prettyidentifier")}
      {\ctxlua{buffers.setvisualizer("\prettyidentifier")}%
-      \def\bop{\bgroup\beginofpretty}%
-      \def\eop{\endofpretty\egroup}%
-      \def\sop{\endofpretty\egroup\bgroup\beginofpretty}}%
-     {\let\bop\donothing
-      \let\eop\donothing
-      \let\sop\donothing}%
+      \let\bop\doverbatimbop
+      \let\eop\doverbatimeop
+      \let\sop\doverbatimsop}%
+     {\let\bop\noverbatimbop
+      \let\eop\noverbatimeop
+      \let\sop\noverbatimsop}%
    \relax\the\everyinitializeverbatim\relax}
 
 \appendtoks
@@ -772,9 +783,6 @@
 \def\normalstopverbatimcolor {\stopcolor}
 \def\normalverbatimcolor     {\getvalue{\typingparameter\c!color}}% command !
 
-\def\ignorebeginofpretty [#1]{}
-\def\ignoreendofpretty       {}
-
 \def\setupnormalprettyverbatim
   {\edef\prettypalet{\prettyidentifier\prettypalet}%
    \let\beginofpretty     \normalbeginofpretty
diff --git a/tex/context/base/core-fil.mkiv b/tex/context/base/core-fil.mkiv
index 1c7e701c7..2f77a0021 100644
--- a/tex/context/base/core-fil.mkiv
+++ b/tex/context/base/core-fil.mkiv
@@ -114,44 +114,47 @@
 %D their dedicated manuals. We use \type {\next} so that we
 %D can \type {\end} in modules.
 
-\newconditional\moduleisloaded
-
-\def\dododousemodules#1#2% no \unprotect/\protect when loading,
-  {\relax                % since we need to use ? ! unprotected
-   \ifconditional\moduleisloaded % sometimes (see xtag-map)
-     \let\next\relax % or: \expandafter\gobbleoneargument
-   \else
-     \makeshortfilename[#1\truefilename{#2}]% beware: *- is not part of syn
-     \doifelseflagged\shortfilename
-       {\showmessage\m!systems7{#2 (line \number\inputlineno)}%
-        \settrue\moduleisloaded
-        \let\next\relax}
-       {\doglobal\setflag\shortfilename
-        \def\next
-          {\startreadingfile
-             \readsysfile\shortfilename
-               {\showmessage\m!systems5{#2}\settrue\moduleisloaded}
-               {\readsysfile{\shortfilename.\mksuffix} % new
-                  {\showmessage\m!systems5{#2 (\mksuffix)}\settrue\moduleisloaded}
-                  \donothing}%
-           \stopreadingfile}}%
-   \fi
-   \next}
+% \newconditional\moduleisloaded
+
+% \def\dododousemodules#1#2% no \unprotect/\protect when loading,
+%   {\relax                % since we need to use ? ! unprotected
+%    \ifconditional\moduleisloaded % sometimes (see xtag-map)
+%      \let\next\relax % or: \expandafter\gobbleoneargument
+%    \else
+%      \makeshortfilename[#1\truefilename{#2}]% beware: *- is not part of syn
+%      \doifelseflagged\shortfilename
+%        {\showmessage\m!systems7{#2 (line \number\inputlineno)}%
+%         \settrue\moduleisloaded
+%         \let\next\relax}
+%        {\doglobal\setflag\shortfilename
+%         \def\next
+%           {\startreadingfile
+%              \readsysfile\shortfilename
+%                {\showmessage\m!systems5{#2}\settrue\moduleisloaded}
+%                {\readsysfile{\shortfilename.\mksuffix} % new
+%                   {\showmessage\m!systems5{#2 (\mksuffix)}\settrue\moduleisloaded}
+%                   \donothing}%
+%            \stopreadingfile}}%
+%    \fi
+%    \next}
+
+% \def\dodousemodules#1#2%
+%   {\setfalse\moduleisloaded
+%    \doifelsenothing{#1}
+%      {\dododousemodules\f!moduleprefix {#2}%
+%       \dododousemodules\f!privateprefix{#2}%
+%       \dododousemodules\f!styleprefix  {#2}%
+%       \dododousemodules\f!xstyleprefix {#2}%
+%       \dododousemodules\f!thirdprefix  {#2}%
+%       \dododousemodules\empty          {#2}}% new, fall back on raw name
+%      {\dododousemodules{#1-}{#2}}%
+%    \ifconditional\moduleisloaded\else
+%      \showmessage\m!systems6{#2}%
+%      \appendtoks\showmessage\m!systems6{#2}\to\everynotabene
+%    \fi}
 
 \def\dodousemodules#1#2%
-  {\setfalse\moduleisloaded
-   \doifelsenothing{#1}
-     {\dododousemodules\f!moduleprefix {#2}%
-      \dododousemodules\f!privateprefix{#2}%
-      \dododousemodules\f!styleprefix  {#2}%
-      \dododousemodules\f!xstyleprefix {#2}%
-      \dododousemodules\f!thirdprefix  {#2}%
-      \dododousemodules\empty          {#2}}% new, fall back on raw name
-     {\dododousemodules{#1-}{#2}}%
-   \ifconditional\moduleisloaded\else
-     \showmessage\m!systems6{#2}%
-     \appendtoks\showmessage\m!systems6{#2}\to\everynotabene
-   \fi}
+  {\ctxlua{support.usemodules("#1","#2","\truefilename{#2}")}}
 
 \def\usemodules
   {\dotripleempty\dousemodules}
diff --git a/tex/context/base/data-sch.lua b/tex/context/base/data-sch.lua
new file mode 100644
index 000000000..e1aaf5eda
--- /dev/null
+++ b/tex/context/base/data-sch.lua
@@ -0,0 +1,142 @@
+if not modules then modules = { } end modules ['data-sch'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local http  = require("socket.http")
+local ltn12 = require("ltn12")
+
+local gsub, concat, format = string.gsub, table.concat, string.format
+
+local trace_schemes = false  trackers.register("resolvers.schemes",function(v) trace_schemes = v end)
+
+schemes = schemes or { }
+
+schemes.cached    = { }
+schemes.cachepath = caches.definepath("schemes")
+schemes.threshold = 24 * 60 * 60
+
+directives.register("schemes.threshold", function(v) schemes.threshold = tonumber(v) or schemes.threshold end)
+
+local cached, loaded, reused = schemes.cached, { }, { }
+
+local finders, openers, loaders = resolvers.finders, resolvers.openers, resolvers.loaders
+
+function schemes.curl(name,cachename)
+    local command = "curl --silent --create-dirs --output " .. cachename .. " " .. name -- no protocol .. "://"
+    os.spawn(command)
+end
+
+function schemes.fetch(protocol,name,handler)
+    local cachename = schemes.cachepath() .. "/" .. gsub(name,"[^%a%d%.]+","-")
+    cachename = gsub(cachename,"[\\]", "/") -- cleanup
+    if not cached[name] then
+        statistics.starttiming(schemes)
+        if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification) > schemes.threshold) then
+            cached[name] = cachename
+            if handler then
+                if trace_schemes then
+                    logs.report("schemes","fetching '%s', protocol '%s', method 'built-in'",name,protocol)
+                end
+                io.flush()
+                handler(protocol,name,cachename)
+            else
+                if trace_schemes then
+                    logs.report("schemes","fetching '%s', protocol '%s', method 'curl'",name,protocol)
+                end
+                io.flush()
+                schemes.curl(name,cachename)
+            end
+        end
+        if io.exists(cachename) then
+            cached[name] = cachename
+            if trace_schemes then
+                logs.report("schemes","using cached '%s', protocol '%s', cachename '%s'",name,protocol,cachename)
+            end
+        else
+            cached[name] = ""
+            if trace_schemes then
+                logs.report("schemes","using missing '%s', protocol '%s'",name,protocol)
+            end
+        end
+        loaded[protocol] = loaded[protocol] + 1
+        statistics.stoptiming(schemes)
+    else
+        if trace_schemes then
+            logs.report("schemes","reusing '%s', protocol '%s'",name,protocol)
+        end
+        reused[protocol] = reused[protocol] + 1
+    end
+    return cached[name]
+end
+
+function finders.schemes(protocol,filename,handler)
+    local foundname = schemes.fetch(protocol,filename,handler)
+    return finders.generic(protocol,foundname,filetype)
+end
+
+function openers.schemes(protocol,filename)
+    return openers.generic(protocol,filename)
+end
+
+function loaders.schemes(protocol,filename)
+    return loaders.generic(protocol,filename)
+end
+
+-- could be metatable
+
+function schemes.install(protocol,handler)
+    loaded [protocol] = 0
+    reused [protocol] = 0
+    finders[protocol] = function (filename,filetype) return finders.schemes(protocol,filename,handler) end
+    openers[protocol] = function (filename)          return openers.schemes(protocol,filename)         end
+    loaders[protocol] = function (filename)          return loaders.schemes(protocol,filename)         end
+end
+
+local function http_handler(protocol,name,cachename)
+    local tempname = cachename .. ".tmp"
+    local f = io.open(tempname,"wb")
+    local status, message = http.request {
+        url = name,
+        sink = ltn12.sink.file(f)
+    }
+    if not status then
+        os.remove(tempname)
+    else
+        os.remove(cachename)
+        os.rename(tempname,cachename)
+    end
+end
+
+schemes.install('http',http_handler)
+schemes.install('https')
+schemes.install('ftp')
+
+statistics.register("scheme handling time", function()
+    local l, r = { }, { }
+    for k, v in table.sortedpairs(loaded) do
+        if v > 0 then
+            l[#l+1] = k .. ":" .. v
+        end
+    end
+    for k, v in table.sortedpairs(reused) do
+        if v > 0 then
+            r[#r+1] = k .. ":" .. v
+        end
+    end
+    local n = #l + #r
+    if n > 0 then
+        l = (#l > 0 and concat(l)) or "none"
+        r = (#r > 0 and concat(r)) or "none"
+        return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s",
+            statistics.elapsedtime(schemes), n, schemes.threshold, l, r)
+    else
+        return nil
+    end
+end)
+
+--~ trace_schemes = true
+--~ print(schemes.fetch("http","http://www.pragma-ade.com/show-man.pdf",http_handler))
diff --git a/tex/context/base/l-url.lua b/tex/context/base/l-url.lua
index 5290df2fc..e3e6f8130 100644
--- a/tex/context/base/l-url.lua
+++ b/tex/context/base/l-url.lua
@@ -42,22 +42,48 @@ local fragment  = hash          * lpeg.Cs((escaped+(1-           endofstring))^0
 
 local parser = lpeg.Ct(scheme * authority * path * query * fragment)
 
+-- todo: reconsider Ct as we can as well have five return values (saves a table)
+-- so we can have two parsers, one with and one without
+
 function url.split(str)
     return (type(str) == "string" and lpegmatch(parser,str)) or str
 end
 
+-- todo: cache them
+
 function url.hashed(str)
     local s = url.split(str)
+    local somescheme = s[1] ~= ""
     return {
-        scheme = (s[1] ~= "" and s[1]) or "file",
+        scheme    = (somescheme and s[1]) or "file",
         authority = s[2],
-        path = s[3],
-        query = s[4],
-        fragment = s[5],
-        original = str
+        path      = s[3],
+        query     = s[4],
+        fragment  = s[5],
+        original  = str,
+        noscheme  = not somescheme,
     }
 end
 
+function url.hasscheme(str)
+    return url.split(str)[1] ~= ""
+end
+
+function url.addscheme(str,scheme)
+    return (url.hasscheme(str) and str) or ((scheme or "file:///") .. str)
+end
+
+function url.construct(hash)
+    local fullurl = hash.sheme .. "://".. hash.authority .. hash.path
+    if hash.query then
+        fullurl = fullurl .. "?".. hash.query
+    end
+    if hash.fragment then
+        fullurl = fullurl .. "?".. hash.fragment
+    end
+    return fullurl
+end
+
 function url.filename(filename)
     local t = url.hashed(filename)
     return (t.scheme == "file" and (gsub(t.path,"^/([a-zA-Z])([:|])/)","%1:"))) or filename
diff --git a/tex/context/base/luat-lib.mkiv b/tex/context/base/luat-lib.mkiv
index 53a2b2379..91ddec0aa 100644
--- a/tex/context/base/luat-lib.mkiv
+++ b/tex/context/base/luat-lib.mkiv
@@ -27,7 +27,8 @@
 \registerctxluafile{data-tex} {1.001}
 \registerctxluafile{data-bin} {1.001}
 \registerctxluafile{data-zip} {1.001}
-\registerctxluafile{data-crl} {1.001}
+%registerctxluafile{data-crl} {1.001}
+\registerctxluafile{data-sch} {1.001}
 \registerctxluafile{data-tre} {1.001}
 \registerctxluafile{data-lua} {1.001}
 \registerctxluafile{data-ctx} {1.001}
diff --git a/tex/context/base/lxml-tex.lua b/tex/context/base/lxml-tex.lua
index a2a295614..6e16388dc 100644
--- a/tex/context/base/lxml-tex.lua
+++ b/tex/context/base/lxml-tex.lua
@@ -1364,7 +1364,6 @@ statistics.register("lxml lpath profile", function()
     end
 end)
 
-
 -- misc
 
 function lxml.nonspace(id,pattern) -- slow, todo loop
diff --git a/tex/context/base/supp-fil.lua b/tex/context/base/supp-fil.lua
index 23b70782a..974e39143 100644
--- a/tex/context/base/supp-fil.lua
+++ b/tex/context/base/supp-fil.lua
@@ -12,8 +12,10 @@ if not modules then modules = { } end modules ['supp-fil'] = {
 at the <l n='tex'/> side.</p>
 --ldx]]--
 
-local find, gsub, match, format = string.find, string.gsub, string.match, string.format
-local texsprint, ctxcatcodes = tex.sprint, tex.ctxcatcodes
+local find, gsub, match, format, concat = string.find, string.gsub, string.match, string.format, table.concat
+local texsprint, texwrite, ctxcatcodes = tex.sprint, tex.write, tex.ctxcatcodes
+
+local trace_modules = false  trackers.register("modules.loading", function(v) trace_modules = v end)
 
 support     = support     or { }
 environment = environment or { }
@@ -25,7 +27,7 @@ function support.checkfilename(str) -- "/whatever..." "c:..." "http://..."
 end
 
 function support.thesanitizedfilename(str)
-    tex.write((gsub(str,"\\","/")))
+    texwrite((gsub(str,"\\","/")))
 end
 
 function support.splitfilename(fullname)
@@ -83,7 +85,7 @@ function support.doiffileexistelse(name)
 end
 
 function support.lastexistingfile()
-    tex.sprint(ctxcatcodes,lastexistingfile)
+    texsprint(ctxcatcodes,lastexistingfile)
 end
 
 -- more, we can cache matches
@@ -94,7 +96,7 @@ local found = { } -- can best be done in the resolver itself
 
 -- todo: tracing
 
-local function readfile(specification,backtrack,treetoo)
+local function readfilename(specification,backtrack,treetoo)
     local fnd = found[specification]
     if not fnd then
         local splitspec = resolvers.splitmethod(specification)
@@ -113,19 +115,22 @@ local function readfile(specification,backtrack,treetoo)
             end
         end
         if not fnd and treetoo then
-            fnd = resolvers.find_file(filename)
+--~             fnd = resolvers.find_file(filename)
+            fnd = resolvers.findtexfile(filename)
         end
         found[specification] = fnd
     end
     return fnd or ""
 end
 
-function finders.job(filename) return readfile(filename,nil,false) end -- current path, no backtracking
-function finders.loc(filename) return readfile(filename,2,  false) end -- current path, backtracking
-function finders.sys(filename) return readfile(filename,nil,true ) end -- current path, obeys tex search
-function finders.fix(filename) return readfile(filename,2,  false) end -- specified path, backtracking
-function finders.set(filename) return readfile(filename,nil,false) end -- specified path, no backtracking
-function finders.any(filename) return readfile(filename,2,  true ) end -- loc job sys
+support.readfilename = readfilename
+
+function finders.job(filename) return readfilename(filename,nil,false) end -- current path, no backtracking
+function finders.loc(filename) return readfilename(filename,2,  false) end -- current path, backtracking
+function finders.sys(filename) return readfilename(filename,nil,true ) end -- current path, obeys tex search
+function finders.fix(filename) return readfilename(filename,2,  false) end -- specified path, backtracking
+function finders.set(filename) return readfilename(filename,nil,false) end -- specified path, no backtracking
+function finders.any(filename) return readfilename(filename,2,  true ) end -- loc job sys
 
 openers.job = openers.generic loaders.job = loaders.generic
 openers.loc = openers.generic loaders.loc = loaders.generic
@@ -134,7 +139,136 @@ openers.fix = openers.generic loaders.fix = loaders.generic
 openers.set = openers.generic loaders.set = loaders.generic
 openers.any = openers.generic loaders.any = loaders.generic
 
-function support.doreadfile(protocol,path,name)
-    local specification = ((path == "") and format("%s:///%s",protocol,name)) or format("%s:///%s/%s",protocol,path,name)
+function support.doreadfile(protocol,path,name) -- better do a split and then pass table
+    local specification
+    if url.hasscheme(name) then
+        specification = name
+    else
+        specification = ((path == "") and format("%s:///%s",protocol,name)) or format("%s:///%s/%s",protocol,path,name)
+    end
     texsprint(ctxcatcodes,resolvers.findtexfile(specification))
 end
+
+-- modules can only have a tex or mkiv suffix or can have a specified one
+
+local prefixes  = { "m", "p", "s", "x", "t" }
+local suffixes  = { "tex", "mkiv" }
+local modstatus = { }
+
+local function usemodule(name,hassheme)
+    local foundname
+    if hasscheme then
+        -- no auto suffix as http will return a home page or error page
+        -- so we only add one if missing
+        local fullname = file.addsuffix(name,"tex")
+        if trace_modules then
+            logs.report("modules","checking scheme driven file '%s'",fullname)
+        end
+        foundname = resolvers.findtexfile(fullname) or ""
+    elseif file.extname(name) ~= "" then
+        if trace_modules then
+            logs.report("modules","checking suffix driven file '%s'",name)
+        end
+        foundname = support.readfilename(name,false,true) or ""
+    else
+        for i=1,#suffixes do
+            local fullname = file.addsuffix(name,suffixes[i])
+            if trace_modules then
+                logs.report("modules","checking suffix driven file '%s'",fullname)
+            end
+            foundname = support.readfilename(fullname,false,true) or ""
+            if foundname ~= "" then
+                break
+            end
+        end
+    end
+    if foundname ~= "" then
+        if trace_modules then
+            logs.report("modules","loading '%s'",foundname)
+        end
+        context.startreadingfile()
+        context.input(foundname)
+        context.stopreadingfile()
+        return true
+    else
+        return false
+    end
+end
+
+function support.usemodules(prefix,askedname,truename)
+    local status = modstatus[truename]
+    if status == 0 then
+        -- not found
+    elseif status == 1 then
+        status = status + 1
+    else
+        if trace_modules then
+            logs.report("modules","locating '%s'",truename)
+        end
+        local hasscheme = url.hasscheme(truename)
+        if hasscheme then
+            -- no prefix and suffix done
+            if usemodule(truename,true) then
+                status = 1
+            else
+                status = 0
+            end
+        elseif prefix and prefix ~= "" then
+            if usemodule(prefix .. "-" .. truename) then
+                status = 1
+            else
+                status = 0
+            end
+        else
+            for i=1,#prefixes do
+                -- todo: reconstruct name i.e. basename
+                if usemodule(prefixes[i] .. "-" .. truename) then
+                    status = 1
+                    break
+                end
+            end
+            if not status and usemodule(truename) then
+                status = 1
+            else
+                status = 0
+            end
+        end
+    end
+    if status == 0 then
+        if trace_modules then
+            logs.report("modules","skipping '%s' (not found)",truename)
+        else
+            context.showmessage("systems",6,askedname)
+        end
+    elseif status == 1 then
+        if not trace_modules then
+            context.showmessage("systems",5,askedname)
+        end
+    else
+        if trace_modules then
+            logs.report("modules","skipping '%s' (already loaded)",truename)
+        else
+            context.showmessage("systems",7,askedname)
+        end
+    end
+    modstatus[truename] = status
+end
+
+statistics.register("loaded tex modules", function()
+    if next(modstatus) then
+        local t, f = { }, { }
+        for k, v in table.sortedpairs(modstatus) do
+            k = file.basename(k)
+            if v == 0 then
+                f[#f+1] = k
+            else
+                t[#t+1] = k
+            end
+        end
+        local ts = (#t>0 and format(" (%s)",concat(t," "))) or ""
+        local fs = (#f>0 and format(" (%s)",concat(f," "))) or ""
+        return format("%s requested, %s found%s, %s missing%s",#t+#f,#t,ts,#f,fs)
+    else
+        return nil
+    end
+end)
diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua
index 5248cfd85..01927ba5a 100644
--- a/tex/generic/context/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/texmf/tex/generic/context/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/texmf/tex/generic/context/luatex-fonts.lua
--- merge date  : 01/14/10 18:30:13
+-- merge date  : 01/15/10 15:53:50
 
 do -- begin closure to overcome local limits and interference