beta 2009.12.18 11:12

7 files changed, 342 insertions, 164 deletions

diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua
index 06e2a1bfc..ae5122509 100644
--- a/scripts/context/lua/luatools.lua
+++ b/scripts/context/lua/luatools.lua
@@ -322,6 +322,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+lpeg = require("lpeg")
+
 local P, R, S, Ct, C, Cs, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
@@ -1383,7 +1385,7 @@ function io.savedata(filename,data,joiner)
         elseif type(data) == "function" then
             data(f)
         else
-            f:write(data)
+            f:write(data or "")
         end
         f:close()
         return true
@@ -1702,7 +1704,13 @@ local find, format = string.find, string.format
 local random, ceil = math.random, math.ceil
 
 function os.resultof(command)
-    return io.popen(command,"r"):read("*all")
+    local handle = io.popen(command,"r")
+    if not handle then
+    --  print("unknown command '".. command .. "' in os.resultof")
+        return ""
+    else
+        return handle:read("*all") or ""
+    end
 end
 
 if not os.exec  then os.exec  = os.execute end
@@ -1762,14 +1770,6 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
-os.arch = os.arch or function()
-    local a = os.resultof("uname -m") or "linux"
-    os.arch = function()
-        return a
-    end
-    return a
-end
-
 -- no need for function anymore as we have more clever code and helpers now
 
 os.platform  = os.name
@@ -1785,7 +1785,13 @@ if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" th
     end
     os.libsuffix = 'dll'
 else
-    local architecture = os.arch()
+    local architecture = os.getenv("HOSTTYPE") or ""
+    if architecture == "" then
+        architecture = os.resultof("uname -m") or ""
+    end
+    if architecture == "" then
+        local architecture = os.resultof("echo $HOSTTYPE")
+    end
     if name == "linux" then
         if find(architecture,"x86_64") then
             os.platform = "linux-64"
@@ -1795,7 +1801,6 @@ else
             os.platform = "linux"
         end
     elseif name == "macosx" then
-        local architecture = os.resultof("echo $HOSTTYPE")
         if find(architecture,"i386") then
             os.platform = "osx-intel"
         elseif find(architecture,"x86_64") then
@@ -2720,13 +2725,34 @@ if not modules then modules = { } end modules ['l-unicode'] = {
     license   = "see context related readme files"
 }
 
+if not unicode then
+
+    unicode = { utf8 = { } }
+
+    local floor, char = math.floor, string.char
+
+    function unicode.utf8.utfchar(n)
+        if n < 0x80 then
+            return char(n)
+        elseif n < 0x800 then
+            return char(0xC0 + floor(n/0x40))  .. char(0x80 + (n % 0x40))
+        elseif n < 0x10000 then
+            return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+        elseif n < 0x40000 then
+            return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+        else -- wrong:
+          -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+            return "?"
+        end
+    end
+
+end
+
 utf = utf or unicode.utf8
 
 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
 local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs
 
-unicode = unicode or { }
-
 -- 0  EF BB BF      UTF-8
 -- 1  FF FE         UTF-16-little-endian
 -- 2  FE FF         UTF-16-big-endian
@@ -4238,12 +4264,12 @@ function logs.tex.stop_page_number()
     if real > 0 then
         if user > 0 then
             if sub > 0 then
-                logs.report("pages", "flushing page, realpage %s, userpage %s, subpage %s",real,user,sub)
+                logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub)
             else
-                logs.report("pages", "flushing page, realpage %s, userpage %s",real,user)
+                logs.report("pages", "flushing realpage %s, userpage %s",real,user)
             end
         else
-            logs.report("pages", "flushing page, realpage %s",real)
+            logs.report("pages", "flushing realpage %s",real)
         end
     else
         logs.report("pages", "flushing page")
@@ -4818,7 +4844,7 @@ end
 local function splitpathexpr(str, t, validate)
     -- no need for further optimization as it is only called a
     -- few times, we can use lpeg for the sub
-    if trace_expansion then
+    if trace_expansions then
         logs.report("fileio","expanding variable '%s'",str)
     end
     t = t or { }
@@ -5305,10 +5331,13 @@ local function split_kpse_path(str) -- beware, this can be either a path or a {s
     local found = cache[str]
     if not found then
         str = gsub(str,"\\","/")
-        if find(str,";") then
-            found = checkedsplit(str,";")
-        else
-            found = checkedsplit(str,io.pathseparator)
+        local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator)
+        found = { }
+        for i=1,#split do
+            local s = split[i]
+            if not find(s,"^{*unset}*") then
+                found[#found+1] = s
+            end
         end
         if trace_expansions then
             logs.report("fileio","splitting path specification '%s'",str)
diff --git a/scripts/context/lua/mtx-watch.lua b/scripts/context/lua/mtx-watch.lua
index 15e0f81e9..d897d6beb 100644
--- a/scripts/context/lua/mtx-watch.lua
+++ b/scripts/context/lua/mtx-watch.lua
@@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['mtx-watch'] = {
 scripts       = scripts       or { }
 scripts.watch = scripts.watch or { }
 
-local format, concat, time, difftime = string.format, table.concat, os.difftime
+local format, concat, difftime, time = string.format, table.concat, os.difftime, os.time
 local pairs, ipairs, next, type = pairs, ipairs, next, type
 
 function scripts.watch.save_exa_modes(joblog,ctmname)
@@ -74,7 +74,10 @@ end
 local clock = os.gettimeofday or os.time -- we cannot trust os.clock on linux
 
 function scripts.watch.watch()
-    local delay   = environment.argument("delay")   or 5
+    local delay   = tonumber(environment.argument("delay") or 5) or 5
+    if delay == 0 then
+        delay = .25
+    end
     local logpath = environment.argument("logpath") or ""
     local pipe    = environment.argument("pipe")    or false
     local watcher = "mtxwatch.run"
@@ -156,6 +159,18 @@ function scripts.watch.watch()
             end
         end
         local n, start = 0, time()
+--~         local function wait()
+--~             io.flush()
+--~             if not done then
+--~                 n = n + 1
+--~                 if n >= 10 then
+--~                     logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000))
+--~                     n = 0
+--~                 end
+--~                 os.sleep(delay)
+--~             end
+--~         end
+        local wtime = 0
         local function wait()
             io.flush()
             if not done then
@@ -164,40 +179,58 @@ function scripts.watch.watch()
                     logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000))
                     n = 0
                 end
-                os.sleep(delay)
+                local ttime = 0
+                while ttime <= delay do
+                    local wt = lfs.attributes(watcher,"mtime")
+                    if wt and wt ~= wtime then
+                        -- fast signal that there is a request
+                        wtime = wt
+                        break
+                    end
+                    ttime = ttime + 0.2
+                    os.sleep(0.2)
+                end
             end
         end
-local function wait()
-    io.flush()
-    local wtime, ttime = 0, 0
-    if not done then
-        n = n + 1
-        if n >= 10 then
-            logs.report("watch", format("run time: %i seconds, memory usage: %0.3g MB", difftime(time(),start), (status.luastate_bytes/1024)/1000))
-            n = 0
-        end
-        while true do
-            local wt = lfs.attributes(watcher,"mtime")
-            if wt ~= wtime then
-                -- fast signal that there is a request
-                wtime = wt
-                break
-            else
-                ttime = ttime + 0.2
-                if ttime >= delay then
-                    break
+        local cleanupdelay, cleanup = environment.argument("cleanup"), false
+        if cleanupdelay then
+            local lasttime = time()
+            cleanup = function()
+                local currenttime = time()
+                local delta = difftime(currenttime,lasttime)
+                if delta > cleanupdelay then
+                    lasttime = currenttime
+                    for _, path in ipairs(environment.files) do
+                        if string.find(path,"%.") then
+                            -- safeguard, we want a fully qualified path
+                        else
+                            local files = dir.glob(file.join(path,"*"))
+                            for _, name in ipairs(files) do
+                                local filetime = lfs.attributes(name,"modification")
+                                local delta = difftime(currenttime,filetime)
+                                if delta > cleanupdelay then
+                                 -- logs.report("watch",format("cleaning up '%s'",name))
+                                    os.remove(name)
+                                end
+                            end
+                        end
+                    end
                 end
             end
-            os.sleep(0.2)
+        else
+            cleanup = function()
+                -- nothing
+            end
         end
-    end
-end
         while true do
             if false then
+--~             if true then
                 process()
+                cleanup()
                 wait()
             else
                 pcall(process)
+                pcall(cleanup)
                 pcall(wait)
             end
         end
@@ -280,14 +313,40 @@ function scripts.watch.show_logs(path) -- removes duplicates
     end
 end
 
+function scripts.watch.cleanup_stale_files() -- removes duplicates
+    local path  = environment.files[1]
+    local delay = tonumber(environment.argument("cleanup"))
+    local force = environment.argument("force")
+    if not path or path == "." then
+        logs.report("watch","provide qualified path")
+    elseif not delay then
+        logs.report("watch","missing --cleanup=delay")
+    else
+        logs.report("watch","dryrun, use --force for real cleanup")
+        local files = dir.glob(file.join(path,"*"))
+        local rtime = time()
+        for _, name in ipairs(files) do
+            local mtime = lfs.attributes(name,"modification")
+            local delta = difftime(rtime,mtime)
+            if delta > delay then
+                logs.report("watch",format("cleaning up '%s'",name))
+                if force then
+                    os.remove(name)
+                end
+            end
+        end
+    end
+end
+
 logs.extendbanner("ConTeXt Request Watchdog 1.00",true)
 
 messages.help = [[
 --logpath             optional path for log files
---watch               watch given path
+--watch               watch given path [--delay]
 --pipe                use pipe instead of execute
 --delay               delay between sweeps
 --collect             condense log files
+--cleanup=delay       remove files in given path [--force]
 --showlog             show log data
 ]]
 
@@ -295,6 +354,8 @@ if environment.argument("watch") then
     scripts.watch.watch()
 elseif environment.argument("collect") then
     scripts.watch.save_logs(scripts.watch.collect_logs())
+elseif environment.argument("cleanup") then
+    scripts.watch.save_logs(scripts.watch.cleanup_stale_files())
 elseif environment.argument("showlog") then
     scripts.watch.show_logs()
 else
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index c5b0acd04..03b454993 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -331,6 +331,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+lpeg = require("lpeg")
+
 local P, R, S, Ct, C, Cs, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
@@ -1392,7 +1394,7 @@ function io.savedata(filename,data,joiner)
         elseif type(data) == "function" then
             data(f)
         else
-            f:write(data)
+            f:write(data or "")
         end
         f:close()
         return true
@@ -1711,7 +1713,13 @@ local find, format = string.find, string.format
 local random, ceil = math.random, math.ceil
 
 function os.resultof(command)
-    return io.popen(command,"r"):read("*all")
+    local handle = io.popen(command,"r")
+    if not handle then
+    --  print("unknown command '".. command .. "' in os.resultof")
+        return ""
+    else
+        return handle:read("*all") or ""
+    end
 end
 
 if not os.exec  then os.exec  = os.execute end
@@ -1771,14 +1779,6 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
-os.arch = os.arch or function()
-    local a = os.resultof("uname -m") or "linux"
-    os.arch = function()
-        return a
-    end
-    return a
-end
-
 -- no need for function anymore as we have more clever code and helpers now
 
 os.platform  = os.name
@@ -1794,7 +1794,13 @@ if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" th
     end
     os.libsuffix = 'dll'
 else
-    local architecture = os.arch()
+    local architecture = os.getenv("HOSTTYPE") or ""
+    if architecture == "" then
+        architecture = os.resultof("uname -m") or ""
+    end
+    if architecture == "" then
+        local architecture = os.resultof("echo $HOSTTYPE")
+    end
     if name == "linux" then
         if find(architecture,"x86_64") then
             os.platform = "linux-64"
@@ -1804,7 +1810,6 @@ else
             os.platform = "linux"
         end
     elseif name == "macosx" then
-        local architecture = os.resultof("echo $HOSTTYPE")
         if find(architecture,"i386") then
             os.platform = "osx-intel"
         elseif find(architecture,"x86_64") then
@@ -3811,13 +3816,15 @@ local text_parsed      = Cs(((1-open-ampersand)^1 + entity)^1)
 local somespace        = space^1
 local optionalspace    = space^0
 
-local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value
+----- value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value
+local value            = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value
 
 local endofattributes  = slash * close + close -- recovery of flacky html
 local whatever         = space * name * optionalspace * equal
 local wrongvalue       = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error
------ local wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
-local wrongvalue       = C(P(1-space-endofattributes)^1) / attribute_value_error
+----- wrongvalue       = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
+----- wrongvalue       = C(P(1-space-endofattributes)^1) / attribute_value_error
+local wrongvalue       = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error
 
 local attributevalue   = value + wrongvalue
 
@@ -5530,7 +5537,7 @@ local type, next, tonumber, tostring, setmetatable, loadstring = type, next, ton
 local format, gsub = string.format, string.gsub
 
 --[[ldx--
-<p>The following helper functions best belong to the <t>lmxl-ini</t>
+<p>The following helper functions best belong to the <t>lxml-ini</t>
 module. Some are here because we need then in the <t>mk</t>
 document and other manuals, others came up when playing with
 this module. Since this module is also used in <l n='mtxrun'/> we've
@@ -6982,12 +6989,12 @@ function logs.tex.stop_page_number()
     if real > 0 then
         if user > 0 then
             if sub > 0 then
-                logs.report("pages", "flushing page, realpage %s, userpage %s, subpage %s",real,user,sub)
+                logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub)
             else
-                logs.report("pages", "flushing page, realpage %s, userpage %s",real,user)
+                logs.report("pages", "flushing realpage %s, userpage %s",real,user)
             end
         else
-            logs.report("pages", "flushing page, realpage %s",real)
+            logs.report("pages", "flushing realpage %s",real)
         end
     else
         logs.report("pages", "flushing page")
@@ -7562,7 +7569,7 @@ end
 local function splitpathexpr(str, t, validate)
     -- no need for further optimization as it is only called a
     -- few times, we can use lpeg for the sub
-    if trace_expansion then
+    if trace_expansions then
         logs.report("fileio","expanding variable '%s'",str)
     end
     t = t or { }
@@ -8049,10 +8056,13 @@ local function split_kpse_path(str) -- beware, this can be either a path or a {s
     local found = cache[str]
     if not found then
         str = gsub(str,"\\","/")
-        if find(str,";") then
-            found = checkedsplit(str,";")
-        else
-            found = checkedsplit(str,io.pathseparator)
+        local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator)
+        found = { }
+        for i=1,#split do
+            local s = split[i]
+            if not find(s,"^{*unset}*") then
+                found[#found+1] = s
+            end
         end
         if trace_expansions then
             logs.report("fileio","splitting path specification '%s'",str)
diff --git a/scripts/context/stubs/mswin/luatools.lua b/scripts/context/stubs/mswin/luatools.lua
index 06e2a1bfc..ae5122509 100644
--- a/scripts/context/stubs/mswin/luatools.lua
+++ b/scripts/context/stubs/mswin/luatools.lua
@@ -322,6 +322,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+lpeg = require("lpeg")
+
 local P, R, S, Ct, C, Cs, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
@@ -1383,7 +1385,7 @@ function io.savedata(filename,data,joiner)
         elseif type(data) == "function" then
             data(f)
         else
-            f:write(data)
+            f:write(data or "")
         end
         f:close()
         return true
@@ -1702,7 +1704,13 @@ local find, format = string.find, string.format
 local random, ceil = math.random, math.ceil
 
 function os.resultof(command)
-    return io.popen(command,"r"):read("*all")
+    local handle = io.popen(command,"r")
+    if not handle then
+    --  print("unknown command '".. command .. "' in os.resultof")
+        return ""
+    else
+        return handle:read("*all") or ""
+    end
 end
 
 if not os.exec  then os.exec  = os.execute end
@@ -1762,14 +1770,6 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
-os.arch = os.arch or function()
-    local a = os.resultof("uname -m") or "linux"
-    os.arch = function()
-        return a
-    end
-    return a
-end
-
 -- no need for function anymore as we have more clever code and helpers now
 
 os.platform  = os.name
@@ -1785,7 +1785,13 @@ if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" th
     end
     os.libsuffix = 'dll'
 else
-    local architecture = os.arch()
+    local architecture = os.getenv("HOSTTYPE") or ""
+    if architecture == "" then
+        architecture = os.resultof("uname -m") or ""
+    end
+    if architecture == "" then
+        local architecture = os.resultof("echo $HOSTTYPE")
+    end
     if name == "linux" then
         if find(architecture,"x86_64") then
             os.platform = "linux-64"
@@ -1795,7 +1801,6 @@ else
             os.platform = "linux"
         end
     elseif name == "macosx" then
-        local architecture = os.resultof("echo $HOSTTYPE")
         if find(architecture,"i386") then
             os.platform = "osx-intel"
         elseif find(architecture,"x86_64") then
@@ -2720,13 +2725,34 @@ if not modules then modules = { } end modules ['l-unicode'] = {
     license   = "see context related readme files"
 }
 
+if not unicode then
+
+    unicode = { utf8 = { } }
+
+    local floor, char = math.floor, string.char
+
+    function unicode.utf8.utfchar(n)
+        if n < 0x80 then
+            return char(n)
+        elseif n < 0x800 then
+            return char(0xC0 + floor(n/0x40))  .. char(0x80 + (n % 0x40))
+        elseif n < 0x10000 then
+            return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+        elseif n < 0x40000 then
+            return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+        else -- wrong:
+          -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+            return "?"
+        end
+    end
+
+end
+
 utf = utf or unicode.utf8
 
 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
 local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs
 
-unicode = unicode or { }
-
 -- 0  EF BB BF      UTF-8
 -- 1  FF FE         UTF-16-little-endian
 -- 2  FE FF         UTF-16-big-endian
@@ -4238,12 +4264,12 @@ function logs.tex.stop_page_number()
     if real > 0 then
         if user > 0 then
             if sub > 0 then
-                logs.report("pages", "flushing page, realpage %s, userpage %s, subpage %s",real,user,sub)
+                logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub)
             else
-                logs.report("pages", "flushing page, realpage %s, userpage %s",real,user)
+                logs.report("pages", "flushing realpage %s, userpage %s",real,user)
             end
         else
-            logs.report("pages", "flushing page, realpage %s",real)
+            logs.report("pages", "flushing realpage %s",real)
         end
     else
         logs.report("pages", "flushing page")
@@ -4818,7 +4844,7 @@ end
 local function splitpathexpr(str, t, validate)
     -- no need for further optimization as it is only called a
     -- few times, we can use lpeg for the sub
-    if trace_expansion then
+    if trace_expansions then
         logs.report("fileio","expanding variable '%s'",str)
     end
     t = t or { }
@@ -5305,10 +5331,13 @@ local function split_kpse_path(str) -- beware, this can be either a path or a {s
     local found = cache[str]
     if not found then
         str = gsub(str,"\\","/")
-        if find(str,";") then
-            found = checkedsplit(str,";")
-        else
-            found = checkedsplit(str,io.pathseparator)
+        local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator)
+        found = { }
+        for i=1,#split do
+            local s = split[i]
+            if not find(s,"^{*unset}*") then
+                found[#found+1] = s
+            end
         end
         if trace_expansions then
             logs.report("fileio","splitting path specification '%s'",str)
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index c5b0acd04..03b454993 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -331,6 +331,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+lpeg = require("lpeg")
+
 local P, R, S, Ct, C, Cs, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
@@ -1392,7 +1394,7 @@ function io.savedata(filename,data,joiner)
         elseif type(data) == "function" then
             data(f)
         else
-            f:write(data)
+            f:write(data or "")
         end
         f:close()
         return true
@@ -1711,7 +1713,13 @@ local find, format = string.find, string.format
 local random, ceil = math.random, math.ceil
 
 function os.resultof(command)
-    return io.popen(command,"r"):read("*all")
+    local handle = io.popen(command,"r")
+    if not handle then
+    --  print("unknown command '".. command .. "' in os.resultof")
+        return ""
+    else
+        return handle:read("*all") or ""
+    end
 end
 
 if not os.exec  then os.exec  = os.execute end
@@ -1771,14 +1779,6 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
-os.arch = os.arch or function()
-    local a = os.resultof("uname -m") or "linux"
-    os.arch = function()
-        return a
-    end
-    return a
-end
-
 -- no need for function anymore as we have more clever code and helpers now
 
 os.platform  = os.name
@@ -1794,7 +1794,13 @@ if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" th
     end
     os.libsuffix = 'dll'
 else
-    local architecture = os.arch()
+    local architecture = os.getenv("HOSTTYPE") or ""
+    if architecture == "" then
+        architecture = os.resultof("uname -m") or ""
+    end
+    if architecture == "" then
+        local architecture = os.resultof("echo $HOSTTYPE")
+    end
     if name == "linux" then
         if find(architecture,"x86_64") then
             os.platform = "linux-64"
@@ -1804,7 +1810,6 @@ else
             os.platform = "linux"
         end
     elseif name == "macosx" then
-        local architecture = os.resultof("echo $HOSTTYPE")
         if find(architecture,"i386") then
             os.platform = "osx-intel"
         elseif find(architecture,"x86_64") then
@@ -3811,13 +3816,15 @@ local text_parsed      = Cs(((1-open-ampersand)^1 + entity)^1)
 local somespace        = space^1
 local optionalspace    = space^0
 
-local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value
+----- value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value
+local value            = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value
 
 local endofattributes  = slash * close + close -- recovery of flacky html
 local whatever         = space * name * optionalspace * equal
 local wrongvalue       = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error
------ local wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
-local wrongvalue       = C(P(1-space-endofattributes)^1) / attribute_value_error
+----- wrongvalue       = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
+----- wrongvalue       = C(P(1-space-endofattributes)^1) / attribute_value_error
+local wrongvalue       = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error
 
 local attributevalue   = value + wrongvalue
 
@@ -5530,7 +5537,7 @@ local type, next, tonumber, tostring, setmetatable, loadstring = type, next, ton
 local format, gsub = string.format, string.gsub
 
 --[[ldx--
-<p>The following helper functions best belong to the <t>lmxl-ini</t>
+<p>The following helper functions best belong to the <t>lxml-ini</t>
 module. Some are here because we need then in the <t>mk</t>
 document and other manuals, others came up when playing with
 this module. Since this module is also used in <l n='mtxrun'/> we've
@@ -6982,12 +6989,12 @@ function logs.tex.stop_page_number()
     if real > 0 then
         if user > 0 then
             if sub > 0 then
-                logs.report("pages", "flushing page, realpage %s, userpage %s, subpage %s",real,user,sub)
+                logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub)
             else
-                logs.report("pages", "flushing page, realpage %s, userpage %s",real,user)
+                logs.report("pages", "flushing realpage %s, userpage %s",real,user)
             end
         else
-            logs.report("pages", "flushing page, realpage %s",real)
+            logs.report("pages", "flushing realpage %s",real)
         end
     else
         logs.report("pages", "flushing page")
@@ -7562,7 +7569,7 @@ end
 local function splitpathexpr(str, t, validate)
     -- no need for further optimization as it is only called a
     -- few times, we can use lpeg for the sub
-    if trace_expansion then
+    if trace_expansions then
         logs.report("fileio","expanding variable '%s'",str)
     end
     t = t or { }
@@ -8049,10 +8056,13 @@ local function split_kpse_path(str) -- beware, this can be either a path or a {s
     local found = cache[str]
     if not found then
         str = gsub(str,"\\","/")
-        if find(str,";") then
-            found = checkedsplit(str,";")
-        else
-            found = checkedsplit(str,io.pathseparator)
+        local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator)
+        found = { }
+        for i=1,#split do
+            local s = split[i]
+            if not find(s,"^{*unset}*") then
+                found[#found+1] = s
+            end
         end
         if trace_expansions then
             logs.report("fileio","splitting path specification '%s'",str)
diff --git a/scripts/context/stubs/unix/luatools b/scripts/context/stubs/unix/luatools
index 06e2a1bfc..ae5122509 100755
--- a/scripts/context/stubs/unix/luatools
+++ b/scripts/context/stubs/unix/luatools
@@ -322,6 +322,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+lpeg = require("lpeg")
+
 local P, R, S, Ct, C, Cs, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
@@ -1383,7 +1385,7 @@ function io.savedata(filename,data,joiner)
         elseif type(data) == "function" then
             data(f)
         else
-            f:write(data)
+            f:write(data or "")
         end
         f:close()
         return true
@@ -1702,7 +1704,13 @@ local find, format = string.find, string.format
 local random, ceil = math.random, math.ceil
 
 function os.resultof(command)
-    return io.popen(command,"r"):read("*all")
+    local handle = io.popen(command,"r")
+    if not handle then
+    --  print("unknown command '".. command .. "' in os.resultof")
+        return ""
+    else
+        return handle:read("*all") or ""
+    end
 end
 
 if not os.exec  then os.exec  = os.execute end
@@ -1762,14 +1770,6 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
-os.arch = os.arch or function()
-    local a = os.resultof("uname -m") or "linux"
-    os.arch = function()
-        return a
-    end
-    return a
-end
-
 -- no need for function anymore as we have more clever code and helpers now
 
 os.platform  = os.name
@@ -1785,7 +1785,13 @@ if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" th
     end
     os.libsuffix = 'dll'
 else
-    local architecture = os.arch()
+    local architecture = os.getenv("HOSTTYPE") or ""
+    if architecture == "" then
+        architecture = os.resultof("uname -m") or ""
+    end
+    if architecture == "" then
+        local architecture = os.resultof("echo $HOSTTYPE")
+    end
     if name == "linux" then
         if find(architecture,"x86_64") then
             os.platform = "linux-64"
@@ -1795,7 +1801,6 @@ else
             os.platform = "linux"
         end
     elseif name == "macosx" then
-        local architecture = os.resultof("echo $HOSTTYPE")
         if find(architecture,"i386") then
             os.platform = "osx-intel"
         elseif find(architecture,"x86_64") then
@@ -2720,13 +2725,34 @@ if not modules then modules = { } end modules ['l-unicode'] = {
     license   = "see context related readme files"
 }
 
+if not unicode then
+
+    unicode = { utf8 = { } }
+
+    local floor, char = math.floor, string.char
+
+    function unicode.utf8.utfchar(n)
+        if n < 0x80 then
+            return char(n)
+        elseif n < 0x800 then
+            return char(0xC0 + floor(n/0x40))  .. char(0x80 + (n % 0x40))
+        elseif n < 0x10000 then
+            return char(0xE0 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+        elseif n < 0x40000 then
+            return char(0xF0 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+        else -- wrong:
+          -- return char(0xF1 + floor(n/0x1000000)) .. char(0x80 + floor(n/0x40000)) .. char(0x80 + floor(n/0x1000)) .. char(0x80 + (floor(n/0x40) % 0x40)) .. char(0x80 + (n % 0x40))
+            return "?"
+        end
+    end
+
+end
+
 utf = utf or unicode.utf8
 
 local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
 local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs
 
-unicode = unicode or { }
-
 -- 0  EF BB BF      UTF-8
 -- 1  FF FE         UTF-16-little-endian
 -- 2  FE FF         UTF-16-big-endian
@@ -4238,12 +4264,12 @@ function logs.tex.stop_page_number()
     if real > 0 then
         if user > 0 then
             if sub > 0 then
-                logs.report("pages", "flushing page, realpage %s, userpage %s, subpage %s",real,user,sub)
+                logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub)
             else
-                logs.report("pages", "flushing page, realpage %s, userpage %s",real,user)
+                logs.report("pages", "flushing realpage %s, userpage %s",real,user)
             end
         else
-            logs.report("pages", "flushing page, realpage %s",real)
+            logs.report("pages", "flushing realpage %s",real)
         end
     else
         logs.report("pages", "flushing page")
@@ -4818,7 +4844,7 @@ end
 local function splitpathexpr(str, t, validate)
     -- no need for further optimization as it is only called a
     -- few times, we can use lpeg for the sub
-    if trace_expansion then
+    if trace_expansions then
         logs.report("fileio","expanding variable '%s'",str)
     end
     t = t or { }
@@ -5305,10 +5331,13 @@ local function split_kpse_path(str) -- beware, this can be either a path or a {s
     local found = cache[str]
     if not found then
         str = gsub(str,"\\","/")
-        if find(str,";") then
-            found = checkedsplit(str,";")
-        else
-            found = checkedsplit(str,io.pathseparator)
+        local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator)
+        found = { }
+        for i=1,#split do
+            local s = split[i]
+            if not find(s,"^{*unset}*") then
+                found[#found+1] = s
+            end
         end
         if trace_expansions then
             logs.report("fileio","splitting path specification '%s'",str)
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index c5b0acd04..03b454993 100755
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -331,6 +331,8 @@ if not modules then modules = { } end modules ['l-lpeg'] = {
     license   = "see context related readme files"
 }
 
+lpeg = require("lpeg")
+
 local P, R, S, Ct, C, Cs, Cc = lpeg.P, lpeg.R, lpeg.S, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc
 
 --~ l-lpeg.lua :
@@ -1392,7 +1394,7 @@ function io.savedata(filename,data,joiner)
         elseif type(data) == "function" then
             data(f)
         else
-            f:write(data)
+            f:write(data or "")
         end
         f:close()
         return true
@@ -1711,7 +1713,13 @@ local find, format = string.find, string.format
 local random, ceil = math.random, math.ceil
 
 function os.resultof(command)
-    return io.popen(command,"r"):read("*all")
+    local handle = io.popen(command,"r")
+    if not handle then
+    --  print("unknown command '".. command .. "' in os.resultof")
+        return ""
+    else
+        return handle:read("*all") or ""
+    end
 end
 
 if not os.exec  then os.exec  = os.execute end
@@ -1771,14 +1779,6 @@ end
 --~ print(os.date("%H:%M:%S",os.gettimeofday()))
 --~ print(os.date("%H:%M:%S",os.time()))
 
-os.arch = os.arch or function()
-    local a = os.resultof("uname -m") or "linux"
-    os.arch = function()
-        return a
-    end
-    return a
-end
-
 -- no need for function anymore as we have more clever code and helpers now
 
 os.platform  = os.name
@@ -1794,7 +1794,13 @@ if name == "windows" or name == "mswin" or name == "win32" or name == "msdos" th
     end
     os.libsuffix = 'dll'
 else
-    local architecture = os.arch()
+    local architecture = os.getenv("HOSTTYPE") or ""
+    if architecture == "" then
+        architecture = os.resultof("uname -m") or ""
+    end
+    if architecture == "" then
+        local architecture = os.resultof("echo $HOSTTYPE")
+    end
     if name == "linux" then
         if find(architecture,"x86_64") then
             os.platform = "linux-64"
@@ -1804,7 +1810,6 @@ else
             os.platform = "linux"
         end
     elseif name == "macosx" then
-        local architecture = os.resultof("echo $HOSTTYPE")
         if find(architecture,"i386") then
             os.platform = "osx-intel"
         elseif find(architecture,"x86_64") then
@@ -3811,13 +3816,15 @@ local text_parsed      = Cs(((1-open-ampersand)^1 + entity)^1)
 local somespace        = space^1
 local optionalspace    = space^0
 
-local value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value
+----- value            = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value
+local value            = (squote * Cs((entity + (1 - squote))^0) * squote) + (dquote * Cs((entity + (1 - dquote))^0) * dquote) -- ampersand and < also invalid in value
 
 local endofattributes  = slash * close + close -- recovery of flacky html
 local whatever         = space * name * optionalspace * equal
 local wrongvalue       = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error
------ local wrongvalue = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
-local wrongvalue       = C(P(1-space-endofattributes)^1) / attribute_value_error
+----- wrongvalue       = C(P(1-whatever-endofattributes)^1 + P(1-endofattributes)^1) / attribute_value_error
+----- wrongvalue       = C(P(1-space-endofattributes)^1) / attribute_value_error
+local wrongvalue       = Cs(P(entity + (1-space-endofattributes))^1) / attribute_value_error
 
 local attributevalue   = value + wrongvalue
 
@@ -5530,7 +5537,7 @@ local type, next, tonumber, tostring, setmetatable, loadstring = type, next, ton
 local format, gsub = string.format, string.gsub
 
 --[[ldx--
-<p>The following helper functions best belong to the <t>lmxl-ini</t>
+<p>The following helper functions best belong to the <t>lxml-ini</t>
 module. Some are here because we need then in the <t>mk</t>
 document and other manuals, others came up when playing with
 this module. Since this module is also used in <l n='mtxrun'/> we've
@@ -6982,12 +6989,12 @@ function logs.tex.stop_page_number()
     if real > 0 then
         if user > 0 then
             if sub > 0 then
-                logs.report("pages", "flushing page, realpage %s, userpage %s, subpage %s",real,user,sub)
+                logs.report("pages", "flushing realpage %s, userpage %s, subpage %s",real,user,sub)
             else
-                logs.report("pages", "flushing page, realpage %s, userpage %s",real,user)
+                logs.report("pages", "flushing realpage %s, userpage %s",real,user)
             end
         else
-            logs.report("pages", "flushing page, realpage %s",real)
+            logs.report("pages", "flushing realpage %s",real)
         end
     else
         logs.report("pages", "flushing page")
@@ -7562,7 +7569,7 @@ end
 local function splitpathexpr(str, t, validate)
     -- no need for further optimization as it is only called a
     -- few times, we can use lpeg for the sub
-    if trace_expansion then
+    if trace_expansions then
         logs.report("fileio","expanding variable '%s'",str)
     end
     t = t or { }
@@ -8049,10 +8056,13 @@ local function split_kpse_path(str) -- beware, this can be either a path or a {s
     local found = cache[str]
     if not found then
         str = gsub(str,"\\","/")
-        if find(str,";") then
-            found = checkedsplit(str,";")
-        else
-            found = checkedsplit(str,io.pathseparator)
+        local split = (find(str,";") and checkedsplit(str,";")) or checkedsplit(str,io.pathseparator)
+        found = { }
+        for i=1,#split do
+            local s = split[i]
+            if not find(s,"^{*unset}*") then
+                found[#found+1] = s
+            end
         end
         if trace_expansions then
             logs.report("fileio","splitting path specification '%s'",str)