2021-08-07 22:51:00

16 files changed, 728 insertions, 9 deletions

diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii
index a901e296e..86a342700 100644
--- a/tex/context/base/mkii/cont-new.mkii
+++ b/tex/context/base/mkii/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2021.08.06 01:19}
+\newcontextversion{2021.08.07 22:49}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii
index 99705f3bd..0fe42e4a1 100644
--- a/tex/context/base/mkii/context.mkii
+++ b/tex/context/base/mkii/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2021.08.06 01:19}
+\edef\contextversion{2021.08.07 22:49}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index 9863f8484..b3b71009e 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2021.08.06 01:19}
+\newcontextversion{2021.08.07 22:49}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index e984d930d..fb5f34e21 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -45,7 +45,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2021.08.06 01:19}
+\edef\contextversion{2021.08.07 22:49}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/mtx-context-hashed.tex b/tex/context/base/mkiv/mtx-context-hashed.tex
new file mode 100644
index 000000000..88de10f3f
--- /dev/null
+++ b/tex/context/base/mkiv/mtx-context-hashed.tex
@@ -0,0 +1,136 @@
+%D \module
+%D   [       file=mtx-context-hashed,
+%D        version=2009.03.21,
+%D          title=\CONTEXT\ Extra Trickry,
+%D       subtitle=Combine Files,
+%D         author=Hans Hagen,
+%D           date=\currentdate,
+%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
+%C
+%C This module is part of the \CONTEXT\ macro||package and is
+%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
+%C details.
+
+% begin help
+%
+% usage: context --extra=hashed [options] list-of-files
+%
+% --database    : database to create or extend
+% --pattern     : file pattern
+% --patterns    : pattern file
+% --compression : apply compression
+%
+% end help
+
+% \startluacode
+%     if not resolvers.finders.helpers.validhashed("hasheddata") then
+%         resolvers.finders.helpers.createhashed {
+%             database = "hasheddata",
+%             pattern  = "m4all/**.jpg$",
+%             compress = false,
+%         }
+%         resolvers.finders.helpers.createhashed {
+%             database = "hasheddata",
+%             pattern  = "m4all/**.png$",
+%             compress = false,
+%         }
+%         resolvers.finders.helpers.createhashed {
+%             database = "hasheddata",
+%             pattern  = "m4all/**.xml$",
+%             compress = true,
+%         }
+%     end
+% \stopluacode
+
+% \startluacode
+%     if not resolvers.finders.helpers.validhashed("hasheddata") then
+%         resolvers.finders.helpers.createhashed {
+%             database = "hasheddata",
+%             patterns = {
+%                 { pattern  = "m4all/**.jpg$", compress = false },
+%                 { pattern  = "m4all/**.png$", compress = false },
+%                 { pattern  = "m4all/**.xml$", compress = true  },
+%             },
+%         }
+%     end
+% \stopluacode
+
+% context --extra=hashed --database=hasheddata --pattern=m4all/**.jpg
+% context --extra=hashed --database=hasheddata --pattern=m4all/**.png
+% context --extra=hashed --database=hasheddata --pattern=m4all/**.xml --compress
+% context --extra=hashed --database=hasheddata --pattern=m4all/**.svg --compress
+
+% -- m4all.lua:
+%
+% return {
+%     { pattern  = "m4all/**.jpg$", compress = false },
+%     { pattern  = "m4all/**.png$", compress = false },
+%     { pattern  = "m4all/**.svg$", compress = true  },
+%     { pattern  = "m4all/**.xml$", compress = true  },
+% }
+%
+% context --extra=hashed --database=hasheddata --patterns=m4all.lua
+
+% \registerhashedfiles[hasheddata]
+% \registerfilescheme[hashed]
+%
+% \externalfigure[hashed:///m4all/books/chapters/h3/h3-if1/images/highres/casino.jpg]      [height=1cm]
+% \externalfigure[hashed:///m4all/books/chapters/ha/ha-c4/images/highres/ha-c44-ex2-s1.png][height=1cm]
+% \externalfigure[m4all/books/chapters/h3/h3-if1/images/highres/casino.jpg]                [height=1cm]
+% \externalfigure[m4all/books/chapters/ha/ha-c4/images/highres/ha-c44-ex2-s1.png]          [height=1cm]
+
+\input mtx-context-common.tex
+
+\noheaderandfooterlines \setupbodyfont[tt]
+
+\starttext
+
+\startluacode
+
+    if CONTEXTLMTXMODE > 0 then
+
+        local database = document.arguments.database
+
+        if database then
+            local metadata = false
+            if tex.systemmodes["first"] then
+                metadata = resolvers.finders.helpers.createhashed {
+                    database = database,
+                    pattern  = document.arguments.pattern,
+                    compress = document.arguments.compress,
+                    patterns = document.arguments.patterns and table.load(document.arguments.patterns),
+                }
+            else
+                metadata = resolvers.finders.helpers.validhashed(database)
+                if metadata then
+                    metadata = metadata.metadata
+                end
+            end
+            if metadata then
+                local function show(what)
+                    context.NC() context(what)
+                    context.EQ() context(metadata[what])
+                    context.NC() context.NR()
+                end
+                context.starttitle { title = "database: " .. database }
+                    context.starttabulate { "||r|" }
+                        show("nofnames")
+                        show("nofpaths")
+                        show("nofblobs")
+                        show("nofcompressed")
+                    context.stoptabulate()
+                context.stoptitle()
+            else
+                context("something went wrong, invalid database")
+            end
+        else
+            context("something went wrong, no database specified")
+        end
+    else
+        context("you need lmtx")
+    end
+
+\stopluacode
+
+\stoptext
+
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index 912ec565f..654c04404 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index 6c3ca2ec2..de0ad44a4 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
diff --git a/tex/context/base/mkxl/cont-new.mkxl b/tex/context/base/mkxl/cont-new.mkxl
index d7196efb4..c9a545148 100644
--- a/tex/context/base/mkxl/cont-new.mkxl
+++ b/tex/context/base/mkxl/cont-new.mkxl
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2021.08.06 01:19}
+\newcontextversion{2021.08.07 22:49}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkxl/context.mkxl b/tex/context/base/mkxl/context.mkxl
index 75388d7b3..8c2bcc2aa 100644
--- a/tex/context/base/mkxl/context.mkxl
+++ b/tex/context/base/mkxl/context.mkxl
@@ -29,7 +29,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \immutable\edef\contextformat {\jobname}
-\immutable\edef\contextversion{2021.08.06 01:19}
+\immutable\edef\contextversion{2021.08.07 22:49}
 
 %overloadmode 1 % check frozen / warning
 %overloadmode 2 % check frozen / error
diff --git a/tex/context/base/mkxl/data-fil.lmt b/tex/context/base/mkxl/data-fil.lmt
new file mode 100644
index 000000000..bbcc954b2
--- /dev/null
+++ b/tex/context/base/mkxl/data-fil.lmt
@@ -0,0 +1,144 @@
+if not modules then modules = { } end modules ['data-fil'] = {
+    version   = 1.001,
+    comment   = "companion to luat-lib.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local ioopen = io.open
+local isdir = lfs.isdir
+
+local trace_locating = false  trackers.register("resolvers.locating", function(v) trace_locating = v end)
+
+local report_files = logs.reporter("resolvers","files")
+
+local resolvers        = resolvers
+local resolveprefix    = resolvers.resolve
+local findfile         = resolvers.findfile
+local scanfiles        = resolvers.scanfiles
+local registerfilehash = resolvers.registerfilehash
+local appendhash       = resolvers.appendhash
+
+local loadcachecontent = caches.loadcontent
+
+local checkgarbage     = utilities.garbagecollector and utilities.garbagecollector.check
+
+function resolvers.locators.file(specification)
+    local filename = specification.filename
+    local realname = resolveprefix(filename) -- no shortcut
+    if realname and realname ~= '' and isdir(realname) then
+        if trace_locating then
+            report_files("file locator %a found as %a",filename,realname)
+        end
+        appendhash('file',filename,true) -- cache
+    elseif trace_locating then
+        report_files("file locator %a not found",filename)
+    end
+end
+
+function resolvers.hashers.file(specification)
+    local pathname = specification.filename
+    local content  = loadcachecontent(pathname,'files')
+    registerfilehash(pathname,content,content==nil)
+end
+
+function resolvers.generators.file(specification)
+    local pathname = specification.filename
+    local content  = scanfiles(pathname,false,true) -- scan once
+    registerfilehash(pathname,content,true)
+end
+
+resolvers.concatinators.file = file.join
+
+local finders  = resolvers.finders
+local notfound = finders.notfound
+
+function finders.file(specification,filetype)
+    local filename  = specification.filename
+    local foundname = findfile(filename,filetype)
+    if foundname and foundname ~= "" then
+        if trace_locating then
+            report_files("file finder: %a found",filename)
+        end
+        return foundname
+    else
+        if trace_locating then
+            report_files("file finder: %a not found",filename)
+        end
+        return notfound()
+    end
+end
+
+-- The default textopener will be overloaded later on.
+
+local openers    = resolvers.openers
+local notfound   = openers.notfound
+local overloaded = false
+
+local function textopener(tag,filename,f)
+    return {
+        reader = function() return f:read () end,
+        close  = function() return f:close() end,
+    }
+end
+
+function openers.helpers.textopener(...)
+    return textopener(...)
+end
+
+function openers.helpers.settextopener(opener)
+    if overloaded then
+        report_files("file opener: %s overloaded","already")
+    else
+        if trace_locating then
+            report_files("file opener: %s overloaded","once")
+        end
+        overloaded = true
+        textopener = opener
+    end
+end
+
+function openers.file(specification,filetype)
+    local filename = specification.filename
+    if filename and filename ~= "" then
+        local f = ioopen(filename,"r")
+        if f then
+            if trace_locating then
+                report_files("file opener: %a opened",filename)
+            end
+            return textopener("file",filename,f)
+        end
+    end
+    if trace_locating then
+        report_files("file opener: %a not found",filename)
+    end
+    return notfound()
+end
+
+local loaders  = resolvers.loaders
+local notfound = loaders.notfound
+
+function loaders.file(specification,filetype)
+    local filename = specification.filename
+    if filename and filename ~= "" then
+        local f = ioopen(filename,"rb")
+        if f then
+            if trace_locating then
+                report_files("file loader: %a loaded",filename)
+            end
+            local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
+            if checkgarbage then
+                checkgarbage(#s)
+            end
+            f:close()
+            if s then
+                return true, s, #s
+            end
+        end
+    end
+    if trace_locating then
+        report_files("file loader: %a not found",filename)
+    end
+    return notfound()
+end
diff --git a/tex/context/base/mkxl/data-hsh.lmt b/tex/context/base/mkxl/data-hsh.lmt
new file mode 100644
index 000000000..0a2d94f81
--- /dev/null
+++ b/tex/context/base/mkxl/data-hsh.lmt
@@ -0,0 +1,382 @@
+-- only lmt because the backend code doesn't deal with it and it makes
+-- no sense to waste time on that for mkiv
+
+if not modules then modules = { } end modules ['data-hsh'] = {
+    version   = 0.002,
+    comment   = "companion to luat-lib.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- todo: options
+--
+-- lowercase
+-- cleanupnames (normalize)
+-- use database from project tree
+
+local type = type
+local gsub = string.gsub
+local addsuffix, basename, pathpart, filesuffix, filesize = file.addsuffix, file.basename, file.pathpart, file.suffix, file.size
+local loadtable, savetable = table.load, table.save
+local loaddata, savedata, open = io.loaddata, io.savedata, io.open
+
+local trace_hashed  = false
+local report_hashed = logs.reporter("resolvers","hashed")
+
+trackers.register("resolvers.locating", function(v) trace_hashed = v end)
+trackers.register("resolvers.hashed",   function(v) trace_hashed = v end)
+
+-- we can have a virtual file: open at the position, make sure read and seek don't
+-- go beyond the boundaries
+
+local resolvers = resolvers
+local finders   = resolvers.finders
+local openers   = resolvers.openers
+local loaders   = resolvers.loaders
+
+local ordered = { }
+local hashed  = { }
+local version = 0.002
+
+-- local lowercase = characters.lower
+
+local function showstatus(database,metadata)
+    report_hashed("database %a, %i paths, %i names, %i unique blobs, %i compressed blobs",
+        database, metadata.nofpaths, metadata.nofnames, metadata.nofblobs, metadata.nofcompressed
+    )
+end
+
+local function validhashed(database)
+    local found = hashed[database]
+    if found then
+        return found
+    else
+        local metaname = addsuffix(database,"lua")
+        local dataname = addsuffix(database,"dat")
+        local metadata = loadtable(metaname)
+        if type(metadata) ~= "table" then
+            report_hashed("invalid database %a",metaname)
+        elseif metadata.version ~= version then
+            report_hashed("version mismatch in database %a",metaname)
+        elseif not lfs.isfile(dataname) then
+            report_hashed("missing data data file for %a",metaname)
+        else
+            return {
+                database = database,
+                metadata = metadata,
+                dataname = dataname,
+            }
+        end
+    end
+end
+
+local function registerhashed(database)
+    if not hashed[database] then
+        local valid = validhashed(database)
+        if valid then
+            ordered[#ordered + 1] = valid
+            hashed[database] = ordered[#ordered]
+            showstatus(database,valid.metadata)
+        end
+    end
+end
+
+local registerfilescheme  do
+
+    local findfile = finders.file
+
+    local list = { }
+    local done = { }
+    local hash = { }
+
+    registerfilescheme = function(name)
+        if not done[name] then
+            list[#list+1] = name
+            done[name]    = true
+        end
+    end
+
+    -- why does the finder not remember ?
+
+    function finders.file(specification,filetype)
+        if type(specification) == "table" then
+            local original = specification.original
+         -- print(original)
+            if original then
+                local found = hash[original]
+                if found == nil then
+                    for i=1,#list do
+                        local scheme = list[i]
+                        local found  = finders[scheme](specification,filetype)
+                        if found then
+                            hash[original] = found
+                            if trace_hashed then
+                                report_hashed("found by auto scheme %s: %s",scheme,found)
+                            end
+                            return found
+                        end
+                    end
+                    local found = findfile(specification,filetype)
+                    if found then
+                        hash[original] = found
+                        if trace_hashed then
+                            report_hashed("found by normal file scheme: %s",found)
+                        end
+                        return found
+                    end
+                    hash[original] = false
+                elseif found then
+                    return found
+                end
+                return false
+            else
+                -- something is wrong here, maybe we should trace it (scheme can be "unknown")
+            end
+        end
+        -- again, something is wrong
+        return findfile(specification,filetype)
+    end
+
+end
+
+finders.helpers.validhashed        = validhashed
+finders.helpers.registerhashed     = registerhashed
+finders.helpers.registerfilescheme = registerfilescheme
+
+local function locate(found,path,name)
+    local files  = found.metadata.files
+    local hashes = found.metadata.hashes
+    local fp = files[path]
+    local hash = fp and fp[name]
+    if hash and hashes[hash] then
+        return hash
+    end
+end
+
+local function locatehash(filename,database)
+    if filename then
+        local name = basename(filename)
+        local path = pathpart(filename)
+        local hash = false
+        if database then
+            local found = hashed[database]
+            if found then
+                hash = locate(found,path,name), database, path, name
+            end
+        else
+            for i=1,#ordered do
+                local found = ordered[i]
+                hash = locate(found,path,name)
+                if hash then
+                    database = found.database
+                    break
+                end
+            end
+        end
+        if hash then
+            return {
+                hash = hash,
+                name = name,
+                path = path,
+                base = database,
+            }
+        end
+    end
+end
+
+-- no caching yet, we don't always want the file and it's fast enough
+
+local function locateblob(filename,database)
+    local found = locatehash(filename,database)
+    if found then
+        local database = found.base
+        local data     = hashed[database]
+        if data then
+            local metadata = data.metadata
+            local dataname = data.dataname
+            local hashes   = metadata.hashes
+            local blobdata = hashes[found.hash]
+            if blobdata and dataname then
+                local position = blobdata.position
+                local f = open(dataname,"rb")
+                if f then
+                    f:seek("set",position)
+                    local blob = f:read(blobdata.datasize)
+                    if blobdata.compress == "zip" then
+                        blob = zlib.decompresssize(blob,blobdata.filesize)
+                    end
+                    return blob
+                end
+            end
+        end
+    end
+end
+
+local finders  = resolvers.finders
+local notfound = finders.notfound
+
+function finders.hashed(specification)
+    local original = specification.original
+    local fullpath = specification.path
+    if fullpath then
+        local found = locatehash(fullpath)
+        if found then
+            if trace_hashed then
+                report_hashed("finder: file %a found",original)
+            end
+            return original
+        end
+    end
+    if trace_hashed then
+        report_hashed("finder: unknown file %a",original)
+    end
+    return notfound()
+end
+
+local notfound   = openers.notfound
+local textopener = openers.helpers.textopener
+
+function openers.hashed(specification)
+    local original = specification.original
+    local fullpath = specification.path
+    if fullpath then
+        local found = locateblob(fullpath)
+        if found then
+            if trace_hashed then
+                report_hashed("finder: file %a found",original)
+            end
+            return textopener("hashed",original,found,"utf-8")
+        end
+    end
+    if trace_hashed then
+        report_hashed("finder: unknown file %a",original)
+    end
+    return notfound()
+end
+
+local notfound = loaders.notfound
+
+function loaders.hashed(specification)
+    local original = specification.original
+    local fullpath = specification.path
+    if fullpath then
+        local found = locateblob(fullpath)
+        if found then
+            if trace_hashed then
+                report_hashed("finder: file %a found",original)
+            end
+            return true, found, found and #found or 0
+        end
+    end
+    if trace_hashed then
+        report_hashed("finder: unknown file %a",original)
+    end
+    return notfound()
+end
+
+-- this actually could end up in the generate namespace but it is not
+-- really a 'generic' feature, more a module (at least for now)
+
+local calculatehash = sha2.HEX256 -- md5.HEX is not unique enough
+
+function resolvers.finders.helpers.createhashed(specification)
+    local database = specification.database
+    local patterns = specification.patterns
+    if not patterns then
+        local pattern = specification.pattern
+        if pattern then
+            patterns = {
+                {
+                    pattern  = pattern,
+                    compress = specification.compress,
+                }
+            }
+        end
+    end
+    local datname  = addsuffix(database,"dat")
+    local luaname  = addsuffix(database,"lua")
+    local metadata = loadtable(luaname)
+    if type(metadata) ~= "table" then
+        metadata = false
+    elseif metadata.kind == "hashed" and metadata.version ~= version then
+        report_hashed("version mismatch, starting with new table")
+        metadata = false
+    end
+    if not metadata then
+        metadata = {
+            version       = version,
+            kind          = "hashed",
+            files         = { },
+            hashes        = { },
+            nofnames      = 0,
+            nofpaths      = 0,
+            nofblobs      = 0,
+            nofcompressed = 0,
+        }
+    end
+    local files         = metadata.files
+    local hashes        = metadata.hashes
+    local nofpaths      = metadata.nofpaths
+    local nofnames      = metadata.nofnames
+    local nofblobs      = metadata.nofblobs
+    local nofcompressed = metadata.nofcompressed
+    if type(patterns) == "table" then
+        for i=1,#patterns do
+            local pattern = patterns[i].pattern
+            if pattern then
+                local compress = patterns[i].compress
+                local list     = dir.glob(pattern)
+                local total    = #list
+                report_hashed("database %a, adding pattern %a, compression %l",database,pattern,compress)
+                for i=1,total do
+                    local filename = list[i]
+                    local name     = basename(filename)
+                    local path     = pathpart(filename)
+                    local data     = loaddata(filename)
+                    -- cleanup
+                    path = gsub(path,"^[./]*","")
+                    --
+                    if data then
+                        local fp = files[path]
+                        if not fp then
+                            fp = { }
+                            files[path] = fp
+                            nofpaths = nofpaths + 1
+                        end
+                        local ff = fp[name]
+                        if not ff then
+                            local hash = calculatehash(data)
+                            if not hashes[hash] then
+                                local size = #data
+                                if compress then
+                                    data = zlib.compresssize(data,size)
+                                    nofcompressed = nofcompressed + 1
+                                end
+                                local position = filesize(datname)
+                                savedata(datname,data,"",true)
+                                hashes[hash] = {
+                                    filesize = size,
+                                    datasize = #data,
+                                    compress = compress and "zip",
+                                    position = position,
+                                }
+                                nofblobs = nofblobs + 1
+                            end
+                            fp[name] = hash
+                            nofnames = nofnames + 1
+                        end
+                    end
+                end
+            end
+        end
+    end
+    metadata.nofpaths      = nofpaths
+    metadata.nofnames      = nofnames
+    metadata.nofblobs      = nofblobs
+    metadata.nofcompressed = nofcompressed
+    savetable(luaname, metadata)
+    showstatus(database,metadata)
+    return metadata
+end
+
diff --git a/tex/context/base/mkxl/file-job.lmt b/tex/context/base/mkxl/file-job.lmt
index 90aea19e6..dac6f6d9a 100644
--- a/tex/context/base/mkxl/file-job.lmt
+++ b/tex/context/base/mkxl/file-job.lmt
@@ -10,7 +10,7 @@ if not modules then modules = { } end modules ['file-job'] = {
 -- and push/poppign at the tex end
 
 local next, rawget, tostring, tonumber = next, rawget, tostring, tonumber
-local gsub, match, find = string.gsub, string.match, string.find
+local gsub, match, gmatch, ind = string.gsub, string.match, string.gmatch, string.find
 local insert, remove, concat = table.insert, table.remove, table.concat
 local validstring, formatters = string.valid, string.formatters
 local sortedhash = table.sortedhash
@@ -1300,3 +1300,45 @@ implement {
 --         ctx_doifelse(continue)
     end
 }
+
+-- data-hsh.lmt:
+
+local helpers            = resolvers.finders.helpers
+local validhashed        = helpers.validhashed
+local registerhashed     = helpers.registerhashed
+local registerfilescheme = helpers.registerfilescheme
+
+implement {
+    name      = "registerhashedfiles",
+    public    = true,
+    protected = true,
+    arguments = "optional",
+    actions   = function(list)
+        for name in gmatch(list,"[^, ]+") do
+            registerhashed(name)
+        end
+    end,
+}
+
+implement {
+    name      = "registerfilescheme",
+    public    = true,
+    protected = true,
+    arguments = "optional",
+    actions   = function(list)
+        for name in gmatch(list,"[^, ]+") do
+            registerfilescheme(name)
+        end
+    end,
+}
+
+implement {
+    name      = "doifelsevalidhashedfiles",
+    public    = true,
+    protected = true,
+    arguments = "string",
+    actions   = function(name)
+        ctx_doifelse(validhashed(name))
+    end,
+}
+
diff --git a/tex/context/base/mkxl/file-job.mklx b/tex/context/base/mkxl/file-job.mklx
index e39ed17ed..58c6aa3d5 100644
--- a/tex/context/base/mkxl/file-job.mklx
+++ b/tex/context/base/mkxl/file-job.mklx
@@ -18,6 +18,7 @@
 %D This module delegates most of the work to \LUA\ and therefore also let it
 %D define the commands, which is more efficient.
 
+\registerctxluafile{data-hsh}{autosuffix}
 \registerctxluafile{file-job}{autosuffix}
 
 %D Here are some helpers for processing and path control. In the following example
diff --git a/tex/context/base/mkxl/font-sym.mklx b/tex/context/base/mkxl/font-sym.mklx
index c908ad849..e1de316ef 100644
--- a/tex/context/base/mkxl/font-sym.mklx
+++ b/tex/context/base/mkxl/font-sym.mklx
@@ -170,6 +170,17 @@
    \currentsymbolfont
    \gletcsname\??symbolfont\askedsymbolfont\endcsname\lastrawfontcall}
 
+% \definefontfeature[colored][colr=yes]
+% \definefontsynonym[flags][file:BabelStoneFlagsDual.ttf*colored]
+% \definesymbol[BR][{\getnamedglyphdirect          {flags}{br}}]
+% \definesymbol[PT][{\getnamedglyphdirect          {flags}{pt}}]
+% \definesymbol[BR][{\getnamedglyphdirectscaled{.7}{flags}{br}}]
+% \definesymbol[PT][{\getnamedglyphdirectscaled{.7}{flags}{pt}}]
+%
+% \def\glyphscaled#1{\cldcontext{math.floor(\the\glyphscale*#1)}\relax}
+% \definesymbol[BR][{\glyphscale\glyphscaled{.7}\getnamedglyphdirect{flags}{br}}]
+% \definesymbol[PT][{\glyphscale\glyphscaled{.7}\getnamedglyphdirect{flags}{pt}}]
+
 \permanent\protected\def\getnamedglyphstyled#fontname#character{{\setstyledsymbolicfont{#fontname}\clf_fontchar{#character}}}
 \permanent\protected\def\getnamedglyphdirect#fontname#character{{\setdirectsymbolicfont{#fontname}\clf_fontchar{#character}}}
 \permanent\protected\def\getglyphstyled     #fontname#character{{\setstyledsymbolicfont{#fontname}\doifelsenumber{#character}\char\donothing#character}}
@@ -177,6 +188,9 @@
 \permanent\protected\def\resolvedglyphstyled#fontname#character{{\setstyledsymbolicfont{#fontname}\clf_tochar{#character}}}
 \permanent\protected\def\resolvedglyphdirect#fontname#character{{\setdirectsymbolicfont{#fontname}\clf_tochar{#character}}}
 
+\permanent\protected\def\getnamedglyphdirectscaled#scale#fontname#character%
+  {{\setscaleddirectsymbolicfont\fontbody{#scale}{#fontname}\clf_fontchar{#character}}}
+
 % this one is wrong:
 
 \permanent\protected\def\getscaledglyph#scale#name#content%
diff --git a/tex/context/base/mkxl/lpdf-tag.lmt b/tex/context/base/mkxl/lpdf-tag.lmt
index 3ff058a4e..5b52f56fd 100644
--- a/tex/context/base/mkxl/lpdf-tag.lmt
+++ b/tex/context/base/mkxl/lpdf-tag.lmt
@@ -223,7 +223,7 @@ local function initializepage()
     if pagenum > lastintree then
         lastintree = pagenum
     else
-        report_tags("beware: page order problem in tree at page %i", pagenum)
+     -- report_tags("beware: page order problem in tree at page %i", pagenum)
     end
     tree[pagenum] = list -- we can flush after done, todo
 end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 472079bfb..8985f55de 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date  : 2021-08-06 01:19
+-- merge date  : 2021-08-07 22:49
 
 do -- begin closure to overcome local limits and interference