diff options
Diffstat (limited to 'tex/context/base/mkxl')
| -rw-r--r-- | tex/context/base/mkxl/cont-new.mkxl | 2 | ||||
| -rw-r--r-- | tex/context/base/mkxl/context.mkxl | 2 | ||||
| -rw-r--r-- | tex/context/base/mkxl/data-fil.lmt | 144 | ||||
| -rw-r--r-- | tex/context/base/mkxl/data-hsh.lmt | 382 | ||||
| -rw-r--r-- | tex/context/base/mkxl/file-job.lmt | 44 | ||||
| -rw-r--r-- | tex/context/base/mkxl/file-job.mklx | 1 | ||||
| -rw-r--r-- | tex/context/base/mkxl/font-sym.mklx | 14 | ||||
| -rw-r--r-- | tex/context/base/mkxl/lpdf-tag.lmt | 2 |
8 files changed, 587 insertions, 4 deletions
diff --git a/tex/context/base/mkxl/cont-new.mkxl b/tex/context/base/mkxl/cont-new.mkxl index d7196efb4..c9a545148 100644 --- a/tex/context/base/mkxl/cont-new.mkxl +++ b/tex/context/base/mkxl/cont-new.mkxl @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2021.08.06 01:19} +\newcontextversion{2021.08.07 22:49} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkxl/context.mkxl b/tex/context/base/mkxl/context.mkxl index 75388d7b3..8c2bcc2aa 100644 --- a/tex/context/base/mkxl/context.mkxl +++ b/tex/context/base/mkxl/context.mkxl @@ -29,7 +29,7 @@ %D {YYYY.MM.DD HH:MM} format. \immutable\edef\contextformat {\jobname} -\immutable\edef\contextversion{2021.08.06 01:19} +\immutable\edef\contextversion{2021.08.07 22:49} %overloadmode 1 % check frozen / warning %overloadmode 2 % check frozen / error diff --git a/tex/context/base/mkxl/data-fil.lmt b/tex/context/base/mkxl/data-fil.lmt new file mode 100644 index 000000000..bbcc954b2 --- /dev/null +++ b/tex/context/base/mkxl/data-fil.lmt @@ -0,0 +1,144 @@ +if not modules then modules = { } end modules ['data-fil'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local ioopen = io.open +local isdir = lfs.isdir + +local trace_locating = false trackers.register("resolvers.locating", function(v) trace_locating = v end) + +local report_files = logs.reporter("resolvers","files") + +local resolvers = resolvers +local resolveprefix = resolvers.resolve +local findfile = resolvers.findfile +local scanfiles = resolvers.scanfiles +local registerfilehash = resolvers.registerfilehash +local appendhash = resolvers.appendhash + +local loadcachecontent = caches.loadcontent + +local checkgarbage = utilities.garbagecollector and utilities.garbagecollector.check + +function resolvers.locators.file(specification) + local filename = specification.filename + local realname = resolveprefix(filename) -- no shortcut + if realname and realname ~= '' and isdir(realname) then + if trace_locating then + report_files("file locator %a found as %a",filename,realname) + end + appendhash('file',filename,true) -- cache + elseif trace_locating then + report_files("file locator %a not found",filename) + end +end + +function resolvers.hashers.file(specification) + local pathname = specification.filename + local content = loadcachecontent(pathname,'files') + registerfilehash(pathname,content,content==nil) +end + +function resolvers.generators.file(specification) + local pathname = specification.filename + local content = scanfiles(pathname,false,true) -- scan once + registerfilehash(pathname,content,true) +end + +resolvers.concatinators.file = file.join + +local finders = resolvers.finders +local notfound = finders.notfound + +function finders.file(specification,filetype) + local filename = specification.filename + local foundname = findfile(filename,filetype) + if foundname and foundname ~= "" then + if trace_locating then + report_files("file finder: %a found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %a not found",filename) + end + return notfound() + end +end + +-- The default textopener will be overloaded later on. + +local openers = resolvers.openers +local notfound = openers.notfound +local overloaded = false + +local function textopener(tag,filename,f) + return { + reader = function() return f:read () end, + close = function() return f:close() end, + } +end + +function openers.helpers.textopener(...) + return textopener(...) +end + +function openers.helpers.settextopener(opener) + if overloaded then + report_files("file opener: %s overloaded","already") + else + if trace_locating then + report_files("file opener: %s overloaded","once") + end + overloaded = true + textopener = opener + end +end + +function openers.file(specification,filetype) + local filename = specification.filename + if filename and filename ~= "" then + local f = ioopen(filename,"r") + if f then + if trace_locating then + report_files("file opener: %a opened",filename) + end + return textopener("file",filename,f) + end + end + if trace_locating then + report_files("file opener: %a not found",filename) + end + return notfound() +end + +local loaders = resolvers.loaders +local notfound = loaders.notfound + +function loaders.file(specification,filetype) + local filename = specification.filename + if filename and filename ~= "" then + local f = ioopen(filename,"rb") + if f then + if trace_locating then + report_files("file loader: %a loaded",filename) + end + local s = f:read("*a") -- io.readall(f) is faster but we never have large files here + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true, s, #s + end + end + end + if trace_locating then + report_files("file loader: %a not found",filename) + end + return notfound() +end diff --git a/tex/context/base/mkxl/data-hsh.lmt b/tex/context/base/mkxl/data-hsh.lmt new file mode 100644 index 000000000..0a2d94f81 --- /dev/null +++ b/tex/context/base/mkxl/data-hsh.lmt @@ -0,0 +1,382 @@ +-- only lmt because the backend code doesn't deal with it and it makes +-- no sense to waste time on that for mkiv + +if not modules then modules = { } end modules ['data-hsh'] = { + version = 0.002, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- todo: options +-- +-- lowercase +-- cleanupnames (normalize) +-- use database from project tree + +local type = type +local gsub = string.gsub +local addsuffix, basename, pathpart, filesuffix, filesize = file.addsuffix, file.basename, file.pathpart, file.suffix, file.size +local loadtable, savetable = table.load, table.save +local loaddata, savedata, open = io.loaddata, io.savedata, io.open + +local trace_hashed = false +local report_hashed = logs.reporter("resolvers","hashed") + +trackers.register("resolvers.locating", function(v) trace_hashed = v end) +trackers.register("resolvers.hashed", function(v) trace_hashed = v end) + +-- we can have a virtual file: open at the position, make sure read and seek don't +-- go beyond the boundaries + +local resolvers = resolvers +local finders = resolvers.finders +local openers = resolvers.openers +local loaders = resolvers.loaders + +local ordered = { } +local hashed = { } +local version = 0.002 + +-- local lowercase = characters.lower + +local function showstatus(database,metadata) + report_hashed("database %a, %i paths, %i names, %i unique blobs, %i compressed blobs", + database, metadata.nofpaths, metadata.nofnames, metadata.nofblobs, metadata.nofcompressed + ) +end + +local function validhashed(database) + local found = hashed[database] + if found then + return found + else + local metaname = addsuffix(database,"lua") + local dataname = addsuffix(database,"dat") + local metadata = loadtable(metaname) + if type(metadata) ~= "table" then + report_hashed("invalid database %a",metaname) + elseif metadata.version ~= version then + report_hashed("version mismatch in database %a",metaname) + elseif not lfs.isfile(dataname) then + report_hashed("missing data data file for %a",metaname) + else + return { + database = database, + metadata = metadata, + dataname = dataname, + } + end + end +end + +local function registerhashed(database) + if not hashed[database] then + local valid = validhashed(database) + if valid then + ordered[#ordered + 1] = valid + hashed[database] = ordered[#ordered] + showstatus(database,valid.metadata) + end + end +end + +local registerfilescheme do + + local findfile = finders.file + + local list = { } + local done = { } + local hash = { } + + registerfilescheme = function(name) + if not done[name] then + list[#list+1] = name + done[name] = true + end + end + + -- why does the finder not remember ? + + function finders.file(specification,filetype) + if type(specification) == "table" then + local original = specification.original + -- print(original) + if original then + local found = hash[original] + if found == nil then + for i=1,#list do + local scheme = list[i] + local found = finders[scheme](specification,filetype) + if found then + hash[original] = found + if trace_hashed then + report_hashed("found by auto scheme %s: %s",scheme,found) + end + return found + end + end + local found = findfile(specification,filetype) + if found then + hash[original] = found + if trace_hashed then + report_hashed("found by normal file scheme: %s",found) + end + return found + end + hash[original] = false + elseif found then + return found + end + return false + else + -- something is wrong here, maybe we should trace it (scheme can be "unknown") + end + end + -- again, something is wrong + return findfile(specification,filetype) + end + +end + +finders.helpers.validhashed = validhashed +finders.helpers.registerhashed = registerhashed +finders.helpers.registerfilescheme = registerfilescheme + +local function locate(found,path,name) + local files = found.metadata.files + local hashes = found.metadata.hashes + local fp = files[path] + local hash = fp and fp[name] + if hash and hashes[hash] then + return hash + end +end + +local function locatehash(filename,database) + if filename then + local name = basename(filename) + local path = pathpart(filename) + local hash = false + if database then + local found = hashed[database] + if found then + hash = locate(found,path,name), database, path, name + end + else + for i=1,#ordered do + local found = ordered[i] + hash = locate(found,path,name) + if hash then + database = found.database + break + end + end + end + if hash then + return { + hash = hash, + name = name, + path = path, + base = database, + } + end + end +end + +-- no caching yet, we don't always want the file and it's fast enough + +local function locateblob(filename,database) + local found = locatehash(filename,database) + if found then + local database = found.base + local data = hashed[database] + if data then + local metadata = data.metadata + local dataname = data.dataname + local hashes = metadata.hashes + local blobdata = hashes[found.hash] + if blobdata and dataname then + local position = blobdata.position + local f = open(dataname,"rb") + if f then + f:seek("set",position) + local blob = f:read(blobdata.datasize) + if blobdata.compress == "zip" then + blob = zlib.decompresssize(blob,blobdata.filesize) + end + return blob + end + end + end + end +end + +local finders = resolvers.finders +local notfound = finders.notfound + +function finders.hashed(specification) + local original = specification.original + local fullpath = specification.path + if fullpath then + local found = locatehash(fullpath) + if found then + if trace_hashed then + report_hashed("finder: file %a found",original) + end + return original + end + end + if trace_hashed then + report_hashed("finder: unknown file %a",original) + end + return notfound() +end + +local notfound = openers.notfound +local textopener = openers.helpers.textopener + +function openers.hashed(specification) + local original = specification.original + local fullpath = specification.path + if fullpath then + local found = locateblob(fullpath) + if found then + if trace_hashed then + report_hashed("finder: file %a found",original) + end + return textopener("hashed",original,found,"utf-8") + end + end + if trace_hashed then + report_hashed("finder: unknown file %a",original) + end + return notfound() +end + +local notfound = loaders.notfound + +function loaders.hashed(specification) + local original = specification.original + local fullpath = specification.path + if fullpath then + local found = locateblob(fullpath) + if found then + if trace_hashed then + report_hashed("finder: file %a found",original) + end + return true, found, found and #found or 0 + end + end + if trace_hashed then + report_hashed("finder: unknown file %a",original) + end + return notfound() +end + +-- this actually could end up in the generate namespace but it is not +-- really a 'generic' feature, more a module (at least for now) + +local calculatehash = sha2.HEX256 -- md5.HEX is not unique enough + +function resolvers.finders.helpers.createhashed(specification) + local database = specification.database + local patterns = specification.patterns + if not patterns then + local pattern = specification.pattern + if pattern then + patterns = { + { + pattern = pattern, + compress = specification.compress, + } + } + end + end + local datname = addsuffix(database,"dat") + local luaname = addsuffix(database,"lua") + local metadata = loadtable(luaname) + if type(metadata) ~= "table" then + metadata = false + elseif metadata.kind == "hashed" and metadata.version ~= version then + report_hashed("version mismatch, starting with new table") + metadata = false + end + if not metadata then + metadata = { + version = version, + kind = "hashed", + files = { }, + hashes = { }, + nofnames = 0, + nofpaths = 0, + nofblobs = 0, + nofcompressed = 0, + } + end + local files = metadata.files + local hashes = metadata.hashes + local nofpaths = metadata.nofpaths + local nofnames = metadata.nofnames + local nofblobs = metadata.nofblobs + local nofcompressed = metadata.nofcompressed + if type(patterns) == "table" then + for i=1,#patterns do + local pattern = patterns[i].pattern + if pattern then + local compress = patterns[i].compress + local list = dir.glob(pattern) + local total = #list + report_hashed("database %a, adding pattern %a, compression %l",database,pattern,compress) + for i=1,total do + local filename = list[i] + local name = basename(filename) + local path = pathpart(filename) + local data = loaddata(filename) + -- cleanup + path = gsub(path,"^[./]*","") + -- + if data then + local fp = files[path] + if not fp then + fp = { } + files[path] = fp + nofpaths = nofpaths + 1 + end + local ff = fp[name] + if not ff then + local hash = calculatehash(data) + if not hashes[hash] then + local size = #data + if compress then + data = zlib.compresssize(data,size) + nofcompressed = nofcompressed + 1 + end + local position = filesize(datname) + savedata(datname,data,"",true) + hashes[hash] = { + filesize = size, + datasize = #data, + compress = compress and "zip", + position = position, + } + nofblobs = nofblobs + 1 + end + fp[name] = hash + nofnames = nofnames + 1 + end + end + end + end + end + end + metadata.nofpaths = nofpaths + metadata.nofnames = nofnames + metadata.nofblobs = nofblobs + metadata.nofcompressed = nofcompressed + savetable(luaname, metadata) + showstatus(database,metadata) + return metadata +end + diff --git a/tex/context/base/mkxl/file-job.lmt b/tex/context/base/mkxl/file-job.lmt index 90aea19e6..dac6f6d9a 100644 --- a/tex/context/base/mkxl/file-job.lmt +++ b/tex/context/base/mkxl/file-job.lmt @@ -10,7 +10,7 @@ if not modules then modules = { } end modules ['file-job'] = { -- and push/poppign at the tex end local next, rawget, tostring, tonumber = next, rawget, tostring, tonumber -local gsub, match, find = string.gsub, string.match, string.find +local gsub, match, gmatch, ind = string.gsub, string.match, string.gmatch, string.find local insert, remove, concat = table.insert, table.remove, table.concat local validstring, formatters = string.valid, string.formatters local sortedhash = table.sortedhash @@ -1300,3 +1300,45 @@ implement { -- ctx_doifelse(continue) end } + +-- data-hsh.lmt: + +local helpers = resolvers.finders.helpers +local validhashed = helpers.validhashed +local registerhashed = helpers.registerhashed +local registerfilescheme = helpers.registerfilescheme + +implement { + name = "registerhashedfiles", + public = true, + protected = true, + arguments = "optional", + actions = function(list) + for name in gmatch(list,"[^, ]+") do + registerhashed(name) + end + end, +} + +implement { + name = "registerfilescheme", + public = true, + protected = true, + arguments = "optional", + actions = function(list) + for name in gmatch(list,"[^, ]+") do + registerfilescheme(name) + end + end, +} + +implement { + name = "doifelsevalidhashedfiles", + public = true, + protected = true, + arguments = "string", + actions = function(name) + ctx_doifelse(validhashed(name)) + end, +} + diff --git a/tex/context/base/mkxl/file-job.mklx b/tex/context/base/mkxl/file-job.mklx index e39ed17ed..58c6aa3d5 100644 --- a/tex/context/base/mkxl/file-job.mklx +++ b/tex/context/base/mkxl/file-job.mklx @@ -18,6 +18,7 @@ %D This module delegates most of the work to \LUA\ and therefore also let it %D define the commands, which is more efficient. +\registerctxluafile{data-hsh}{autosuffix} \registerctxluafile{file-job}{autosuffix} %D Here are some helpers for processing and path control. In the following example diff --git a/tex/context/base/mkxl/font-sym.mklx b/tex/context/base/mkxl/font-sym.mklx index c908ad849..e1de316ef 100644 --- a/tex/context/base/mkxl/font-sym.mklx +++ b/tex/context/base/mkxl/font-sym.mklx @@ -170,6 +170,17 @@ \currentsymbolfont \gletcsname\??symbolfont\askedsymbolfont\endcsname\lastrawfontcall} +% \definefontfeature[colored][colr=yes] +% \definefontsynonym[flags][file:BabelStoneFlagsDual.ttf*colored] +% \definesymbol[BR][{\getnamedglyphdirect {flags}{br}}] +% \definesymbol[PT][{\getnamedglyphdirect {flags}{pt}}] +% \definesymbol[BR][{\getnamedglyphdirectscaled{.7}{flags}{br}}] +% \definesymbol[PT][{\getnamedglyphdirectscaled{.7}{flags}{pt}}] +% +% \def\glyphscaled#1{\cldcontext{math.floor(\the\glyphscale*#1)}\relax} +% \definesymbol[BR][{\glyphscale\glyphscaled{.7}\getnamedglyphdirect{flags}{br}}] +% \definesymbol[PT][{\glyphscale\glyphscaled{.7}\getnamedglyphdirect{flags}{pt}}] + \permanent\protected\def\getnamedglyphstyled#fontname#character{{\setstyledsymbolicfont{#fontname}\clf_fontchar{#character}}} \permanent\protected\def\getnamedglyphdirect#fontname#character{{\setdirectsymbolicfont{#fontname}\clf_fontchar{#character}}} \permanent\protected\def\getglyphstyled #fontname#character{{\setstyledsymbolicfont{#fontname}\doifelsenumber{#character}\char\donothing#character}} @@ -177,6 +188,9 @@ \permanent\protected\def\resolvedglyphstyled#fontname#character{{\setstyledsymbolicfont{#fontname}\clf_tochar{#character}}} \permanent\protected\def\resolvedglyphdirect#fontname#character{{\setdirectsymbolicfont{#fontname}\clf_tochar{#character}}} +\permanent\protected\def\getnamedglyphdirectscaled#scale#fontname#character% + {{\setscaleddirectsymbolicfont\fontbody{#scale}{#fontname}\clf_fontchar{#character}}} + % this one is wrong: \permanent\protected\def\getscaledglyph#scale#name#content% diff --git a/tex/context/base/mkxl/lpdf-tag.lmt b/tex/context/base/mkxl/lpdf-tag.lmt index 3ff058a4e..5b52f56fd 100644 --- a/tex/context/base/mkxl/lpdf-tag.lmt +++ b/tex/context/base/mkxl/lpdf-tag.lmt @@ -223,7 +223,7 @@ local function initializepage() if pagenum > lastintree then lastintree = pagenum else - report_tags("beware: page order problem in tree at page %i", pagenum) + -- report_tags("beware: page order problem in tree at page %i", pagenum) end tree[pagenum] = list -- we can flush after done, todo end |