if not modules then modules = { } end modules ['mtx-update'] = {
version = 1.002,
comment = "companion to mtxrun.lua",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
-- This script is dedicated to Mojca Miklavec, who is the driving force behind
-- moving minimal generation from our internal machines to the context garden.
-- Together with Arthur Reutenauer she made sure that it worked well on all
-- platforms that matter.
-- LuaTeX and LuajitTeX are now always installed together.
local helpinfo = [[
mtx-update
ConTeXt Minimals Updater
1.03
platform (windows, linux, linux-64, osx-intel, osx-ppc, linux-ppc)
repository url (rsync://contextgarden.net)
repository url (minimals)
specify version (current, experimental)
specify version (current, latest, beta, yyyy.mm.dd)
rsync binary (rsync)
installation directory (not guessed for the moment)
tex engine (luatex, pdftex, xetex)
extra modules (can be list or 'all')
additional fonts (can be list or 'all')
extra binaries (like scite and texworks)
instead of a dryrun, do the real thing
update minimal tree
also make formats and generate file databases
don't delete unused or obsolete files
update tree using saved state
adapt drive specs to cygwin
assume mingw binaries being used
less (or no) logging
]]
local application = logs.application {
name = "mtx-update",
banner = "ConTeXt Minimals Updater 1.03",
helpinfo = helpinfo,
}
local report = application.report
local format, concat, gmatch, gsub, find = string.format, table.concat, string.gmatch, string.gsub, string.find
scripts = scripts or { }
scripts.update = scripts.update or { }
local update = scripts.update
minimals = minimals or { }
minimals.config = minimals.config or { }
-- this is needed under windows
-- else rsync fails to set the right chmod flags to files
os.setenv("CYGWIN","nontsec")
update.texformats = {
"cont-en",
"cont-nl",
"cont-cz",
"cont-de",
"cont-fa",
"cont-it",
"cont-ro",
"cont-uk",
"cont-pe",
-- "cont-xp",
"mptopdf",
"plain"
}
-- update.mpformats = {
-- -- "metafun",
-- -- "mpost",
-- }
-- experimental is not functional at the moment
update.repositories = {
"current",
"experimental"
}
-- list of basic folders that are needed to make a functional distribution
update.base = {
{ "base/tex/", "texmf" },
{ "base/metapost/", "texmf" },
{ "fonts/common/", "texmf" },
{ "fonts/other/", "texmf" }, -- not *really* needed, but helpful
{ "context//", "texmf-context" },
{ "misc/setuptex/", "." },
{ "misc/web2c", "texmf" },
{ "bin/common//", "texmf-" },
{ "bin/context//", "texmf-" },
{ "bin/metapost//", "texmf-" },
{ "bin/man/", "texmf-" },
}
-- binaries and font-related files
-- for pdftex we don't need OpenType fonts, for LuaTeX/XeTeX we don't need TFM files
update.defaultengine = "luatex"
update.rsyncvariant = "cygwin" -- will be come mingw
update.engines = {
["luatex"] = {
{ "fonts/new/", "texmf" },
{ "bin/luatex//", "texmf-" },
-- { "bin/luajittex//","texmf-" },
},
["xetex"] = {
{ "base/xetex/", "texmf" },
{ "fonts/new/", "texmf" },
{ "bin/luatex//", "texmf-" }, -- tools
{ "bin/xetex//", "texmf-" },
},
["pdftex"] = {
{ "fonts/old/", "texmf" },
{ "bin/luatex//", "texmf-" }, -- tools
{ "bin/pdftex//", "texmf-" },
},
["all"] = {
{ "fonts/new/", "texmf" },
{ "fonts/old/", "texmf" },
{ "base/xetex/", "texmf" },
{ "bin/luatex//", "texmf-" },
-- { "bin/luajittex//","texmf-" },
{ "bin/xetex//", "texmf-" },
{ "bin/pdftex//", "texmf-" },
},
}
update.goodies = {
["scite"] = {
{ "bin//scite/", "texmf-" },
},
["texworks"] = {
{ "bin//texworks/", "texmf-" },
},
}
update.platforms = {
["mswin"] = "mswin",
["windows"] = "mswin",
["win32"] = "mswin",
["win"] = "mswin",
--
["mswin-64"] = "win64",
["windows-64"] = "win64",
["win64"] = "win64",
--
["linux"] = "linux",
["linux-32"] = "linux",
["linux32"] = "linux",
--
["linux-64"] = "linux-64",
["linux64"] = "linux-64",
--
["linuxmusl-64"] = "linuxmusl-64",
--
["linux-armhf"] = "linux-armhf",
--
["openbsd"] = "openbsd7.3",
["openbsd-i386"] = "openbsd7.3",
["openbsd-amd64"] = "openbsd7.3-amd64",
--
["freebsd"] = "freebsd",
["freebsd-i386"] = "freebsd",
["freebsd-amd64"] = "freebsd-amd64",
--
-- ["kfreebsd"] = "kfreebsd-i386",
-- ["kfreebsd-i386"] = "kfreebsd-i386",
-- ["kfreebsd-amd64"] = "kfreebsd-amd64",
--
-- ["linux-ppc"] = "linux-ppc",
-- ["ppc"] = "linux-ppc",
--
-- ["osx"] = "osx-intel",
-- ["macosx"] = "osx-intel",
-- ["osx-intel"] = "osx-intel",
-- ["osxintel"] = "osx-intel",
--
-- ["osx-ppc"] = "osx-ppc",
-- ["osx-powerpc"] = "osx-ppc",
-- ["osxppc"] = "osx-ppc",
-- ["osxpowerpc"] = "osx-ppc",
--
["macosx"] = "osx-64",
["osx"] = "osx-64",
["osx-64"] = "osx-64",
["osx-arm"] = "osx-arm64",
["osx-arm64"] = "osx-arm64",
--
-- ["solaris-intel"] = "solaris-intel",
--
-- ["solaris-sparc"] = "solaris-sparc",
-- ["solaris"] = "solaris-sparc",
--
["unknown"] = "unknown",
}
local windowsplatform = {
["mswin"] = true,
["win32"] = true,
["win64"] = true,
}
update.selfscripts = {
"mtxrun",
-- "luatools",
}
-- the list is filled up later (when we know what modules to download)
update.modules = {
}
update.fonts = {
}
function update.run(str)
-- important, otherwise formats fly to a weird place
-- (texlua sets luatex as the engine, we need to reset that or to fix texexec :)
os.setenv("engine",nil)
if environment.argument("force") then
report("run, %s",str)
os.execute(str)
else
report("dry run, %s",str)
end
end
function update.fullpath(path)
if file.is_rootbased_path(path) then
return path
else
return lfs.currentdir() .. "/" .. path
end
end
local function drive(d)
if update.rsyncvariant == "cygwin" then
d = gsub(d,[[([a-zA-Z]):/]], "/cygdrive/%1/")
else
d = gsub(d,[[([a-zA-Z]):/]], "/%1/")
end
return d
end
function update.synchronize()
report("update, start")
local texroot = update.fullpath(states.get("paths.root"))
local engines = states.get('engines') or { }
local platforms = states.get('platforms') or { }
local repositories = states.get('repositories') -- minimals
local bin = states.get("rsync.program") -- rsync
local url = states.get("rsync.server") -- contextgarden.net
local version = states.get("context.version") -- current (or beta)
local modules = states.get("modules") -- modules (third party)
local fonts = states.get("fonts") -- fonts (experimental or special)
local goodies = states.get("goodies") -- goodies (like editors)
local force = environment.argument("force")
local silent = environment.argument("silent") and "--silent" or ""
local quiet = silent == "" and "" or "--quiet"
bin = gsub(bin,"\\","/")
if not find(url,"::$") then url = url .. "::" end
local ok = lfs.attributes(texroot,"mode") == "directory"
if not ok and force then
dir.mkdirs(texroot)
ok = lfs.attributes(texroot,"mode") == "directory"
end
if force then
dir.mkdirs(format("%s/%s", texroot, "texmf-cache"))
dir.mkdirs(format("%s/%s", texroot, "texmf-local"))
dir.mkdirs(format("%s/%s", texroot, "texmf-project"))
dir.mkdirs(format("%s/%s", texroot, "texmf-fonts"))
dir.mkdirs(format("%s/%s", texroot, "texmf-modules"))
end
if ok or not force then
local fetched, individual, osplatform = { }, { }, os.platform
-- takes a collection as argument and returns a list of folders
local function collection_to_list_of_folders(collection, platform)
local archives = {}
for i=1,#collection do
local archive = collection[i][1]
archive = gsub(archive,"",platform)
archive = gsub(archive,"",version)
archives[#archives+1] = archive
end
return archives
end
-- takes a list of folders as argument and returns a string for rsync
-- sample input:
-- {'bin/common', 'bin/context'}
-- output:
-- 'minimals/current/bin/common minimals/current/bin/context'
local function list_of_folders_to_rsync_string(list_of_folders)
local repository = 'current'
local prefix = format("%s/%s/", states.get('rsync.module'), repository) -- minimals/current/
return prefix .. concat(list_of_folders, format(" %s", prefix))
end
-- example of usage: print(list_of_folders_to_rsync_string(collection_to_list_of_folders(update.base, os.platform)))
-- rename function and add some more functionality:
-- * recursive/non-recursive (default: non-recursive)
-- * filter folders or regular files only (default: no filter)
-- * grep for size of included files (with --stats switch)
local function get_list_of_files_from_rsync(list_of_folders)
-- temporary file to store the output of rsync (could be a more random name; watch for overwrites)
local temp_file = "rsync.tmp.txt"
-- a set of folders
local folders = {}
local command = format("%s %s'%s' > %s", bin, url, list_of_folders_to_rsync_string(list_of_folders), temp_file)
os.execute(command)
-- read output of rsync
local data = io.loaddata(temp_file) or ""
-- for every line extract the filename : drwxr-sr-x 18 2013/10/06 06:16:10 libertine
for chmod, s in gmatch(data,"([d%-][rwxst%-]+).-(%S+)[\n\r]") do
-- skip "current" folder
if s ~= '.' and #chmod >= 10 then
folders[#folders+1] = s
end
end
-- delete the file to which we have put output of rsync
os.remove(temp_file)
return folders
end
-- rsync://contextgarden.net/minimals/current/modules/
local available_platforms = get_list_of_files_from_rsync({"bin/luatex/"})
report("available platforms: % t",table.sorted(available_platforms))
if modules and type(modules) == "table" then
-- fetch the list of available modules from rsync server
-- local available_modules = get_list_of_files_from_rsync({"modules/"})
-- hash of requested modules
-- local h = table.tohash(modules:split(","))
local available_modules = get_list_of_files_from_rsync({"modules/"})
local asked = table.copy(modules)
asked.all = nil
report("available modules: %s",#available_modules)
for i=1,#available_modules do
local s = available_modules[i]
if modules.all or modules[s] then
update.modules[#update.modules+1] = { format("modules/%s/",s), "texmf-modules" }
report("+ %s",s)
else
report(" %s",s)
end
asked[s] = nil
end
if next(asked) then
report("skipping unknown modules: %s",concat(table.sortedkeys(asked),", "))
end
end
-- rsync://contextgarden.net/minimals/current/fonts/extra/
if fonts and type(fonts) == "table" then
local available_fonts = get_list_of_files_from_rsync({"fonts/extra/"})
local asked = table.copy(fonts)
asked.all = nil
for i=1,#available_fonts do
local s = available_fonts[i]
if fonts.all or fonts[s] then
update.fonts[#update.fonts+1] = { format("fonts/extra/%s/",s), "texmf" }
end
asked[s] = nil
end
if next(asked) then
report("skipping unknown fonts: %s",concat(table.sortedkeys(asked),", "))
end
end
local function add_collection(collection,platform)
if collection and platform then
platform = update.platforms[platform]
if platform then
for i=1,#collection do
local c = collection[i]
local archive = gsub(c[1],"",platform)
local destination = format("%s/%s", texroot, gsub(c[2],"", platform))
destination = gsub(destination,"\\","/")
archive = gsub(archive,"",version)
if osplatform == "windows" or osplatform == "mswin" or osplatform == "win64" then
destination = drive(destination)
end
individual[#individual+1] = { archive, destination }
end
end
end
end
for platform in table.sortedhash(platforms) do
add_collection(update.base,platform)
end
for platform in table.sortedhash(platforms) do
add_collection(update.modules,platform)
end
for platform in table.sortedhash(platforms) do
add_collection(update.fonts,platform)
end
for engine in table.sortedhash(engines) do
for platform in table.sortedhash(platforms) do
add_collection(update.engines[engine],platform)
end
end
if goodies and type(goodies) == "table" then
for goodie in table.sortedhash(goodies) do
for platform in table.sortedhash(platforms) do
add_collection(update.goodies[goodie],platform)
end
end
end
local combined = { }
local update_repositories = update.repositories
for i=1,#update_repositories do
local repository = update_repositories[i]
if repositories[repository] then
for _, v in table.sortedhash(individual) do
local archive, destination = v[1], v[2]
local cd = combined[destination]
if not cd then
cd = { }
combined[destination] = cd
end
cd[#cd+1] = format("%s/%s/%s",states.get('rsync.module'),repository,archive)
end
end
end
for destination, archive in table.sortedhash(combined) do
local archives, command = concat(archive," "), ""
local normalflags, deleteflags = states.get("rsync.flags.normal"), ""
if os.name == "windows" then
normalflags = normalflags .. " -L" -- no symlinks
end
local dryrunflags = ""
if not environment.argument("force") then
dryrunflags = "--dry-run"
end
if (find(destination,"texmf$") or find(destination,"texmf%-context$") or find(destination,"texmf%-modules$")) and (not environment.argument("keep")) then
deleteflags = states.get("rsync.flags.delete")
end
command = format("%s %s %s %s %s'%s' '%s'", bin, normalflags, deleteflags, dryrunflags, url, archives, drive(destination))
-- report("running command: %s",command)
if not fetched[command] then
update.run(command,true)
fetched[command] = command
end
end
local function update_script(script, platform)
local bin = gsub(bin,"\\","/")
local texroot = gsub(texroot,"\\","/")
platform = update.platforms[platform]
if platform then
local command
if windowsplatform[platform] then
bin = drive(bin)
texroot = drive(texroot)
command = format([[%s -t "%s/texmf-context/scripts/context/lua/%s.lua" "%s/texmf-%s/bin/"]], bin, texroot, script, texroot, platform)
else
command = format([[%s -tgo --chmod=a+x '%s/texmf-context/scripts/context/lua/%s.lua' '%s/texmf-%s/bin/%s']], bin, texroot, script, texroot, platform, script)
end
report("updating %s for %s: %s", script, platform, command)
update.run(command)
end
end
for platform in table.sortedhash(platforms) do
for i=1, #update.selfscripts do
update_script(update.selfscripts[i],platform)
end
end
else
report("no valid texroot: %s",texroot)
end
if not force then
report("use --force to really update files")
end
resolvers.load_tree(texroot) -- else we operate in the wrong tree
-- update filename database for pdftex/xetex
update.run(format('mtxrun --tree="%s" %s --direct --resolve mktexlsr %s',texroot,silent,quiet))
-- update filename database for luatex
update.run(format('mtxrun --tree="%s" %s --generate',texroot,silent))
report("update, done")
end
function table.fromhash(t)
local h = { }
for k, v in table.sortedhash(t) do -- not indexed
if v then h[#h+1] = k end
end
return h
end
-- make the ConTeXt formats
function update.make()
report("make, start")
local force = environment.argument("force")
local silent = environment.argument("silent") and "--silent" or ""
local quiet = silent == "" and "" or "--quiet"
local texroot = update.fullpath(states.get("paths.root"))
local engines = states.get('engines')
local goodies = states.get('goodies')
local platforms = states.get('platforms')
local formats = states.get('formats')
resolvers.load_tree(texroot)
update.run(format('mtxrun --tree="%s" %s --direct --resolve mktexlsr %s',texroot,silent,quiet))
update.run(format('mtxrun --tree="%s" %s --generate',texroot,silent))
local askedformats = formats
local texformats = table.tohash(update.texformats)
-- local mpformats = table.tohash(update.mpformats)
for k,v in table.sortedhash(texformats) do
if not askedformats[k] then
texformats[k] = nil
end
end
-- for k,v in table.sortedhash(mpformats) do
-- if not askedformats[k] then
-- mpformats[k] = nil
-- end
-- end
local formatlist = concat(table.fromhash(texformats), " ")
if formatlist ~= "" then
for engine in table.sortedhash(engines) do
if engine == "luatex" or engine == "luajittex" then
update.run(format('mtxrun --tree="%s" %s --script context --autogenerate --make %s',texroot,silent,silent))
update.run(format('mtxrun --tree="%s" %s --script context --autogenerate --make --engine=luajittex %s',texroot,silent,silent))
else
-- update.run(format('mtxrun --tree="%s" %s --script texexec --make --all %s --%s %s',texroot,silent,silent,engine,formatlist))
update.run(format('mtxrun --tree="%s" --resolve %s --script context --resolve --make %s --engine=%s %s',texroot,silent,silent,engine,formatlist))
end
end
end
-- local formatlist = concat(table.fromhash(mpformats), " ")
-- if formatlist ~= "" then
-- update.run(format('mtxrun --tree="%s" %s --script texexec --make --all %s %s',texroot,silent,silent,formatlist))
-- end
if not force then
report("make, use --force to really make formats")
end
-- update.run(format('mtxrun --tree="%s" %s --direct --resolve mktexlsr',texroot,silent)) -- needed for mpost
update.run(format('mtxrun --tree="%s" %s --generate',texroot,silent))
report("make, done")
end
scripts.savestate = true
if scripts.savestate then
states.load("status-of-update.lua")
-- tag, value, default, persistent
statistics.starttiming(states)
states.set("info.version",0.1) -- ok
states.set("info.count",(states.get("info.count") or 0) + 1,1,false) -- ok
states.set("info.comment","this file contains the settings of the last 'mtxrun --script update' run",false) -- ok
states.set("info.date",os.date("!%Y-%m-%d %H:%M:%S")) -- ok
states.set("rsync.program", environment.argument("rsync"), "rsync", true) -- ok
states.set("rsync.server", environment.argument("server"), "contextgarden.net::", true) -- ok
states.set("rsync.module", environment.argument("module"), "minimals", true) -- ok
states.set("rsync.flags.normal", environment.argument("flags"), "-rpztlv", true) -- ok
states.set("rsync.flags.delete", nil, "--delete", true) -- ok
states.set("paths.root", environment.argument("texroot"), "tex", true) -- ok
states.set("context.version", environment.argument("context"), "current", true) -- ok
local valid = table.tohash(update.repositories)
for r in gmatch(environment.argument("repository") or "current","([^, ]+)") do
if valid[r] then states.set(" ." .. r, true) end
end
local valid = update.engines
local engine = environment.argument("engine") or ""
if engine == "" then
local e = states.get("engines")
if not e or not next(e) then
engine = update.defaultengine
end
end
if engine ~= "" then
for r in gmatch(engine,"([^, ]+)") do
if r == "all" then
for k, v in next, valid do
if k ~= "all" then
states.set("engines." .. k, true)
end
end
break
elseif valid[r] then
states.set("engines." .. r, true)
end
end
end
-- old
local valid = update.platforms
for r in gmatch(environment.argument("platform") or os.platform,"([^, ]+)") do
if valid[r] then states.set("platforms." .. r, true) end
end
-- new
-- local osplatform = environment.arguments.platform or nil
-- local platform = platforms[osplatform or os.platform or ""]
--
-- if (platform == "unknown" or platform == "" or not platform) and osplatform then
-- -- catches openbsdN.M kind of specifications
-- platform = osplatform
-- elseif not osplatform then
-- osplatform = platform
-- end
-- states.set("platforms." .. platform, true) end
-- so far
local valid = table.tohash(update.texformats)
for r in gmatch(environment.argument("formats") or "","([^, ]+)") do
if valid[r] then states.set("formats." .. r, true) end
end
-- local valid = table.tohash(update.mpformats)
-- for r in gmatch(environment.argument("formats") or "","([^, ]+)") do
-- if valid[r] then states.set("formats." .. r, true) end
-- end
states.set("formats.cont-en", true)
states.set("formats.cont-nl", true)
-- states.set("formats.metafun", true)
for r in gmatch(environment.argument("extras") or "","([^, ]+)") do -- for old times sake
if r ~= "all" and not find(r,"^[a-z]%-") then
r = "t-" .. r
end
states.set("modules." .. r, true)
end
for r in gmatch(environment.argument("modules") or "","([^, ]+)") do
if r ~= "all" and not find(r,"^[a-z]%-") then
r = "t-" .. r
end
states.set("modules." .. r, true)
end
for r in gmatch(environment.argument("fonts") or "","([^, ]+)") do
states.set("fonts." .. r, true)
end
for r in gmatch(environment.argument("goodies") or "","([^, ]+)") do
states.set("goodies." .. r, true)
end
report("state, loaded")
report()
end
if environment.argument("state") then
environment.setargument("update",true)
environment.setargument("force",true)
environment.setargument("make",true)
end
if environment.argument("mingw") then
update.rsyncvariant = "mingw"
elseif environment.argument("cygwin") then
update.rsyncvariant = "cygwin"
end
if environment.argument("update") then
update.synchronize()
if environment.argument("make") then
update.make()
end
elseif environment.argument("make") then
update.make()
elseif environment.argument("exporthelp") then
application.export(environment.argument("exporthelp"),environment.files[1])
else
application.help()
end
if scripts.savestate then
statistics.stoptiming(states)
states.set("info.runtime",tonumber(statistics.elapsedtime(states)))
if environment.argument("force") then
states.save()
report("state","saved")
end
end