1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
|
if not modules then modules = { } end modules ['luat-tmp'] = {
version = 1.001,
comment = "companion to luat-lib.tex",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
}
--[[ldx--
<p>This module deals with caching data. It sets up the paths and
implements loaders and savers for tables. Best is to set the
following variable. When not set, the usual paths will be
checked. Personally I prefer the (users) temporary path.</p>
</code>
TEXMFCACHE=$TMP;$TEMP;$TMPDIR;$TEMPDIR;$HOME;$TEXMFVAR;$VARTEXMF;.
</code>
<p>Currently we do no locking when we write files. This is no real
problem because most caching involves fonts and the chance of them
being written at the same time is small. We also need to extend
luatools with a recache feature.</p>
--ldx]]--
caches = caches or { }
dir = dir or { }
texmf = texmf or { }
caches.path = caches.path or nil
caches.base = caches.base or "luatex-cache"
caches.more = caches.more or "context"
caches.direct = false -- true is faster but may need huge amounts of memory
caches.trace = false
caches.tree = false
caches.temp = caches.temp or os.getenv("TEXMFCACHE") or os.getenv("HOME") or os.getenv("HOMEPATH") or os.getenv("VARTEXMF") or os.getenv("TEXMFVAR") or os.getenv("TMP") or os.getenv("TEMP") or os.getenv("TMPDIR") or nil
caches.paths = caches.paths or { caches.temp }
caches.force = false
input.usecache = not toboolean(os.getenv("TEXMFSHARECACHE") or "false",true) -- true
if caches.temp and caches.temp ~= "" and lfs.attributes(caches.temp,"mode") ~= "directory" then
if caches.force or io.ask(string.format("Should I create the cache path %s?",caches.temp), "no", { "yes", "no" }) == "yes" then
dir.mkdirs(caches.temp)
end
end
if not caches.temp or caches.temp == "" then
print("\nfatal error: there is no valid cache path defined\n")
os.exit()
elseif lfs.attributes(caches.temp,"mode") ~= "directory" then
print(string.format("\nfatal error: cache path %s is not a directory\n",caches.temp))
os.exit()
end
function caches.configpath(instance)
return table.concat(instance.cnffiles,";")
--~ return input.expand_var(instance,"TEXMFCNF")
end
function caches.treehash(instance)
local tree = caches.configpath(instance)
if not tree or tree == "" then
return false
else
return md5.hex(tree)
end
end
function caches.setpath(instance,...)
if not caches.path then
if lfs and instance then
for _,v in pairs(caches.paths) do
for _,vv in pairs(input.expanded_path_list(instance,v)) do
if lfs.isdir(vv) then
caches.path = vv
break
end
end
if caches.path then break end
end
end
if not caches.path then
caches.path = caches.temp
end
caches.path = input.clean_path(caches.path) -- to be sure
if lfs then
caches.tree = caches.tree or caches.treehash(instance)
if caches.tree then
caches.path = dir.mkdirs(caches.path,caches.base,caches.more,caches.tree)
else
caches.path = dir.mkdirs(caches.path,caches.base,caches.more)
end
end
end
if not caches.path then
caches.path = '.'
end
caches.path = input.clean_path(caches.path)
if lfs and not table.is_empty({...}) then
local pth = dir.mkdirs(caches.path,...)
return pth
end
return caches.path
end
function caches.setluanames(path,name)
return path .. "/" .. name .. ".tma", path .. "/" .. name .. ".tmc"
end
function caches.loaddata(path,name)
local tmaname, tmcname = caches.setluanames(path,name)
local loader = loadfile(tmcname) or loadfile(tmaname)
if loader then
return loader()
else
return false
end
end
function caches.is_writable(filepath,filename)
local tmaname, tmcname = caches.setluanames(filepath,filename)
return file.is_writable(tmaname)
end
function caches.savedata(filepath,filename,data,raw) -- raw needed for file cache
local tmaname, tmcname = caches.setluanames(filepath,filename)
local reduce, simplify = true, true
if raw then
reduce, simplify = false, false
end
if caches.direct then
file.savedata(tmaname, table.serialize(data,'return',true,true))
else
table.tofile (tmaname, data,'return',true,true) -- maybe not the last true
end
utils.lua.compile(tmaname, tmcname)
end
-- here we use the cache for format loading (texconfig.[formatname|jobname])
if tex and texconfig and texconfig.formatname and texconfig.formatname == "" then
if not texconfig.luaname then texconfig.luaname = "cont-en.lua" end
texconfig.formatname = caches.setpath(instance,"format") .. "/" .. texconfig.luaname:gsub("%.lu.$",".fmt")
end
--[[ldx--
<p>Once we found ourselves defining similar cache constructs
several times, containers were introduced. Containers are used
to collect tables in memory and reuse them when possible based
on (unique) hashes (to be provided by the calling function).</p>
<p>Caching to disk is disabled by default. Version numbers are
stored in the saved table which makes it possible to change the
table structures without bothering about the disk cache.</p>
<p>Examples of usage can be found in the font related code.</p>
--ldx]]--
containers = { }
containers.trace = false
do -- local report
local function report(container,tag,name)
if caches.trace or containers.trace or container.trace then
logs.report(string.format("%s cache",container.subcategory),string.format("%s: %s",tag,name or 'invalid'))
end
end
function containers.define(category, subcategory, version, enabled)
if category and subcategory then
return {
category = category,
subcategory = subcategory,
storage = { },
enabled = enabled,
version = version or 1.000,
trace = false,
path = caches.setpath(texmf.instance,category,subcategory),
}
else
return nil
end
end
function containers.is_usable(container, name)
return container.enabled and caches.is_writable(container.path, name)
end
function containers.is_valid(container, name)
if name and name ~= "" then
local storage = container.storage[name]
return storage and not table.is_empty(storage) and storage.cache_version == container.version
else
return false
end
end
function containers.read(container,name)
if container.enabled and not container.storage[name] then
container.storage[name] = caches.loaddata(container.path,name)
if containers.is_valid(container,name) then
report(container,"loaded",name)
else
container.storage[name] = nil
end
end
if container.storage[name] then
report(container,"reusing",name)
end
return container.storage[name]
end
function containers.write(container, name, data)
if data then
data.cache_version = container.version
if container.enabled then
local unique, shared = data.unique, data.shared
data.unique, data.shared = nil, nil
caches.savedata(container.path, name, data)
report(container,"saved",name)
data.unique, data.shared = unique, shared
end
report(container,"stored",name)
container.storage[name] = data
end
return data
end
function containers.content(container,name)
return container.storage[name]
end
end
-- since we want to use the cache instead of the tree, we will now
-- reimplement the saver.
function input.aux.save_data(instance, dataname, check)
for cachename, files in pairs(instance[dataname]) do
local name
if input.usecache then
name = file.join(caches.setpath(instance,"trees"),md5.hex(cachename))
else
name = file.join(cachename,dataname)
end
local luaname, lucname = name .. input.luasuffix, name .. input.lucsuffix
input.report("preparing " .. dataname .. " in", luaname)
for k, v in pairs(files) do
if not check or check(v,k) then -- path, name
if type(v) == "table" and #v == 1 then
files[k] = v[1]
end
else
files[k] = nil -- false
end
end
local data = {
type = dataname,
root = cachename,
version = input.cacheversion,
date = os.date("%Y-%m-%d"),
time = os.date("%H:%M:%S"),
content = files,
}
local f = io.open(luaname,'w')
if f then
input.report("saving " .. dataname .. " in", luaname)
-- f:write(table.serialize(data,'return'))
f:write(input.serialize(data))
f:close()
input.report("compiling " .. dataname .. " to", lucname)
if not utils.lua.compile(luaname,lucname) then
input.report("compiling failed for " .. dataname .. ", deleting file " .. lucname)
os.remove(lucname)
end
else
input.report("unable to save " .. dataname .. " in " .. name..input.luasuffix)
end
end
end
function input.serialize(files)
-- This version is somewhat optimized for the kind of
-- tables that we deal with, so it's much faster than
-- the generic serializer. This makes sense because
-- luatools and mtxtools are called frequently. Okay,
-- we pay a small price for properly tabbed tables.
local t = { }
local concat = table.concat
local sorted = table.sortedkeys
local function dump(k,v,m)
if type(v) == 'string' then
return m .. "['" .. k .. "']='" .. v .. "',"
elseif #v == 1 then
return m .. "['" .. k .. "']='" .. v[1] .. "',"
else
return m .. "['" .. k .. "']={'" .. concat(v,"','").. "'},"
end
end
t[#t+1] = "return {"
if instance.sortdata then
for _, k in pairs(sorted(files)) do
local fk = files[k]
if type(fk) == 'table' then
t[#t+1] = "\t['" .. k .. "']={"
for _, kk in pairs(sorted(fk)) do
t[#t+1] = dump(kk,fk[kk],"\t\t")
end
t[#t+1] = "\t},"
else
t[#t+1] = dump(k,fk,"\t")
end
end
else
for k, v in pairs(files) do
if type(v) == 'table' then
t[#t+1] = "\t['" .. k .. "']={"
for kk,vv in pairs(v) do
t[#t+1] = dump(kk,vv,"\t\t")
end
t[#t+1] = "\t},"
else
t[#t+1] = dump(k,v,"\t")
end
end
end
t[#t+1] = "}"
return concat(t,"\n")
end
function input.aux.load_data(instance,pathname,dataname,filename)
local luaname, lucname, pname, fname
if input.usecache then
pname, fname = caches.setpath(instance,"trees"), md5.hex(pathname)
filename = file.join(pname,fname)
else
if not filename or (filename == "") then
filename = dataname
end
pname, fname = pathname, filename
end
luaname = file.join(pname,fname) .. input.luasuffix
lucname = file.join(pname,fname) .. input.lucsuffix
local blob = loadfile(lucname)
if not blob then
blob = loadfile(luaname)
end
if blob then
local data = blob()
if data and data.content and data.type == dataname and data.version == input.cacheversion then
input.report("loading",dataname,"for",pathname,"from",filename)
instance[dataname][pathname] = data.content
else
input.report("skipping",dataname,"for",pathname,"from",filename)
instance[dataname][pathname] = { }
instance.loaderror = true
end
else
input.report("skipping",dataname,"for",pathname,"from",filename)
end
end
-- we will make a better format, maybe something xml or just text or lua
input.automounted = input.automounted or { }
function input.automount(instance,usecache)
local mountpaths = input.simplified_list(input.expansion(instance,'TEXMFMOUNT'))
if table.is_empty(mountpaths) and usecache then
mountpaths = { caches.setpath(instance,"mount") }
end
if not table.is_empty(mountpaths) then
input.starttiming(instance)
for k, root in pairs(mountpaths) do
local f = io.open(root.."/url.tmi")
if f then
for line in f:lines() do
if line then
if line:find("^[%%#%-]") then -- or %W
-- skip
elseif line:find("^zip://") then
input.report("mounting",line)
table.insert(input.automounted,line)
input.usezipfile(instance,line)
end
end
end
f:close()
end
end
input.stoptiming(instance)
end
end
-- store info in format
input.storage = { }
input.storage.data = { }
input.storage.min = 0 -- 500
input.storage.max = input.storage.min - 1
input.storage.trace = false -- true
input.storage.done = 0
input.storage.evaluators = { }
-- (evaluate,message,names)
function input.storage.register(...)
input.storage.data[#input.storage.data+1] = { ... }
end
function input.storage.evaluate(name)
input.storage.evaluators[#input.storage.evaluators+1] = name
end
function input.storage.finalize() -- we can prepend the string with "evaluate:"
for _, t in ipairs(input.storage.evaluators) do
for i, v in pairs(t) do
if type(v) == "string" then
t[i] = loadstring(v)()
elseif type(v) == "table" then
for _, vv in pairs(v) do
if type(vv) == "string" then
t[i] = loadstring(vv)()
end
end
end
end
end
end
function input.storage.dump()
for name, data in ipairs(input.storage.data) do
local evaluate, message, original, target = data[1], data[2], data[3] ,data[4]
local name, initialize, finalize, code = nil, "", "", ""
for str in target:gmatch("([^%.]+)") do
if name then
name = name .. "." .. str
else
name = str
end
initialize = string.format("%s %s = %s or {} ", initialize, name, name)
end
if evaluate then
finalize = "input.storage.evaluate(" .. name .. ")"
end
input.storage.max = input.storage.max + 1
if input.storage.trace then
logs.report('storage',string.format('saving %s in slot %s',message,input.storage.max))
code =
initialize ..
string.format("logs.report('storage','restoring %s from slot %s') ",message,input.storage.max) ..
table.serialize(original,name) ..
finalize
else
code = initialize .. table.serialize(original,name) .. finalize
end
lua.bytecode[input.storage.max] = loadstring(code)
end
end
if lua.bytecode then -- from 0 upwards
local i = input.storage.min
while lua.bytecode[i] do
lua.bytecode[i]()
lua.bytecode[i] = nil
i = i + 1
end
input.storage.done = i
end
|