From f1772caf425af2fe9be87b788eae63559682d51a Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Wed, 19 May 2021 18:48:15 +0200 Subject: 2021-05-19 18:21:00 --- tex/context/base/mkii/cont-new.mkii | 2 +- tex/context/base/mkii/context.mkii | 2 +- tex/context/base/mkiv/cont-new.mkiv | 2 +- tex/context/base/mkiv/context.mkiv | 2 +- tex/context/base/mkiv/font-mis.lua | 2 +- tex/context/base/mkiv/font-ocl.lua | 3 +- tex/context/base/mkiv/font-ocm.lua | 3 +- tex/context/base/mkiv/font-otl.lua | 2 +- tex/context/base/mkiv/font-oup.lua | 93 +++-- tex/context/base/mkiv/font-shp.lua | 2 +- tex/context/base/mkiv/luat-cnf.lua | 37 +- tex/context/base/mkiv/node-pro.lua | 2 - tex/context/base/mkiv/node-syn.lua | 3 +- tex/context/base/mkiv/scrp-cjk.lua | 6 +- tex/context/base/mkiv/scrp-ini.lua | 29 +- tex/context/base/mkiv/status-files.pdf | Bin 23942 -> 23943 bytes tex/context/base/mkiv/status-lua.pdf | Bin 228905 -> 229268 bytes tex/context/base/mkiv/trac-set.lua | 2 +- tex/context/base/mkxl/cont-new.mkxl | 2 +- tex/context/base/mkxl/cont-run.lmt | 6 + tex/context/base/mkxl/context.mkxl | 2 +- tex/context/base/mkxl/font-chk.lmt | 101 ++++- tex/context/base/mkxl/font-ini.mklx | 28 ++ tex/context/base/mkxl/luat-cnf.lmt | 227 +++++++++++ tex/context/base/mkxl/luat-cod.lmt | 48 +-- tex/context/base/mkxl/luat-fio.lmt | 5 - tex/context/base/mkxl/luat-lib.mkxl | 8 +- tex/context/base/mkxl/luat-run.lmt | 332 +++++++++++++++ tex/context/base/mkxl/node-fnt.lmt | 2 +- tex/context/base/mkxl/node-pro.lmt | 135 +++--- tex/context/base/mkxl/node-syn.lmt | 3 +- tex/context/base/mkxl/node-tsk.lmt | 65 ++- tex/context/base/mkxl/scrp-ini.lmt | 29 +- tex/context/base/mkxl/syst-ini.mkxl | 4 - tex/context/base/mkxl/trac-inf.lmt | 149 +++---- tex/context/base/mkxl/trac-set.lmt | 453 +++++++++++++++++++++ tex/context/base/mkxl/typo-dir.lmt | 2 +- tex/context/base/mkxl/util-deb.lmt | 371 +++++++++++++++++ tex/generic/context/luatex/luatex-fonts-merged.lua | 81 ++-- 39 files changed, 1951 insertions(+), 294 deletions(-) create mode 100644 tex/context/base/mkxl/luat-cnf.lmt create mode 100644 tex/context/base/mkxl/luat-run.lmt create mode 100644 tex/context/base/mkxl/trac-set.lmt create mode 100644 tex/context/base/mkxl/util-deb.lmt (limited to 'tex') diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index 5aa85c926..c63412fe3 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2021.05.15 22:41} +\newcontextversion{2021.05.19 18:18} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index dcfe88c5a..8bc3ae15f 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2021.05.15 22:41} +\edef\contextversion{2021.05.19 18:18} %D For those who want to use this: diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 26567e338..b39d55d2c 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2021.05.15 22:41} +\newcontextversion{2021.05.19 18:18} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index 57c7c2001..a3e77c4cd 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -45,7 +45,7 @@ %D {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2021.05.15 22:41} +\edef\contextversion{2021.05.19 18:18} %D Kind of special: diff --git a/tex/context/base/mkiv/font-mis.lua b/tex/context/base/mkiv/font-mis.lua index 8404c04f6..29d4d91c8 100644 --- a/tex/context/base/mkiv/font-mis.lua +++ b/tex/context/base/mkiv/font-mis.lua @@ -21,7 +21,7 @@ local readers = otf.readers if readers then - otf.version = otf.version or 3.114 + otf.version = otf.version or 3.115 otf.cache = otf.cache or containers.define("fonts", "otl", otf.version, true) function fonts.helpers.getfeatures(name,save) diff --git a/tex/context/base/mkiv/font-ocl.lua b/tex/context/base/mkiv/font-ocl.lua index 9e1f49433..7d960922f 100644 --- a/tex/context/base/mkiv/font-ocl.lua +++ b/tex/context/base/mkiv/font-ocl.lua @@ -444,6 +444,7 @@ end -- local indices = fonts.getindices(tfmdata) local descriptions = tfmdata.descriptions local nofshapes = #svgshapes + local s_format = inkscapeformat("pdf") -- hack, this will go away when is >= 0 is everywhere local f_svgfile = formatters["temp-otf-svg-shape-%i.svg"] local f_pdffile = formatters["temp-otf-svg-shape-%i.pdf"] local f_convert = formatters[new and "file-open:%s; export-%s:%s; export-do\n" or "%s --export-%s=%s\n"] @@ -460,7 +461,7 @@ end local svgfile = f_svgfile(index) local pdffile = f_pdffile(index) savedata(svgfile,data) - inkscape:write(f_convert(svgfile,inkscapeformat("pdf"),pdffile)) + inkscape:write(f_convert(svgfile,s_format,pdffile)) processed[index] = true nofdone = nofdone + 1 if nofdone % 25 == 0 then diff --git a/tex/context/base/mkiv/font-ocm.lua b/tex/context/base/mkiv/font-ocm.lua index bb1550ac9..ef0b02a14 100644 --- a/tex/context/base/mkiv/font-ocm.lua +++ b/tex/context/base/mkiv/font-ocm.lua @@ -580,6 +580,7 @@ local initializesvg do -- local indices = fonts.getindices(tfmdata) local descriptions = tfmdata.descriptions local nofshapes = #svgshapes + local s_format = inkscapeformat("pdf") -- hack, this will go away when is >= 0 is everywhere local f_svgfile = formatters["temp-otf-svg-shape-%i.svg"] local f_pdffile = formatters["temp-otf-svg-shape-%i.pdf"] local f_convert = formatters[new and "file-open:%s; export-%s:%s; export-do\n" or "%s --export-%s=%s\n"] @@ -596,7 +597,7 @@ local initializesvg do local svgfile = f_svgfile(index) local pdffile = f_pdffile(index) savedata(svgfile,data) - inkscape:write(f_convert(svgfile,inkscapeformat("pdf"),pdffile)) + inkscape:write(f_convert(svgfile,s_format,pdffile)) processed[index] = true nofdone = nofdone + 1 if nofdone % 25 == 0 then diff --git a/tex/context/base/mkiv/font-otl.lua b/tex/context/base/mkiv/font-otl.lua index 51111fa71..c13011f95 100644 --- a/tex/context/base/mkiv/font-otl.lua +++ b/tex/context/base/mkiv/font-otl.lua @@ -52,7 +52,7 @@ local report_otf = logs.reporter("fonts","otf loading") local fonts = fonts local otf = fonts.handlers.otf -otf.version = 3.114 -- beware: also sync font-mis.lua and in mtx-fonts +otf.version = 3.115 -- beware: also sync font-mis.lua and in mtx-fonts otf.cache = containers.define("fonts", "otl", otf.version, true) otf.svgcache = containers.define("fonts", "svg", otf.version, true) otf.pngcache = containers.define("fonts", "png", otf.version, true) diff --git a/tex/context/base/mkiv/font-oup.lua b/tex/context/base/mkiv/font-oup.lua index 2c49bc993..93a1d526e 100644 --- a/tex/context/base/mkiv/font-oup.lua +++ b/tex/context/base/mkiv/font-oup.lua @@ -2564,6 +2564,34 @@ end -- Because we pack we cannot mix tables and numbers so we can only turn a whole set in -- format kern instead of pair. +local strip_pairs = true + +local compact_pairs = true +local compact_singles = true + +local merge_pairs = true +local merge_singles = true +local merge_substitutions = true +local merge_alternates = true +local merge_multiples = true +local merge_ligatures = true +local merge_cursives = true +local merge_marks = true + +directives.register("otf.strip.pairs", function(v) strip_pairs = v end) + +directives.register("otf.compact.pairs", function(v) compact_pairs = v end) +directives.register("otf.compact.singles", function(v) compact_singles = v end) + +directives.register("otf.merge.pairs", function(v) merge_pairs = v end) +directives.register("otf.merge.singles", function(v) merge_singles = v end) +directives.register("otf.merge.substitutions", function(v) merge_substitutions = v end) +directives.register("otf.merge.alternates", function(v) merge_alternates = v end) +directives.register("otf.merge.multiples", function(v) merge_multiples = v end) +directives.register("otf.merge.ligatures", function(v) merge_ligatures = v end) +directives.register("otf.merge.cursives", function(v) merge_cursives = v end) +directives.register("otf.merge.marks", function(v) merge_marks = v end) + local function checkpairs(lookup) local steps = lookup.steps local nofsteps = lookup.nofsteps @@ -2580,7 +2608,8 @@ local function checkpairs(lookup) local v = d2[1] if v == true then -- all zero - elseif v and (v[1] ~= 0 or v[2] ~= 0 or v[4] ~= 0) then + -- elseif v and (v[1] ~= 0 or v[2] ~= 0 or v[4] ~= 0) then + elseif v and (v[1] ~= 0 or v[2] ~= 0 or v[3] ~= 0 or v[4] ~= 0) then -- vkrn has v[3] ~= 0 return false end end @@ -2617,29 +2646,39 @@ local function checkpairs(lookup) return kerned end -local compact_pairs = true -local compact_singles = true - -local merge_pairs = true -local merge_singles = true -local merge_substitutions = true -local merge_alternates = true -local merge_multiples = true -local merge_ligatures = true -local merge_cursives = true -local merge_marks = true - -directives.register("otf.compact.pairs", function(v) compact_pairs = v end) -directives.register("otf.compact.singles", function(v) compact_singles = v end) +local function strippairs(lookup) + local steps = lookup.steps + local nofsteps = lookup.nofsteps + local stripped = 0 -directives.register("otf.merge.pairs", function(v) merge_pairs = v end) -directives.register("otf.merge.singles", function(v) merge_singles = v end) -directives.register("otf.merge.substitutions", function(v) merge_substitutions = v end) -directives.register("otf.merge.alternates", function(v) merge_alternates = v end) -directives.register("otf.merge.multiples", function(v) merge_multiples = v end) -directives.register("otf.merge.ligatures", function(v) merge_ligatures = v end) -directives.register("otf.merge.cursives", function(v) merge_cursives = v end) -directives.register("otf.merge.marks", function(v) merge_marks = v end) + for i=1,nofsteps do + local step = steps[i] + if step.format == "pair" then + local coverage = step.coverage + for g1, d1 in next, coverage do + for g2, d2 in next, d1 do + if d2[2] then + --- true or { a, b, c, d } + -- else + -- local v = d2[1] + -- if v == true then + -- d1[g2] = nil + -- stripped = stripped + 1 + -- elseif v and (v[1] == 0 and v[2] == 0 and v[4] == 0) then -- vkrn can have v[3] ~= 0 + -- d1[g2] = nil + -- stripped = stripped + 1 + -- end + -- end + elseif d2[1] == true then + d1[g2] = nil + stripped = stripped + 1 + end + end + end + end + end + return stripped +end function readers.compact(data) if not data or data.compacted then @@ -2648,6 +2687,7 @@ function readers.compact(data) data.compacted = true end local resources = data.resources + local stripped = 0 local merged = 0 local kerned = 0 local allsteps = 0 @@ -2678,6 +2718,7 @@ function readers.compact(data) merged = merged + mergesteps_4(lookup) end elseif kind == "gpos_single" then + -- maybe also strip zeros here if merge_singles then merged = merged + mergesteps_1(lookup,true) end @@ -2685,6 +2726,9 @@ function readers.compact(data) kerned = kerned + checkkerns(lookup) end elseif kind == "gpos_pair" then + if strip_pairs then + stripped = stripped + strippairs(lookup) -- noto cjk from 24M -> 8 M + end if merge_pairs then merged = merged + mergesteps_2(lookup) end @@ -2726,6 +2770,9 @@ function readers.compact(data) compact("sequences") compact("sublookups") if trace_optimizations then + if stripped > 0 then + report_optimizations("%i zero positions stripped before merging",stripped) + end if merged > 0 then report_optimizations("%i steps of %i removed due to merging",merged,allsteps) end diff --git a/tex/context/base/mkiv/font-shp.lua b/tex/context/base/mkiv/font-shp.lua index 78e8597b9..ecf0c9418 100644 --- a/tex/context/base/mkiv/font-shp.lua +++ b/tex/context/base/mkiv/font-shp.lua @@ -17,7 +17,7 @@ local pfb = fonts.handlers.pfb local hashes = fonts.hashes local identifiers = hashes.identifiers -local version = 0.009 +local version = 0.010 local shapescache = containers.define("fonts", "shapes", version, true) local streamscache = containers.define("fonts", "streams", version, true) diff --git a/tex/context/base/mkiv/luat-cnf.lua b/tex/context/base/mkiv/luat-cnf.lua index f61ec43d2..f0b18675c 100644 --- a/tex/context/base/mkiv/luat-cnf.lua +++ b/tex/context/base/mkiv/luat-cnf.lua @@ -27,21 +27,6 @@ texconfig.max_strings = 500000 texconfig.hash_extra = 250000 texconfig.function_size = 32768 texconfig.properties_size = 10000 - -if CONTEXTLMTXMODE > 0 then - -texconfig.max_in_open = 2000 -texconfig.nest_size = 10000 -texconfig.param_size = 100000 -texconfig.save_size = 500000 -texconfig.stack_size = 100000 -texconfig.buffer_size = 10000000 -texconfig.token_size = 10000000 -texconfig.node_size = 50000000 -texconfig.max_pool = 10000000 - -else - texconfig.max_in_open = 1000 texconfig.nest_size = 1000 texconfig.param_size = 25000 @@ -50,27 +35,7 @@ texconfig.stack_size = 10000 texconfig.buf_size = 10000000 texconfig.fix_mem_init = 1000000 -end - -local variablenames = CONTEXTLMTXMODE > 0 and { - error_line = false, - half_error_line = false, - max_print_line = false, - max_in_open = false, - expand_depth = true, - hash_extra = true, - nest_size = true, - max_strings = true, - max_pool = true, - param_size = true, - save_size = true, - stack_size = true, - function_size = true, - properties_size = true, - token_size = true, - node_size = true, - buffer_size = true, -} or { +local variablenames = { error_line = false, half_error_line = false, max_print_line = false, diff --git a/tex/context/base/mkiv/node-pro.lua b/tex/context/base/mkiv/node-pro.lua index 6613555f8..8c8356e04 100644 --- a/tex/context/base/mkiv/node-pro.lua +++ b/tex/context/base/mkiv/node-pro.lua @@ -65,8 +65,6 @@ do end -processors.enabled = true -- this will become a proper state (like trackers) - do local hasglyph = nodes.hasglyph diff --git a/tex/context/base/mkiv/node-syn.lua b/tex/context/base/mkiv/node-syn.lua index d2eec6714..8b8b628dd 100644 --- a/tex/context/base/mkiv/node-syn.lua +++ b/tex/context/base/mkiv/node-syn.lua @@ -125,7 +125,7 @@ if not modules then modules = { } end modules ['node-syn'] = { local type, rawset = type, rawset local concat = table.concat local formatters = string.formatters -local replacesuffix, suffixonly, nameonly = file.replacesuffix, file.suffix, file.nameonly +local replacesuffix, suffixonly, nameonly, collapsepath = file.replacesuffix, file.suffix, file.nameonly, file.collapsepath local openfile, renamefile, removefile = io.open, os.rename, os.remove local report_system = logs.reporter("system") @@ -232,6 +232,7 @@ local blockedsuffixes = { } local sttags = table.setmetatableindex(function(t,name) + name = collapsepath(name) if blockedsuffixes[suffixonly(name)] then -- Just so that I don't get the ones on my development tree. nofblocked = nofblocked + 1 diff --git a/tex/context/base/mkiv/scrp-cjk.lua b/tex/context/base/mkiv/scrp-cjk.lua index 541ea9f81..d28b7f922 100644 --- a/tex/context/base/mkiv/scrp-cjk.lua +++ b/tex/context/base/mkiv/scrp-cjk.lua @@ -19,8 +19,6 @@ if not modules then modules = { } end modules ['scrp-cjk'] = { local nuts = nodes.nuts -local insertnodeafter = nuts.insertafter -local insertnodebefore = nuts.insertbefore local copy_node = nuts.copy local remove_node = nuts.remove local nextglyph = nuts.traversers.glyph @@ -70,6 +68,10 @@ local report_details = logs.reporter("scripts","detail") -- the intercharacter spacing interferes with this; the solution is to patch the -- nodelist but better is to use veryraggedleft +local insertnodeafter = scripts.helpers.insertnodeafter +local insertnodebefore = scripts.helpers.insertnodebefore + + local inter_char_shrink = 0 local inter_char_stretch = 0 local inter_char_half_shrink = 0 diff --git a/tex/context/base/mkiv/scrp-ini.lua b/tex/context/base/mkiv/scrp-ini.lua index 0fafd9854..857d2ac6e 100644 --- a/tex/context/base/mkiv/scrp-ini.lua +++ b/tex/context/base/mkiv/scrp-ini.lua @@ -57,9 +57,6 @@ local setglyphdata = nuts.setglyphdata local isglyph = nuts.isglyph -local insertnodeafter = nuts.insertafter -local insertnodebefore = nuts.insertbefore - local firstglyph = nuts.firstglyph local nextglyph = nuts.traversers.glyph @@ -86,6 +83,9 @@ scripts.injectors = handlers local splitters = allocate() scripts.splitters = splitters +local helpers = allocate() +scripts.helpers = helpers + -- we need to fake it in luatex local getscript = node.direct.getscript @@ -116,6 +116,29 @@ if not getscript then end +local insertnodebefore, insertnodeafter do + + local insertafter = nuts.insertnodeafter + local insertbefore = nuts.insertnodebefore + local setattributelist = nuts.setattributelist + + local function insertnodebefore(head,current,what) -- todo : lmtx + head, current = insertbefore(head,current,what) + setattributelist(what,current) + return head, current + end + + local function insertnodeafter(head,current,what) -- todo : lmtx + head, current = insertafter(head,current,what) + setattributelist(what,current) + return head, current + end + + helpers.insertnodebefore = insertnodebefore + helpers.insertnodeafter = insertnodeafter + +end + local hash = { -- we could put these presets in char-def.lua -- -- half width opening parenthesis diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf index 461037595..59d7fb57e 100644 Binary files a/tex/context/base/mkiv/status-files.pdf and b/tex/context/base/mkiv/status-files.pdf differ diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf index cd95f9e07..1cb90cb26 100644 Binary files a/tex/context/base/mkiv/status-lua.pdf and b/tex/context/base/mkiv/status-lua.pdf differ diff --git a/tex/context/base/mkiv/trac-set.lua b/tex/context/base/mkiv/trac-set.lua index 4edb5d129..0441f386b 100644 --- a/tex/context/base/mkiv/trac-set.lua +++ b/tex/context/base/mkiv/trac-set.lua @@ -409,7 +409,7 @@ if texconfig then -- this happens too late in ini mode but that is no problem local function set(k,v) - v = tonumber(v) + local v = tonumber(v) if v then texconfig[k] = v end diff --git a/tex/context/base/mkxl/cont-new.mkxl b/tex/context/base/mkxl/cont-new.mkxl index fd33d7db1..229b27d2b 100644 --- a/tex/context/base/mkxl/cont-new.mkxl +++ b/tex/context/base/mkxl/cont-new.mkxl @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2021.05.15 22:41} +\newcontextversion{2021.05.19 18:18} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkxl/cont-run.lmt b/tex/context/base/mkxl/cont-run.lmt index 5c9ed4003..a256a4104 100644 --- a/tex/context/base/mkxl/cont-run.lmt +++ b/tex/context/base/mkxl/cont-run.lmt @@ -230,6 +230,12 @@ local function processjob() } end + logs.registerfinalactions(function() + logs.pushtarget("log") + statistics.showusage("finish") + logs.poptarget() + end) + setoverloadmode(arguments.overloadmode) if not filename or filename == "" then diff --git a/tex/context/base/mkxl/context.mkxl b/tex/context/base/mkxl/context.mkxl index 0f19da913..18963a7ec 100644 --- a/tex/context/base/mkxl/context.mkxl +++ b/tex/context/base/mkxl/context.mkxl @@ -29,7 +29,7 @@ %D {YYYY.MM.DD HH:MM} format. \immutable\edef\contextformat {\jobname} -\immutable\edef\contextversion{2021.05.15 22:41} +\immutable\edef\contextversion{2021.05.19 18:18} %overloadmode 1 % check frozen / warning %overloadmode 2 % check frozen / error diff --git a/tex/context/base/mkxl/font-chk.lmt b/tex/context/base/mkxl/font-chk.lmt index eb1be3f1b..2beec268f 100644 --- a/tex/context/base/mkxl/font-chk.lmt +++ b/tex/context/base/mkxl/font-chk.lmt @@ -12,7 +12,8 @@ if not modules then modules = { } end modules ['font-chk'] = { -- instead we just keep the method we use but slightly adapted to the backend -- of lmtx. -local next = next +local type, next = type, next +local find, lower, gmatch = string.find, string.lower, string.gmatch local floor = math.floor local context = context @@ -22,12 +23,15 @@ local bpfactor = number.dimenfactors.bp local fastcopy = table.fastcopy local sortedkeys = table.sortedkeys local sortedhash = table.sortedhash +local contains = table.contains local report = logs.reporter("fonts") local report_checking = logs.reporter("fonts","checking") local allocate = utilities.storage.allocate +local getmacro = tokens.getters.macro + local fonts = fonts fonts.checkers = fonts.checkers or { } @@ -40,6 +44,7 @@ local fontcharacters = fonthashes.characters local currentfont = font.current local addcharacters = font.addcharacters +local definers = fonts.definers local helpers = fonts.helpers local addprivate = helpers.addprivate @@ -452,3 +457,97 @@ local visualspace_specification = { registerotffeature(visualspace_specification) registerafmfeature(visualspace_specification) + +do + + + local reference = 88 -- string.byte("X") + local mapping = { ss = "sans", rm = "serif", tt = "mono" } + local order = { "sans", "serif", "mono" } + local fallbacks = { sans = { }, serif = { }, mono = { } } + + local function locate(fallbacks,n,f,c) + for i=1,#fallbacks do + local id = fallbacks[i] + if type(id) == "string" then + id = definers.define { name = id } + fallbacks[i] = id + end + if type(id) == "number" then + local cid = fontcharacters[id] + if cid[c] then + local fc = fontcharacters[f] + local sc = (fc[reference].height / cid[reference].height) * (n.scale or 1000) + return { id, sc } + end + end + end + return false + end + + local cache = table.setmetatableindex("table") + + callback.register("missing_character", function(n,f,c) + local cached = cache[f] + local found = cached[c] + if found == nil then + local metadata = fontdata[f].shared + if metadata then + metadata = metadata.rawdata + if metadata then + metadata = metadata.metadata + if metadata then + if metadata.monospaced then + found = locate(fallbacks.mono,n,f,c) + if found then + cached[c] = found + goto done + end + end + local fn = lower(metadata.fullname) + for i=1,3 do + local o = order[i] + if find(fn,o) then + found = locate(fallbacks[o],n,f,c) + if found then + cached[c] = found + goto done + end + end + end + end + end + end + found = locate(fallbacks[mapping[getmacro("fontstyle")] or "mono"],n,f,c) + if found then + cached[c] = found + goto done + end + end + ::done:: + if found then + n.font = found[1] + n.scale = found[2] + end + end) + + function definers.registerfallbackfont(style,list) + local l = fallbacks[style] + if l then + for s in gmatch(list,"%S+") do + if not contains(l,s) then + l[#l+1] = s + end + end + end + end + + implement { + name = "registerfallbackfont", + public = true, + protected = true, + arguments = { "optional", "optional" }, + actions = definers.registerfallbackfont, + } + +end diff --git a/tex/context/base/mkxl/font-ini.mklx b/tex/context/base/mkxl/font-ini.mklx index 557ac3c49..fa512b9b0 100644 --- a/tex/context/base/mkxl/font-ini.mklx +++ b/tex/context/base/mkxl/font-ini.mklx @@ -2690,4 +2690,32 @@ \permanent\protected\def\usefontpath[#1]% {\clf_addfontpath{#1}} +%D Experiment (one can use a list): +%D +%D \starttyping +%D \setupfonts +%D [serif=dejavuserif*default, +%D sans=dejavusans*default, +%D mono=dejavusansmono*none] +%D +%D {\rm A \char1234\ B \char1236\ C} +%D {\ss A \char1234\ B \char1236\ C} +%D {\ss A \char1234\ B \char1236\ C} +%D \stoptyping + +\appendtoks + \edef\m_list{\fontsparameter\s!serif}% + \ifempty\m_list\else + \registerfallbackfont[\s!serif][\m_list]% + \fi + \edef\m_list{\fontsparameter\s!sans}% + \ifempty\m_list\else + \registerfallbackfont[\s!sans][\m_list]% + \fi + \edef\m_list{\fontsparameter\s!mono}% + \ifempty\m_list\else + \registerfallbackfont[\s!mono][\m_list]% + \fi +\to \everysetupfonts + \protect \endinput diff --git a/tex/context/base/mkxl/luat-cnf.lmt b/tex/context/base/mkxl/luat-cnf.lmt new file mode 100644 index 000000000..2a8d40788 --- /dev/null +++ b/tex/context/base/mkxl/luat-cnf.lmt @@ -0,0 +1,227 @@ +if not modules then modules = { } end modules ['luat-cnf'] = { + version = 1.001, + comment = "companion to luat-lib.mkxl", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local type = type +local format, concat = string.format, table.concat + +local report = logs.reporter("system") + +luatex = luatex or { } +local luatex = luatex + +texconfig.functionsize = 32768 +texconfig.propertiessize = 10000 + +-- These lists need updating! If we decide to keep them at all. + +local stub = [[ + +-- checking + +storage = storage or { } +luatex = luatex or { } + +-- as soon as possible + +texconfig.functionsize = 32768 +texconfig.propertiessize = 10000 + +luatex.starttime = os.gettimeofday() + +-- this will happen after the format is loaded + +function texconfig.init() + + -- development + + local builtin, globals = { }, { } + + libraries = { -- we set it here as we want libraries also 'indexed' + basiclua = { + -- always + "string", "table", "coroutine", "debug", "file", "io", "lpeg", "math", "os", "package", + -- bonus + "bit32", "utf8", + }, + basictex = { + -- always + "callback", "font", "lua", "node", "status", "tex", "texconfig", "texio", "token", + -- not in luametatex + "img", "pdf", "lang", + -- in luametatex + "language", + }, + extralua = { + -- not in luametatex + "unicode", "utf", "gzip", "zip", "zlib", + -- in luametatex + "xzip", "xmath", "xcomplex", "xdecimal", "basexx", + -- maybe some day in luametatex + "lz4", "lzo", + -- always (mime can go) + "lfs","socket", "mime", "md5", "sha2", "fio", "sio", + }, + extratex = { + -- not in luametatex + "kpse", + -- always + "pdfe", "mplib", + -- in luametatex + "pdfdecode", "pngdecode", + }, + obsolete = { + "epdf", + "fontloader", -- can be filled by luat-log + "kpse", + }, + functions = { + "assert", "pcall", "xpcall", "error", "collectgarbage", + "dofile", "load","loadfile", "require", "module", + "getmetatable", "setmetatable", + "ipairs", "pairs", "rawequal", "rawget", "rawset", "next", + "tonumber", "tostring", + "type", "unpack", "select", "print", + }, + builtin = builtin, -- to be filled + globals = globals, -- to be filled + } + + for k, v in next, _G do + globals[k] = tostring(v) + end + + local function collect(t,fnc) + local lib = { } + for k, v in next, t do + if fnc then + lib[v] = _G[v] + else + local keys = { } + local gv = _G[v] + local tv = type(gv) + if tv == "table" then + for k, v in next, gv do + keys[k] = tostring(v) -- true -- by tostring we cannot call overloads functions (security) + end + end + lib[v] = keys + builtin[v] = keys + end + end + return lib + end + + libraries.basiclua = collect(libraries.basiclua) + libraries.basictex = collect(libraries.basictex) + libraries.extralua = collect(libraries.extralua) + libraries.extratex = collect(libraries.extratex) + libraries.functions = collect(libraries.functions,true) + libraries.obsolete = collect(libraries.obsolete) + + -- shortcut and helper + + local setbytecode = lua.setbytecode + local getbytecode = lua.getbytecode + local callbytecode = lua.callbytecode or function(i) + local b = getbytecode(i) + if type(b) == "function" then + b() + return true + else + return false + end + end + + local function init(start) + local i = start + local t = os.clock() + while true do + -- local b = callbytecode(i) + local e, b = pcall(callbytecode,i) + if not e then + print(string.format("\nfatal error : unable to load bytecode register %%i, maybe wipe the cache first\n",i)) + os.exit() + end + if b then + setbytecode(i,nil) ; + i = i + 1 + else + break + end + end + return i - start, os.clock() - t + end + + -- the stored tables and modules + + storage.noftables , storage.toftables = init(0) + storage.nofmodules, storage.tofmodules = init(%s) + + if modules then + local loaded = package.loaded + for module, _ in next, modules do + loaded[module] = true + end + end + + texconfig.init = function() end + +end + +CONTEXTLMTXMODE = 1 + +-- we provide a qualified path + +callback.register('find_format_file',function(name) + texconfig.formatname = name + return name +end) + +-- done, from now on input and callbacks are internal +]] + +local keys = { + "buffersize", "expandsize", "filesize", "fontsize", "hashsize", "inputsize", + "languagesize", "marksize", "nestsize", "nodesize", "parametersize", "poolsize", + "savesize", "stringsize", "tokensize", "errorlinesize", "halferrorlinesize", +} + +local function makestub() + name = name or (environment.jobname .. ".lui") + report("creating stub file %a using directives:",name) + report() + firsttable = firsttable or lua.firstbytecode + local t = { + "-- This file is generated, don't change it!\n" + } + for i=1,#keys do + local target = keys[i] + local key = "luametatex." .. target + local val = directives.value(key) + if type(val) == "number" then + val = { size = val } + end + if type(val) == "table" then + local s = { } + local v = val.size if v then s[#s+1] = format("size = %10i",v) end + local v = val.plus if v then s[#s+1] = format("plus = %10i",v) end + local v = val.step if v then s[#s+1] = format("step = %10i",v) end + if #s > 0 then + s = format("%-17s = { %s }",target,concat(s,", ")) + report(" %s",s) + t[#t+1] = "texconfig." .. s + end + end + end + t[#t+1] = "" + t[#t+1] = format(stub,firsttable) + io.savedata(name,concat(t,"\n")) + report() +end + +lua.registerfinalizer(makestub,"create stub file") diff --git a/tex/context/base/mkxl/luat-cod.lmt b/tex/context/base/mkxl/luat-cod.lmt index 64c5e71f3..7047726b5 100644 --- a/tex/context/base/mkxl/luat-cod.lmt +++ b/tex/context/base/mkxl/luat-cod.lmt @@ -17,29 +17,31 @@ local texconfig, lua = texconfig, lua -- some basic housekeeping -texconfig.kpse_init = false -- not needed in lmtx -texconfig.shell_escape = 't' -- not needed in lmtx - -texconfig.max_in_open = 2000 -texconfig.nest_size = 10000 -texconfig.param_size = 100000 -texconfig.save_size = 500000 -texconfig.stack_size = 100000 -texconfig.buffer_size = 10000000 -texconfig.token_size = 10000000 -texconfig.node_size = 50000000 - -texconfig.max_print_line = 100000 -texconfig.max_strings = 500000 -texconfig.max_pool = 10000000 - -texconfig.hash_extra = 250000 - -texconfig.expand_depth = 10000 -texconfig.function_size = 32768 -texconfig.properties_size = 10000 -texconfig.error_line = 250 -texconfig.half_error_line = 125 +---------.kpse_init = false -- not needed in lmtx +---------.shell_escape = 't' -- not needed in lmtx + +-- texconfig.max_in_open = 2000 +-- texconfig.nest_size = 10000 +-- texconfig.param_size = 100000 +-- texconfig.save_size = 500000 +-- texconfig.stack_size = 100000 +-- texconfig.buffer_size = 10000000 +-- texconfig.token_size = 10000000 +-- texconfig.token_size = { size = 10000000, step = 2000000 } +-- texconfig.node_size = 50000000 + +---------.max_print_line = 100000 +-- texconfig.max_strings = 500000 +-- texconfig.max_strings = { size = 600000, step = 200000 } +-- texconfig.max_pool = 10000000 +-- texconfig.hash_extra = 250000 +-- texconfig.expand_depth = 10000 + +-- texconfig.error_line = 250 +-- texconfig.half_error_line = 125 + +texconfig.functionsize = 32768 +texconfig.propertiessize = 10000 -- registering bytecode chunks diff --git a/tex/context/base/mkxl/luat-fio.lmt b/tex/context/base/mkxl/luat-fio.lmt index 228b346bc..2c6247225 100644 --- a/tex/context/base/mkxl/luat-fio.lmt +++ b/tex/context/base/mkxl/luat-fio.lmt @@ -9,11 +9,6 @@ if not modules then modules = { } end modules ['luat-fio'] = { local format = string.format local concat = table.concat -texconfig.kpse_init = false -- can go away -texconfig.shell_escape = 't' -- can go away -texconfig.max_print_line = 100000 -- can go away -texconfig.max_in_open = 1000 -- can go away - if not resolvers.initialized() then resolvers.reset() diff --git a/tex/context/base/mkxl/luat-lib.mkxl b/tex/context/base/mkxl/luat-lib.mkxl index 9bbcf7d17..a01e7f86f 100644 --- a/tex/context/base/mkxl/luat-lib.mkxl +++ b/tex/context/base/mkxl/luat-lib.mkxl @@ -23,11 +23,11 @@ \registerctxluafile{util-fmt}{} \registerctxluafile{util-dim}{} -\registerctxluafile{trac-set}{} +\registerctxluafile{trac-set}{autosuffix} \registerctxluafile{luat-log}{autosuffix} \registerctxluafile{trac-inf}{autosuffix} \registerctxluafile{util-lua}{} -\registerctxluafile{util-deb}{} % could also be done in trac-deb.mkiv +\registerctxluafile{util-deb}{autosuffix} \registerctxluafile{util-tpl}{} % needs tracker \registerctxluafile{util-seq}{} @@ -72,9 +72,9 @@ \registerctxluafile{data-aux}{} \registerctxluafile{luat-cbk}{autosuffix} -\registerctxluafile{luat-run}{} +\registerctxluafile{luat-run}{autosuffix} \registerctxluafile{luat-fio}{autosuffix} -\registerctxluafile{luat-cnf}{} +\registerctxluafile{luat-cnf}{autosuffix} \registerctxluafile{luat-lua}{} \registerctxluafile{luat-sto}{} \registerctxluafile{luat-ini}{autosuffix} diff --git a/tex/context/base/mkxl/luat-run.lmt b/tex/context/base/mkxl/luat-run.lmt new file mode 100644 index 000000000..8a9593f3f --- /dev/null +++ b/tex/context/base/mkxl/luat-run.lmt @@ -0,0 +1,332 @@ +if not modules then modules = { } end modules ['luat-run'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local next = next +local find = string.find +local insert, remove = table.insert, table.remove +local osexit = os.exit + +-- trace_job_status is also controlled by statistics.enable that is set via the directive system.nostatistics + +local trace_lua_dump = false trackers.register("system.dump", function(v) trace_lua_dump = v end) +local trace_temp_files = false trackers.register("system.tempfiles", function(v) trace_temp_files = v end) +local trace_job_status = true trackers.register("system.jobstatus", function(v) trace_job_status = v end) +local trace_tex_status = false trackers.register("system.texstatus", function(v) trace_tex_status = v end) + +local report_lua = logs.reporter("system","lua") +local report_tex = logs.reporter("system","status") +local report_tempfiles = logs.reporter("resolvers","tempfiles") + +luatex = luatex or { } +local luatex = luatex +local synctex = luatex.synctex + +if not synctex then + synctex = table.setmetatableindex(function() return function() end end) + luatex.synctex = synctex +end + +local startactions = { } +local stopactions = { } +local dumpactions = { } +local pageactions = { } + +function luatex.registerstartactions(...) insert(startactions, ...) end +function luatex.registerstopactions (...) insert(stopactions, ...) end +function luatex.registerdumpactions (...) insert(dumpactions, ...) end +function luatex.registerpageactions (...) insert(pageactions, ...) end + +local function start_run() + if logs.start_run then + logs.start_run() + end + -- logs.report("engine","%s version %s, format id %s",LUATEXENGINE,LUATEXVERSION,LUATEXFORMATID) + for i=1,#startactions do + startactions[i]() + end +end + +local function stop_run() + for i=1,#stopactions do + stopactions[i]() + end + local quit = logs.finalactions() + if trace_job_status then + statistics.show() + end + if trace_tex_status then + logs.newline() + for k, v in table.sortedhash(status.list()) do + if type(v) ~= "table" then + report_tex("%S=%S",k,v) + end + end + end + if quit then + local setexitcode = lua.setexitcode or status.setexitcode + if setexitcode then + setexitcode(1) + if type(quit) == "table" then + logs.newline() + report_tex("quitting due to: %, t",quit) + logs.newline() + end + end + end + if logs.stop_run then + logs.stop_run() + end +end + +local function start_shipout_page() + synctex.start() + logs.start_page_number() +end + +local function stop_shipout_page() + logs.stop_page_number() + for i=1,#pageactions do + pageactions[i]() + end + synctex.stop() +end + +local function report_output_pages() +end + +local function report_output_log() +end + +local function pre_dump_actions() + for i=1,#dumpactions do + dumpactions[i]() + end + lua.finalize(trace_lua_dump and report_lua or nil) +end + +local function wrapup_synctex() + synctex.wrapup() +end + +-- For Taco ... + +local sequencers = utilities.sequencers +local appendgroup = sequencers.appendgroup +local appendaction = sequencers.appendaction +local wrapupactions = sequencers.new { } +local cleanupactions = sequencers.new { } + +appendgroup(wrapupactions,"system") +appendgroup(wrapupactions,"user") + +appendgroup(cleanupactions,"system") +appendgroup(cleanupactions,"user") + +local function wrapup_run(someerror) + local runner = wrapupactions.runner + if runner then + runner(someerror) -- we could use the error flag in lmtx + end +end + +local function cleanup_run() + local runner = cleanupactions.runner + if runner then + runner() + end +end + +function luatex.wrapup(action) + appendaction(wrapupactions,"user",action) +end + +function luatex.cleanup(action) + appendaction(cleanupactions,"user",action) +end + +function luatex.abort() + cleanup_run() + osexit(1) +end + +appendaction(wrapupactions,"system",synctex.wrapup) + +-- this can be done later + +callbacks.register('start_run', start_run, "actions performed at the beginning of a run") +callbacks.register('stop_run', stop_run, "actions performed at the end of a run") + +---------.register('show_open', show_open, "actions performed when opening a file") +---------.register('show_close', show_close, "actions performed when closing a file") + +callbacks.register('report_output_pages', report_output_pages, "actions performed when reporting pages") +callbacks.register('report_output_log', report_output_log, "actions performed when reporting log file") + +---------.register('start_page_number', start_shipout_page, "actions performed at the beginning of a shipout") +---------.register('stop_page_number', stop_shipout_page, "actions performed at the end of a shipout") + +callbacks.register('start_page_number', function() end, "actions performed at the beginning of a shipout") +callbacks.register('stop_page_number', function() end, "actions performed at the end of a shipout") + +callbacks.register('process_input_buffer', false, "actions performed when reading data") +callbacks.register('process_output_buffer', false, "actions performed when writing data") + +callbacks.register("pre_dump", pre_dump_actions, "lua related finalizers called before we dump the format") -- comes after \everydump + +-- finish_synctex might go away (move to wrapup_run) + +callbacks.register("finish_synctex", wrapup_synctex, "rename temporary synctex file") +callbacks.register('wrapup_run', wrapup_run, "actions performed after closing files") + +-- temp hack for testing: + +callbacks.functions.start_page_number = start_shipout_page +callbacks.functions.stop_page_number = stop_shipout_page + +-- an example: + +local tempfiles = { } + +function luatex.registertempfile(name,extrasuffix,keep) -- namespace might change + if extrasuffix then + name = name .. ".mkiv-tmp" -- maybe just .tmp + end + if trace_temp_files and not tempfiles[name] then + if keep then + report_tempfiles("%s temporary file %a","registering",name) + else + report_tempfiles("%s temporary file %a","unregistering",name) + end + end + tempfiles[name] = keep or false + return name +end + +function luatex.cleanuptempfiles() + for name, keep in next, tempfiles do + if not keep then + if trace_temp_files then + report_tempfiles("%s temporary file %a","removing",name) + end + os.remove(name) + end + end + tempfiles = { } +end + +luatex.registerstopactions(luatex.cleanuptempfiles) + +-- Reporting filenames has been simplified since lmtx because we don't need the +-- traditional () {} <> etc methods (read: that directive option was never chosen). + +local report_open = logs.reporter("open source") +local report_close = logs.reporter("close source") +local report_load = logs.reporter("load resource") + +local register = callbacks.register + +local level = 0 +local total = 0 +local stack = { } + +function luatex.currentfile() + return stack[#stack] or tex.jobname +end + +local function report_start(name,rest) + if rest then + -- luatex + if name ~= 1 then + insert(stack,false) + return + end + name = rest + end + if find(name,"virtual://",1,true) then + insert(stack,false) + else + insert(stack,name) + total = total + 1 + level = level + 1 + -- report_open("%i > %i > %s",level,total,name or "?") + report_open("level %i, order %i, name %a",level,total,name or "?") + synctex.setfilename(name) + end +end + +local function report_stop() + local name = remove(stack) + if name then + -- report_close("%i > %i > %s",level,total,name or "?") + report_close("level %i, order %i, name %a",level,total,name or "?") + level = level - 1 + name = stack[#stack] +-- synctex.setfilename(stack[#stack] or tex.jobname) + if name then + synctex.setfilename(name) + end + end +end + +local function report_none() +end + +register("start_file",report_start) +register("stop_file", report_stop) + +directives.register("system.reportfiles", function(v) + if v then + register("start_file",report_start) + register("stop_file", report_stop) + else + register("start_file",report_none) + register("stop_file", report_none) + end +end) + +-- start_run doesn't work + +-- luatex.registerstartactions(function() +-- if environment.arguments.sandbox then +-- sandbox.enable() +-- end +-- end) + +local report = logs.reporter("csname overload") +local reported = { } + +callback.register("handle_overload", function(fatal,overload,csname,flags) + if not reported[csname] then + logs.newline() + local readstate = status.readstate + local filename = readstate.filename + local linenumber = readstate.linenumber + local flags = tokens.flags and tokens.flags(csname) or { } + if filename and linenumber then + report("%s, protection level %i, control sequence %a, properties '% t', file %a, line %i", + fatal and "fatal error" or "warning",overload,csname,flags,filename,linenumber) + else + report("%s, protection level %i, control sequence %a, properties '% t'", + fatal and "fatal error" or "warning",overload,csname,flags) + end + reported[csname] = true + logs.newline() + if fatal then + cleanup_run() + osexit(1) + end + end +end) + +-- bonus + +if environment.initex then + + luatex.registerdumpactions(statistics.showmemory) + +end diff --git a/tex/context/base/mkxl/node-fnt.lmt b/tex/context/base/mkxl/node-fnt.lmt index f3e8c212f..26e1fc343 100644 --- a/tex/context/base/mkxl/node-fnt.lmt +++ b/tex/context/base/mkxl/node-fnt.lmt @@ -252,7 +252,7 @@ do end end - function handlers.characters(head,groupcode,size,packtype,direction) + function handlers.characters(head,groupcode,direction) -- either next or not, but definitely no already processed list starttiming(nodes) diff --git a/tex/context/base/mkxl/node-pro.lmt b/tex/context/base/mkxl/node-pro.lmt index e84730b87..c360aea4d 100644 --- a/tex/context/base/mkxl/node-pro.lmt +++ b/tex/context/base/mkxl/node-pro.lmt @@ -65,86 +65,107 @@ do end -processors.enabled = true -- this will become a proper state (like trackers) +-- do +-- +-- local count_nodes = nodes.countall +-- local texget = tex.get +-- local tracer = processors.tracer +-- +-- local function pre_linebreak_filter(head,groupcode) +-- if trace_callbacks then +-- local before = count_nodes(head,true) +-- head = actions(head,groupcode) +-- local after = count_nodes(head,true) +-- tracer("pre_linebreak",head,groupcode,before,after,true) +-- else +-- head = actions(head,groupcode) +-- end +-- return head +-- end +-- +-- local function hpack_filter(head,groupcode,size,packtype,direction,attributes) +-- if not direction then +-- direction = texget("textdir") +-- end +-- -- +-- if trace_callbacks then +-- local before = count_nodes(head,true) +-- head = actions(head,groupcode,size,packtype,direction,attributes) +-- local after = count_nodes(head,true) +-- tracer("hpack",head,groupcode,before,after,true) +-- else +-- head = actions(head,groupcode,size,packtype,direction,attributes) +-- end +-- return head +-- end +-- +-- processors.pre_linebreak_filter = pre_linebreak_filter +-- processors.hpack_filter = hpack_filter +-- +-- do +-- +-- local hpack = nodes.hpack +-- +-- function nodes.fullhpack(head,...) +-- return hpack((hpack_filter(head)),...) +-- end +-- +-- end +-- +-- do +-- +-- local hpack = nuts.hpack +-- +-- function nuts.fullhpack(head,...) +-- return hpack(tonut(hpack_filter(tonode(head))),...) +-- end +-- +-- end +-- +-- callbacks.register('pre_linebreak_filter', pre_linebreak_filter, "horizontal manipulations (before par break)") +-- callbacks.register('hpack_filter' , hpack_filter, "horizontal manipulations (before hbox creation)") +-- +-- end do local count_nodes = nodes.countall - local texget = tex.get - local tracer = processors.tracer + local hbox_code = tex.groupcodes.hbox - -- We've set \hlistcallbackmode=1 so glyph checking happens at the other end! - - local function pre_linebreak_filter(head,groupcode) - -- local found = force_processors or hasglyph(head) - -- if found then - if trace_callbacks then - local before = count_nodes(head,true) - head = actions(head,groupcode) - local after = count_nodes(head,true) - tracer("pre_linebreak",head,groupcode,before,after,true) - else - head = actions(head,groupcode) - end - -- elseif trace_callbacks then - -- local n = count_nodes(head,false) - -- tracer("pre_linebreak",head,groupcode,n,n) - -- end - return head - end - - local function hpack_filter(head,groupcode,size,packtype,direction,attributes) - -- local found = force_processors or hasglyph(head) - -- if found then - -- - -- yes or no or maybe an option - -- - if not direction then - direction = texget("textdir") - end - -- - if trace_callbacks then - local before = count_nodes(head,true) - head = actions(head,groupcode,size,packtype,direction,attributes) - local after = count_nodes(head,true) - tracer("hpack",head,groupcode,before,after,true) - else - head = actions(head,groupcode,size,packtype,direction,attributes) - end - -- elseif trace_callbacks then - -- local n = count_nodes(head,false) - -- tracer("hpack",head,groupcode,n,n) - -- end + local function glyph_run(head,groupcode,direction) + if not groupcode then + groupcode = hbox_code + end + if trace_callbacks then + local before = count_nodes(head,true) + head = actions(head,groupcode,direction) + local after = count_nodes(head,true) + tracer("glyph_run",head,groupcode,before,after,true) + else + head = actions(head,groupcode) + end return head end - processors.pre_linebreak_filter = pre_linebreak_filter - processors.hpack_filter = hpack_filter + processors.glyph_run = glyph_run do - local hpack = nodes.hpack - function nodes.fullhpack(head,...) - return hpack((hpack_filter(head)),...) + return hpack((glyph_run(head)),...) end - end do - local hpack = nuts.hpack - function nuts.fullhpack(head,...) - return hpack(tonut(hpack_filter(tonode(head))),...) + return hpack(tonut(glyph_run(tonode(head))),...) end - end - callbacks.register('pre_linebreak_filter', pre_linebreak_filter, "horizontal manipulations (before par break)") - callbacks.register('hpack_filter' , hpack_filter, "horizontal manipulations (before hbox creation)") + callbacks.register("glyph_run", glyph_run, "glyph processing") end diff --git a/tex/context/base/mkxl/node-syn.lmt b/tex/context/base/mkxl/node-syn.lmt index 5b0a92a38..5cf59caad 100644 --- a/tex/context/base/mkxl/node-syn.lmt +++ b/tex/context/base/mkxl/node-syn.lmt @@ -128,7 +128,7 @@ if not modules then modules = { } end modules ['node-syn'] = { local type, rawset = type, rawset local concat = table.concat local formatters = string.formatters -local replacesuffix, suffixonly, nameonly = file.replacesuffix, file.suffix, file.nameonly +local replacesuffix, suffixonly, nameonly, collapsepath = file.replacesuffix, file.suffix, file.nameonly, file.collapsepath local openfile, renamefile, removefile = io.open, os.rename, os.remove local report_system = logs.reporter("system") @@ -231,6 +231,7 @@ local blockedsuffixes = { } local sttags = table.setmetatableindex(function(t,name) + name = collapsepath(name) if blockedsuffixes[suffixonly(name)] then -- Just so that I don't get the ones on my development tree. nofblocked = nofblocked + 1 diff --git a/tex/context/base/mkxl/node-tsk.lmt b/tex/context/base/mkxl/node-tsk.lmt index ca7c7fee4..264952102 100644 --- a/tex/context/base/mkxl/node-tsk.lmt +++ b/tex/context/base/mkxl/node-tsk.lmt @@ -428,7 +428,7 @@ local tonode = nodes.nuts.tonode %localize% -return function(head,groupcode,size,packtype,direction,attributes) +return function(head,groupcode,direction) local nuthead = tonut(head) %actions% @@ -437,24 +437,79 @@ end ]], step = [[ - nuthead = tonut((%action%(tonode(nuthead),groupcode,size,packtype,direction,attributes))) + nuthead = tonut((%action%(tonode(nuthead),groupcode,direction))) ]], nut = [[ - nuthead = %action%(nuthead,groupcode,size,packtype,direction,attributes) + nuthead = %action%(nuthead,groupcode,direction) ]], nohead = [[ - %action%(tonode(nuthead),groupcode,size,packtype,direction,attributes) + %action%(tonode(nuthead),groupcode,direction) ]], nonut = [[ - %action%(nuthead,groupcode,size,packtype,direction,attributes) + %action%(nuthead,groupcode,direction) ]], } } +-- -- hpackers -- -- + +-- tasks.new { +-- name = "processors", +-- processor = nodeprocessor, +-- sequence = { +-- "before", -- for users +-- "normalizers", +-- "characters", +-- "words", +-- "fonts", +-- "lists", +-- "after", -- for users +-- }, +-- templates = { +-- +-- default = [[ +-- return function(head) +-- return head +-- end +-- ]], +-- +-- process = [[ +-- local tonut = nodes.tonut +-- local tonode = nodes.nuts.tonode +-- +-- %localize% +-- +-- return function(head,groupcode,size,packtype,direction,attributes) +-- local nuthead = tonut(head) +-- +-- %actions% +-- return tonode(nuthead) +-- end +-- ]], +-- +-- step = [[ +-- nuthead = tonut((%action%(tonode(nuthead),groupcode,size,packtype,direction,attributes))) +-- ]], +-- +-- nut = [[ +-- nuthead = %action%(nuthead,groupcode,size,packtype,direction,attributes) +-- ]], +-- +-- nohead = [[ +-- %action%(tonode(nuthead),groupcode,size,packtype,direction,attributes) +-- ]], +-- +-- nonut = [[ +-- %action%(nuthead,groupcode,size,packtype,direction,attributes) +-- ]], +-- +-- } +-- } + tasks.new { name = "finalizers", processor = nodeprocessor, diff --git a/tex/context/base/mkxl/scrp-ini.lmt b/tex/context/base/mkxl/scrp-ini.lmt index 60ea15f08..9d8a5d3d5 100644 --- a/tex/context/base/mkxl/scrp-ini.lmt +++ b/tex/context/base/mkxl/scrp-ini.lmt @@ -57,9 +57,6 @@ local setglyphdata = nuts.setglyphdata local isglyph = nuts.isglyph -local insertnodeafter = nuts.insertafter -local insertnodebefore = nuts.insertbefore - local firstglyph = nuts.firstglyph local nextglyph = nuts.traversers.glyph @@ -88,6 +85,32 @@ scripts.injectors = handlers local splitters = allocate() scripts.splitters = splitters +local helpers = allocate() +scripts.helpers = helpers + +local insertnodebefore, insertnodeafter do + + local insertafter = nuts.insertnodeafter + local insertbefore = nuts.insertnodebefore + local setattributelist = nuts.setattributelist + + local function insertnodebefore(head,current,what) -- todo : lmtx + head, current = insertbefore(head,current,what) + setattributelist(what,current) + return head, current + end + + local function insertnodeafter(head,current,what) -- todo : lmtx + head, current = insertafter(head,current,what) + setattributelist(what,current) + return head, current + end + + helpers.insertnodebefore = insertnodebefore + helpers.insertnodeafter = insertnodeafter + +end + local hash = { -- we could put these presets in char-def.lua -- -- half width opening parenthesis diff --git a/tex/context/base/mkxl/syst-ini.mkxl b/tex/context/base/mkxl/syst-ini.mkxl index d4ffccb9b..8eaeea7db 100644 --- a/tex/context/base/mkxl/syst-ini.mkxl +++ b/tex/context/base/mkxl/syst-ini.mkxl @@ -1135,10 +1135,6 @@ % \mutable\let\par\par % \popoverloadmode -%D Also here: - -\listcallbackmode\plusone - %D Often used as cs key: \mutable\let\on \empty diff --git a/tex/context/base/mkxl/trac-inf.lmt b/tex/context/base/mkxl/trac-inf.lmt index ebcd21e07..9a4932fbc 100644 --- a/tex/context/base/mkxl/trac-inf.lmt +++ b/tex/context/base/mkxl/trac-inf.lmt @@ -210,8 +210,8 @@ function statistics.memused() -- no math.round yet -) local luastate = status.getluastate() return format("%s MB, ctx: %s MB, max: %s MB", round(collectgarbage("count")//1024), - round(luastate.state_bytes//1048576), - luastate.state_bytes_max and round(luastate.state_bytes_max//1048576) or "unknown" + round(luastate.statebytes//1048576), + luastate.statebytesmax and round(luastate.statebytesmax//1048576) or "unknown" ) end @@ -264,7 +264,51 @@ status.iocodes = setmetatableindex(tex.getiovalues(), function() return "unknown local report = logs.reporter("system") +local list = { + "string", "pool", "hash", "lookup", "node", "token", "extra", "sparse", "buffer", + "input", "file", "nest", "parameter", "save", "font", "language", "mark", +} + +local function show(data,fields) + local line = rep("-",3+11*#list) + local columns = rep("%11s",#list) + report("") + report("%w%s",2,line) + report("%w"..columns,5,unpack(list)) + report("%w%s",2,line) + for i=1,#fields do + local f = fields[i] + if f then + local t = { } + for i=1,#list do + local n = data[list[i].."state"][f] + t[i] = n < 0 and formatters["%w"](11) or formatters["%11i"](n) + end + report(" %3s"..columns,f,unpack(t)) + else + report("") + end + end + report("%w%s",2,line) + report("") +end + +function statistics.showmemory(when) + report("") + report("memory configuration") + show(status.list(), { "max", "min", "set", "stp" }) +end + +local registered = false +local enabled = false +local finished = false + function statistics.showusage(when) + if finished and when == "finish" then + return + else + finished = true + end local s = status.list() local c = status.getcallbackstate() -- status.callbacks local m = mplib.getcallbackstate() @@ -287,40 +331,7 @@ function statistics.showusage(when) else report("status after shipping out page %s",tex.getcount("realpageno")) end - report("") - local list = { - "stringstate", "poolstate", "hashstate", "lookupstate", - "nodestate", "extrastate", "tokenstate", - "bufferstate", "inputstate", "filestate", - "neststate", "parameterstate", "savestate", - "fontstate", "languagestate", "markstate", "sparsestate", - } - local fields = { "max", "min", "set", "stp", false, "mem", "all", false, "ini", "ptr", "top" } - local line = rep("-",190) - do - local t = { } - for i=1,#list do - t[i] = gsub(list[i],"state","") - end - report("%w%s",2,line) - report("%w%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s",5,unpack(t)) - report("%w%s",2,line) - end - for i=1,#fields do - local f = fields[i] - if f then - local t = { } - for i=1,#list do - local n = s[list[i]][f] - t[i] = n < 0 and formatters["%w"](11) or formatters["%11i"](n) - end - report(" %3s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s",f,unpack(t)) - else - report("") - end - end - report("%w%s",2,line) - report("") + show(s, { "max", "min", "set", "stp", false, "mem", "all", false, "ini", "ptr", "top" }) report(" current input type : %s", iocode) if iocode == "file" then report(" current file name : %s", rstatus.filename or "") @@ -332,9 +343,9 @@ function statistics.showusage(when) report(" expansion depth : min: %s, max: %s, set: %s, top: %s", estatus.min, estatus.max, estatus.set, estatus.top) report("") report(" luabytecode registers : %s", lstatus.bytecodes) - report(" luabytecode bytes : %s (%s MB)", lstatus.bytecode_bytes, lstatus.bytecode_bytes // 1048576) - report(" luastate bytes now : %s (%s MB)", lstatus.state_bytes, lstatus.state_bytes // 1048576) - report(" luastate bytes max : %s (%s MB)", lstatus.state_bytes_max, lstatus.state_bytes_max // 1048576) + report(" luabytecode bytes : %s (%s MB)", lstatus.bytecodebytes, lstatus.bytecodebytes // 1048576) + report(" luastate bytes now : %s (%s MB)", lstatus.statebytes, lstatus.statebytes // 1048576) + report(" luastate bytes max : %s (%s MB)", lstatus.statebytesmax, lstatus.statebytesmax // 1048576) report("") report(" file callbacks : %s", c.file) report(" saved callbacks : %s", c.saved) @@ -355,55 +366,13 @@ function statistics.showusage(when) report(" mp total callbacks : %s", m.count) report(" backend callbacks : %s", b.count) report("") - report(" page numbers : realpage %s, userpage %s, subpage %s",pstatus.page.real,pstatus.page.user,pstatus.page.sub) - report(" page timing : total %0.03f, page %0.03f, average %0.03f",pstatus.time.elapsed,pstatus.time.page,pstatus.time.average) - report("") -end - -function statistics.showmemory(when) - local s = status.list() - -- - report("") - report("memory configuration") - report("") - local list = { - "stringstate", "poolstate", "hashstate", "lookupstate", - "nodestate", "tokenstate", - "bufferstate", "inputstate", "filestate", - "neststate", "parameterstate", "savestate", - "fontstate", "languagestate", "markstate", - } - local fields = { "max", "min", "set", "stp" } - local line = rep("-",168) - do - local t = { } - for i=1,#list do - t[i] = gsub(list[i],"state","") - end - report("%w%s",2,line) - report("%w%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s",5,unpack(t)) - report("%w%s",2,line) - end - for i=1,#fields do - local f = fields[i] - if f then - local t = { } - for i=1,#list do - local n = s[list[i]][f] - t[i] = n < 0 and formatters["%w"](11) or formatters["%11i"](n) - end - report(" %3s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s%11s",f,unpack(t)) - else - report("") - end + if pstatus then + report(" page numbers : realpage %s, userpage %s, subpage %s",pstatus.page.real,pstatus.page.user,pstatus.page.sub) + report(" page timing : total %0.03f, page %0.03f, average %0.03f",pstatus.time.elapsed,pstatus.time.page,pstatus.time.average) + report("") end - report("%w%s",2,line) - report("") end -local registered = false -local enabled = false - trackers.register("system.usage", function(v) if v and not registered then logs.private.enablepagetiming() @@ -423,3 +392,13 @@ trackers.register("system.usage", function(v) end enabled = v end) + +-- can't be done here: +-- +-- luatex.registerstopactions(function() +-- if not enabled then +-- logs.push("logfile") +-- statistics.showusage("finish") +-- logs.pop() +-- end +-- end) diff --git a/tex/context/base/mkxl/trac-set.lmt b/tex/context/base/mkxl/trac-set.lmt new file mode 100644 index 000000000..5c21f6525 --- /dev/null +++ b/tex/context/base/mkxl/trac-set.lmt @@ -0,0 +1,453 @@ +if not modules then modules = { } end modules ['trac-set'] = { -- might become util-set.lua + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- maybe this should be util-set.lua + +local type, next, tostring, tonumber = type, next, tostring, tonumber +local print = print +local concat, sortedhash = table.concat, table.sortedhash +local formatters, find, lower, gsub, topattern = string.formatters, string.find, string.lower, string.gsub, string.topattern +local is_boolean = string.is_boolean +local settings_to_hash = utilities.parsers.settings_to_hash +local allocate = utilities.storage.allocate + +utilities = utilities or { } +local utilities = utilities + +local setters = utilities.setters or { } +utilities.setters = setters + +local data = { } + +-- We can initialize from the cnf file. This is sort of tricky as +-- later defined setters also need to be initialized then. If set +-- this way, we need to ensure that they are not reset later on. +-- +-- The sorting is needed to get a predictable setters in case of *. + +local trace_initialize = false -- only for testing during development +local frozen = true -- this needs checking + +local function initialize_setter(filename,name,values) -- filename only for diagnostics + local setter = data[name] + if setter then + -- trace_initialize = true + local data = setter.data + if data then + for key, newvalue in sortedhash(values) do + local newvalue = is_boolean(newvalue,newvalue,true) -- strict + local functions = data[key] + if functions then + local oldvalue = functions.value + if functions.frozen then + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"frozen",oldvalue) + end + elseif #functions > 0 and not oldvalue then +-- elseif #functions > 0 and oldvalue == nil then + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"set",newvalue) + end + for i=1,#functions do + functions[i](newvalue) + end + functions.value = newvalue + functions.frozen = functions.frozen or frozen + else + if trace_initialize then + setter.report("%s: %a is %s as %a",filename,key,"kept",oldvalue) + end + end + else + -- we do a simple preregistration i.e. not in the + -- list as it might be an obsolete entry + functions = { default = newvalue, frozen = frozen } + data[key] = functions + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"defaulted",newvalue) + end + end + end + return true + end + end +end + +-- user interface code + +local function set(t,what,newvalue) + local data = t.data -- somehow this can be nil + if data and not data.frozen then + local done = t.done + if type(what) == "string" then + what = settings_to_hash(what) -- inefficient but ok + end + if type(what) ~= "table" then + return + end + if not done then -- catch ... why not set? + done = { } + t.done = done + end + for w, value in sortedhash(what) do + if value == "" then + value = newvalue + elseif not value then + value = false -- catch nil + else + value = is_boolean(value,value,true) -- strict + end + w = topattern(w,true,true) + for name, functions in sortedhash(data) do + if done[name] then + -- prevent recursion due to wildcards + elseif find(name,w) then + done[name] = true + for i=1,#functions do + functions[i](value) + end + functions.value = value + end + end + end + end +end + +local function reset(t) + local data = t.data + if data and not data.frozen then + for name, functions in sortedthash(data) do + for i=1,#functions do + functions[i](false) + end + functions.value = false + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +local function register_setter(t,what,...) + local data = t.data + what = lower(what) + local functions = data[what] + if not functions then + functions = { } + data[what] = functions + if trace_initialize then + t.report("defining %a",what) + end + end + local default = functions.default -- can be set from cnf file + for i=1,select("#",...) do + local fnc = select(i,...) + local typ = type(fnc) + if typ == "string" then + if trace_initialize then + t.report("coupling %a to %a",what,fnc) + end + local s = fnc -- else wrong reference + fnc = function(value) set(t,s,value) end + elseif typ == "table" then + functions.values = fnc + fnc = nil + elseif typ ~= "function" then + fnc = nil + end + if fnc then + functions[#functions+1] = fnc + -- default: set at command line or in cnf file + -- value : set in tex run (needed when loading runtime) + local value = functions.value or default + if value ~= nil then + fnc(value) + functions.value = value + end + end + end + return false -- so we can use it in an assignment +end + +local function enable_setter(t,what) + local e = t.enable + t.enable, t.done = enable, { } + set(t,what,true) + enable(t,what) + t.enable, t.done = e, { } +end + +local function disable_setter(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,what) + t.disable, t.done = e, { } +end + +local function reset_setter(t) + t.done = { } + reset(t) +end + +local function list_setter(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +local function show_setter(t,pattern) + local list = list_setter(t) + t.report() + for k=1,#list do + local name = list[k] + if not pattern or find(name,pattern) then + local functions = t.data[name] + if functions then + local value = functions.value + local default = functions.default + local values = functions.values + local modules = #functions + if default == nil then + default = "unset" + elseif type(default) == "table" then + default = concat(default,"|") + else + default = tostring(default) + end + if value == nil then + value = "unset" + elseif type(value) == "table" then + value = concat(value,"|") + else + value = tostring(value) + end + t.report(name) + t.report(" modules : %i",modules) + t.report(" default : %s",default) + t.report(" value : %s",value) + if values then + local v = { } for i=1,#values do v[i] = tostring(values[i]) end + t.report(" values : % t",v) + end + t.report() + end + end + end +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singular tracker + +-- we could make this into a module but we also want the rest avaliable + +function setters.report(setter,fmt,...) + if fmt then + print(formatters["%-15s : %s"](setter.name,formatters[fmt](...))) + else + print("") + end +end + +local function setter_default(setter,name) + local d = setter.data[name] + return d and d.default +end + +local function setter_value(setter,name) + local d = setter.data[name] + return d and (d.value or d.default) +end + +local function setter_values(setter,name) + local d = setter.data[name] + return d and d.values +end + +local function new_setter(name) -- we could use foo:bar syntax (but not used that often) + local setter -- we need to access it in setter itself + setter = { + data = allocate(), -- indexed, but also default and value fields + name = name, + report = function(...) setters.report (setter,...) end, -- setters.report gets implemented later + enable = function(...) enable_setter (setter,...) end, + disable = function(...) disable_setter (setter,...) end, + reset = function(...) reset_setter (setter,...) end, -- can be dangerous + register = function(...) register_setter(setter,...) end, + list = function(...) return list_setter (setter,...) end, + show = function(...) show_setter (setter,...) end, + default = function(...) return setter_default (setter,...) end, + value = function(...) return setter_value (setter,...) end, + values = function(...) return setter_values (setter,...) end, + } + data[name] = setter + return setter +end + +setters.enable = enable_setter +setters.disable = disable_setter +-------.report = report_setter -- todo: adapt after call (defaults to print) +setters.register = register_setter +setters.list = list_setter +setters.show = show_setter +setters.reset = reset_setter +setters.new = new_setter +setters.initialize = initialize_setter + +trackers = new_setter("trackers") +directives = new_setter("directives") +experiments = new_setter("experiments") + +local t_enable, t_disable = trackers .enable, trackers .disable +local d_enable, d_disable = directives .enable, directives .disable +local e_enable, e_disable = experiments.enable, experiments.disable + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) +local trace_experiments = false local trace_experiments = false trackers.register("system.experiments", function(v) trace_experiments = v end) + +function directives.enable(...) + if trace_directives then + directives.report("enabling: % t",{...}) + end + d_enable(...) +end + +function directives.disable(...) + if trace_directives then + directives.report("disabling: % t",{...}) + end + d_disable(...) +end + +function experiments.enable(...) + if trace_experiments then + experiments.report("enabling: % t",{...}) + end + e_enable(...) +end + +function experiments.disable(...) + if trace_experiments then + experiments.report("disabling: % t",{...}) + end + e_disable(...) +end + +-- a useful example + +directives.register("system.nostatistics", function(v) + if statistics then + statistics.enable = not v + else + -- forget about it + end +end) + +directives.register("system.nolibraries", function(v) + if libraries then + libraries = nil -- we discard this tracing for security + else + -- no libraries defined + end +end) + +-- experiment + +if environment then + + -- The engineflags are known earlier than environment.arguments but maybe we + -- need to handle them both as the later are parsed differently. The c: prefix + -- is used by mtx-context to isolate the flags from those that concern luatex. + + local engineflags = environment.engineflags + + if engineflags then + local list = engineflags["c:trackers"] or engineflags["trackers"] + if type(list) == "string" then + initialize_setter("commandline flags","trackers",settings_to_hash(list)) + -- t_enable(list) + end + local list = engineflags["c:directives"] or engineflags["directives"] + if type(list) == "string" then + initialize_setter("commandline flags","directives", settings_to_hash(list)) + -- d_enable(list) + end + end + +end + +-- here + +if texconfig then + + -- this happens too late in ini mode but that is no problem + + local function set(k,v) + if v then + texconfig[k] = v + end + end + + directives.register("luametatex.memory.expand", function(v) set("expand_depth",v) end) + directives.register("luametatex.memory.hash", function(v) set("hash_extra",v) end) + directives.register("luametatex.memory.nest", function(v) set("nest_size",v) end) + directives.register("luametatex.memory.file", function(v) set("max_in_open",v) end) + directives.register("luametatex.memory.string", function(v) set("max_strings",v) end) + directives.register("luametatex.memory.parameter", function(v) set("param_size",v) end) + directives.register("luametatex.memory.save", function(v) set("save_size",v) end) + directives.register("luametatex.memory.stack", function(v) set("stack_size",v) end) + + -- poolstate + -- lookupstate + -- nodestate + -- tokenstate + -- bufferstate + -- fontstate + -- languagestate + -- markstate + -- sparsestate + +end + +-- for now here: + +local data = table.setmetatableindex("table") + +updaters = { + register = function(what,f) + local d = data[what] + d[#d+1] = f + end, + apply = function(what,...) + local d = data[what] + for i=1,#d do + d[i](...) + end + end, +} diff --git a/tex/context/base/mkxl/typo-dir.lmt b/tex/context/base/mkxl/typo-dir.lmt index 7e1d6e29c..c1d2a8601 100644 --- a/tex/context/base/mkxl/typo-dir.lmt +++ b/tex/context/base/mkxl/typo-dir.lmt @@ -164,7 +164,7 @@ local stoptiming = statistics.stoptiming -- -- \enabledirectives[typesetters.directions.onetoo] -function directions.handler(head,where,_,_,direction) +function directions.handler(head,where,direction) local only_one = not getnext(head) if only_one and not one_too then return head diff --git a/tex/context/base/mkxl/util-deb.lmt b/tex/context/base/mkxl/util-deb.lmt new file mode 100644 index 000000000..0021b93b9 --- /dev/null +++ b/tex/context/base/mkxl/util-deb.lmt @@ -0,0 +1,371 @@ +if not modules then modules = { } end modules ['util-deb'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local type, next, tostring, tonumber = type, next, tostring, tonumber +local format, find, sub, gsub = string.format, string.find, string.sub, string.gsub +local insert, remove, sort = table.insert, table.remove, table.sort +local setmetatableindex = table.setmetatableindex + +utilities = utilities or { } +local debugger = utilities.debugger or { } +utilities.debugger = debugger + +local report = logs.reporter("debugger") + +local ticks = os.gettimeofday or os.clock +local seconds = function(n) return n or 0 end +local overhead = 0 +local dummycalls = 10*1000 +local nesting = 0 +local names = { } + +local function initialize() + ticks = lua.getpreciseticks + seconds = lua.getpreciseseconds + initialize = false +end + +setmetatableindex(names,function(t,name) + local v = setmetatableindex(function(t,source) + local v = setmetatableindex(function(t,line) + -- local v = { total = 0, count = 0, nesting = 0, ticks = 0 } + local v = { 0, 0, 0, 0 } + t[line] = v + return v + end) + t[source] = v + return v + end) + t[name] = v + return v +end) + +local getinfo = nil +local sethook = nil + +-- local function hook(where) +-- local f = getinfo(2,"nS") +-- if f then +-- local source = f.short_src +-- if not source then +-- return +-- end +-- local line = f.linedefined or 0 +-- local name = f.name +-- if not name then +-- local what = f.what +-- if what == "C" then +-- name = "" +-- else +-- name = f.namewhat or what or "" +-- end +-- end +-- local data = names[name][source][line] +-- if where == "call" then +-- local nesting = data.nesting +-- if nesting == 0 then +-- data.count = data.count + 1 +-- insert(data,ticks()) +-- data.nesting = 1 +-- else +-- data.nesting = nesting + 1 +-- end +-- elseif where == "return" then +-- local nesting = data.nesting +-- if nesting == 1 then +-- local t = remove(data) +-- if t then +-- data.total = data.total + ticks() - t +-- end +-- data.nesting = 0 +-- else +-- data.nesting = nesting - 1 +-- end +-- end +-- end +-- end + +local getdebuginfo = lua.getdebuginfo + +-- local function hook(where) -- make two hooks +-- local name, source, line = getdebuginfo() +-- if name then +-- local data = names[name][source][line] +-- if where == "call" then +-- local nesting = data.nesting +-- if nesting == 0 then +-- data.count = data.count + 1 +-- -- insert(data,ticks()) +-- data.nesting = 1 +-- data.ticks = ticks() +-- else +-- data.nesting = nesting + 1 +-- end +-- elseif where == "return" then +-- local nesting = data.nesting +-- if nesting == 1 then +-- -- local t = remove(data) +-- local t = data.ticks +-- if t then +-- data.total = data.total + ticks() - t +-- end +-- data.nesting = 0 +-- elseif nesting > 0 then +-- data.nesting = nesting - 1 +-- end +-- elseif where == "tail call" then +-- local nesting = data.nesting +-- if nesting == 1 then +-- -- local t = remove(data) +-- local t = data.ticks +-- if t then +-- data.total = data.total + ticks() - t +-- end +-- data.nesting = 0 +-- elseif nesting > 0 then +-- data.nesting = nesting - 1 +-- end +-- end +-- end +-- end + +local function hook(where) -- make two hooks + local name, source, line = getdebuginfo() + if name then + local data = names[name][source][line] + if where == "call" then + local nesting = data[3] + if nesting == 0 then + data[2] = data[2] + 1 + data[3] = 1 + data[4] = ticks() + else + data[3] = nesting + 1 + end + -- elseif where == "return" then + else + local nesting = data[3] + if nesting == 1 then + local t = data[4] + if t then + data[1] = data[1] + ticks() - t + end + data[3] = 0 + elseif nesting > 0 then + data[3] = nesting - 1 + end + -- elseif where == "tail call" then + -- local nesting = data[3] + -- if nesting == 1 then + -- local t = data[4] + -- if t then + -- data[4] = data[4] + ticks() - t + -- end + -- data[3] = 0 + -- elseif nesting > 0 then + -- data[3] = nesting - 1 + -- end + end + end +end + +function debugger.showstats(printer,threshold) + local printer = printer or report + local calls = 0 + local functions = 0 + local dataset = { } + local length = 0 + local realtime = 0 + local totaltime = 0 + local threshold = threshold or 0 + for name, sources in next, names do + for source, lines in next, sources do + for line, data in next, lines do + -- local count = data.count + local count = data[2] + if count > threshold then + if #name > length then + length = #name + end + -- local total = data.total + local total = data[1] + local real = total + if real > 0 then + real = total - (count * overhead / dummycalls) + if real < 0 then + real = 0 + end + realtime = realtime + real + end + totaltime = totaltime + total + if line < 0 then + line = 0 + end + -- if name = "a" then + -- -- weird name + -- end + dataset[#dataset+1] = { real, total, count, name, source, line } + end + end + end + end + sort(dataset,function(a,b) + if a[1] == b[1] then + if a[2] == b[2] then + if a[3] == b[3] then + if a[4] == b[4] then + if a[5] == b[5] then + return a[6] < b[6] + else + return a[5] < b[5] + end + else + return a[4] < b[4] + end + else + return b[3] < a[3] + end + else + return b[2] < a[2] + end + else + return b[1] < a[1] + end + end) + if length > 50 then + length = 50 + end + local fmt = string.formatters["%4.9k s %3.3k %% %4.9k s %3.3k %% %8i # %-" .. length .. "s %4i %s"] + for i=1,#dataset do + local data = dataset[i] + local real = data[1] + local total = data[2] + local count = data[3] + local name = data[4] + local source = data[5] + local line = data[6] + calls = calls + count + functions = functions + 1 + name = gsub(name,"%s+"," ") + if #name > length then + name = sub(name,1,length) + end + printer(fmt(seconds(total),100*total/totaltime,seconds(real),100*real/realtime,count,name,line,source)) + end + printer("") + printer(format("functions : %i", functions)) + printer(format("calls : %i", calls)) + printer(format("overhead : %f", seconds(overhead/1000))) + + -- table.save("luatex-profile.lua",names) +end + +local function getdebug() + if sethook and getinfo then + return + end + if not debug then + local okay + okay, debug = pcall(require,"debug") + end + if type(debug) ~= "table" then + return + end + getinfo = debug.getinfo + sethook = debug.sethook + if type(getinfo) ~= "function" then + getinfo = nil + end + if type(sethook) ~= "function" then + sethook = nil + end +end + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str,"\n") end,threshold) + f:close() + end +end + +function debugger.enable() + getdebug() + if sethook and getinfo and nesting == 0 then + running = true + if initialize then + initialize() + end + sethook(hook,"cr") +-- sethook(hook_c,"c") +-- sethook(hook_r,"r") + local function dummy() end + local t = ticks() + for i=1,dummycalls do + dummy() + end + overhead = ticks() - t + end + if nesting > 0 then + nesting = nesting + 1 + end +end + +function debugger.disable() + if nesting > 0 then + nesting = nesting - 1 + end + if sethook and getinfo and nesting == 0 then + sethook() + end +end + +-- debugger.enable() +-- +-- print(math.sin(1*.5)) +-- print(math.sin(1*.5)) +-- print(math.sin(1*.5)) +-- print(math.sin(1*.5)) +-- print(math.sin(1*.5)) +-- +-- debugger.disable() +-- +-- print("") +-- debugger.showstats() +-- print("") +-- debugger.showstats(print,3) +-- +-- from the lua book: + +local function showtraceback(rep) -- from lua site / adapted + getdebug() + if getinfo then + local level = 2 -- we don't want this function to be reported + local reporter = rep or report + while true do + local info = getinfo(level, "Sl") + if not info then + break + elseif info.what == "C" then + reporter("%2i : %s",level-1,"C function") + else + reporter("%2i : %s : %s",level-1,info.short_src,info.currentline) + end + level = level + 1 + end + end +end + +debugger.showtraceback = showtraceback +-- debug.showtraceback = showtraceback + +-- showtraceback() diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 267c8361b..f3af37815 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 2021-05-15 22:41 +-- merge date : 2021-05-19 18:18 do -- begin closure to overcome local limits and interference @@ -20858,7 +20858,7 @@ local trace_defining=false registertracker("fonts.defining",function(v) trace_d local report_otf=logs.reporter("fonts","otf loading") local fonts=fonts local otf=fonts.handlers.otf -otf.version=3.114 +otf.version=3.115 otf.cache=containers.define("fonts","otl",otf.version,true) otf.svgcache=containers.define("fonts","svg",otf.version,true) otf.pngcache=containers.define("fonts","png",otf.version,true) @@ -25733,6 +25733,28 @@ local function checkkerns(lookup) end return kerned end +local strip_pairs=true +local compact_pairs=true +local compact_singles=true +local merge_pairs=true +local merge_singles=true +local merge_substitutions=true +local merge_alternates=true +local merge_multiples=true +local merge_ligatures=true +local merge_cursives=true +local merge_marks=true +directives.register("otf.strip.pairs",function(v) strip_pairs=v end) +directives.register("otf.compact.pairs",function(v) compact_pairs=v end) +directives.register("otf.compact.singles",function(v) compact_singles=v end) +directives.register("otf.merge.pairs",function(v) merge_pairs=v end) +directives.register("otf.merge.singles",function(v) merge_singles=v end) +directives.register("otf.merge.substitutions",function(v) merge_substitutions=v end) +directives.register("otf.merge.alternates",function(v) merge_alternates=v end) +directives.register("otf.merge.multiples",function(v) merge_multiples=v end) +directives.register("otf.merge.ligatures",function(v) merge_ligatures=v end) +directives.register("otf.merge.cursives",function(v) merge_cursives=v end) +directives.register("otf.merge.marks",function(v) merge_marks=v end) local function checkpairs(lookup) local steps=lookup.steps local nofsteps=lookup.nofsteps @@ -25746,7 +25768,7 @@ local function checkpairs(lookup) else local v=d2[1] if v==true then - elseif v and (v[1]~=0 or v[2]~=0 or v[4]~=0) then + elseif v and (v[1]~=0 or v[2]~=0 or v[3]~=0 or v[4]~=0) then return false end end @@ -25780,26 +25802,27 @@ local function checkpairs(lookup) end return kerned end -local compact_pairs=true -local compact_singles=true -local merge_pairs=true -local merge_singles=true -local merge_substitutions=true -local merge_alternates=true -local merge_multiples=true -local merge_ligatures=true -local merge_cursives=true -local merge_marks=true -directives.register("otf.compact.pairs",function(v) compact_pairs=v end) -directives.register("otf.compact.singles",function(v) compact_singles=v end) -directives.register("otf.merge.pairs",function(v) merge_pairs=v end) -directives.register("otf.merge.singles",function(v) merge_singles=v end) -directives.register("otf.merge.substitutions",function(v) merge_substitutions=v end) -directives.register("otf.merge.alternates",function(v) merge_alternates=v end) -directives.register("otf.merge.multiples",function(v) merge_multiples=v end) -directives.register("otf.merge.ligatures",function(v) merge_ligatures=v end) -directives.register("otf.merge.cursives",function(v) merge_cursives=v end) -directives.register("otf.merge.marks",function(v) merge_marks=v end) +local function strippairs(lookup) + local steps=lookup.steps + local nofsteps=lookup.nofsteps + local stripped=0 + for i=1,nofsteps do + local step=steps[i] + if step.format=="pair" then + local coverage=step.coverage + for g1,d1 in next,coverage do + for g2,d2 in next,d1 do + if d2[2] then + elseif d2[1]==true then + d1[g2]=nil + stripped=stripped+1 + end + end + end + end + end + return stripped +end function readers.compact(data) if not data or data.compacted then return @@ -25807,6 +25830,7 @@ function readers.compact(data) data.compacted=true end local resources=data.resources + local stripped=0 local merged=0 local kerned=0 local allsteps=0 @@ -25844,6 +25868,9 @@ function readers.compact(data) kerned=kerned+checkkerns(lookup) end elseif kind=="gpos_pair" then + if strip_pairs then + stripped=stripped+strippairs(lookup) + end if merge_pairs then merged=merged+mergesteps_2(lookup) end @@ -25884,6 +25911,9 @@ function readers.compact(data) compact("sequences") compact("sublookups") if trace_optimizations then + if stripped>0 then + report_optimizations("%i zero positions stripped before merging",stripped) + end if merged>0 then report_optimizations("%i steps of %i removed due to merging",merged,allsteps) end @@ -33728,6 +33758,7 @@ end if inkscape then local descriptions=tfmdata.descriptions local nofshapes=#svgshapes + local s_format=inkscapeformat("pdf") local f_svgfile=formatters["temp-otf-svg-shape-%i.svg"] local f_pdffile=formatters["temp-otf-svg-shape-%i.pdf"] local f_convert=formatters[new and "file-open:%s; export-%s:%s; export-do\n" or "%s --export-%s=%s\n"] @@ -33744,7 +33775,7 @@ end local svgfile=f_svgfile(index) local pdffile=f_pdffile(index) savedata(svgfile,data) - inkscape:write(f_convert(svgfile,inkscapeformat("pdf"),pdffile)) + inkscape:write(f_convert(svgfile,s_format,pdffile)) processed[index]=true nofdone=nofdone+1 if nofdone%25==0 then @@ -36132,7 +36163,7 @@ local afm=fonts.handlers.afm local pfb=fonts.handlers.pfb local hashes=fonts.hashes local identifiers=hashes.identifiers -local version=0.009 +local version=0.010 local shapescache=containers.define("fonts","shapes",version,true) local streamscache=containers.define("fonts","streams",version,true) local compact_streams=false -- cgit v1.2.3