diff options
Diffstat (limited to 'tex/context/base/mkiv/cldf-ini.lua')
| -rw-r--r-- | tex/context/base/mkiv/cldf-ini.lua | 471 |
1 files changed, 345 insertions, 126 deletions
diff --git a/tex/context/base/mkiv/cldf-ini.lua b/tex/context/base/mkiv/cldf-ini.lua index f3bd0f563..71529f96c 100644 --- a/tex/context/base/mkiv/cldf-ini.lua +++ b/tex/context/base/mkiv/cldf-ini.lua @@ -52,6 +52,11 @@ if not modules then modules = { } end modules ['cldf-ini'] = { -- watching Trio Hiromi Uehara, Anthony Jackson & Simon Phillips (discovered via the -- later on YT). Hopefully the video breaks made the following better in the end. +-- This module is also a test bed for experimental features so the content changes over +-- time (for the better or worse). There have been no fundamental changes for many years +-- and performance has not changed much either. + + local format, stripstring = string.format, string.strip local next, type, tostring, tonumber, unpack, select, rawset = next, type, tostring, tonumber, unpack, select, rawset local insert, remove, concat = table.insert, table.remove, table.concat @@ -84,6 +89,12 @@ local texgetcount = tex.getcount local isnode = node.is_node local writenode = node.write local copynodelist = node.copy_list +local tonut = node.direct.todirect + +local istoken = token.is_token +local newtoken = token.new +local createtoken = token.create +local setluatoken = token.set_lua local catcodenumbers = catcodes.numbers @@ -104,16 +115,46 @@ local report_cld = logs.reporter("cld","stack") local processlines = true -- experiments.register("context.processlines", function(v) processlines = v end) +local tokenflushmode = false +local nodeflushmode = false +local scannerdefmode = false +local maxflushnodeindex = 0x10FFFF - 1 + +if LUATEXFUNCTIONALITY and LUATEXFUNCTIONALITY > 6780 then + + -- The gain in performasnce is neglectable. + + tokenflushmode = true + nodeflushmode = true + scannerdefmode = true + maxflushnodeindex = token.biggest_char() - 1 + +end + +-- tokenflushmode = false +-- scannerdefmode = false +-- nodeflushmode = false + -- In earlier experiments a function tables was referred to as lua.calls and the -- primitive \luafunctions was \luacall and we used our own implementation of -- a function table (more indirectness). +local trialtypesettingstate = 0 + +function context.trialtypesetting() + return texgetcount(trialtypesettingstate) ~= 0 +end + +function context.registertrialtypesetting(name) + trialtypesettingstate = createtoken(name).index +end + local knownfunctions = lua.get_functions_table() local showstackusage = false trackers.register("context.stack",function(v) showstackusage = v end) -local freed, nofused, noffreed = { }, 0, 0 -- maybe use the number of @@trialtypesetting +local freed, nofused, noffreed = { }, 0, 0 local usedstack = function() return nofused, noffreed @@ -122,7 +163,7 @@ end local flushfunction = function(slot,arg) if arg() then -- keep - elseif texgetcount("@@trialtypesetting") == 0 then -- @@trialtypesetting is private! + elseif texgetcount(trialtypesettingstate) == 0 then noffreed = noffreed + 1 freed[noffreed] = slot knownfunctions[slot] = false @@ -147,7 +188,7 @@ local storefunction = function(arg) end local flushnode = function(slot,arg) - if texgetcount("@@trialtypesetting") == 0 then -- @@trialtypesetting is private! + if texgetcount(trialtypesettingstate) == 0 then writenode(arg) noffreed = noffreed + 1 freed[noffreed] = slot @@ -182,9 +223,13 @@ local f_resolve = nil local p_resolve = ((1-lpegP("."))^1 / function(s) f_resolve = f_resolve[s] end * lpegP(".")^0)^1 local function resolvestoredfunction(str) - f_resolve = global - lpegmatch(p_resolve,str) - return f_resolve + if type(str) == "string" then + f_resolve = global + lpegmatch(p_resolve,str) + return f_resolve + else + return str + end end local function expose(slot,f,...) -- so we can register yet undefined functions @@ -246,17 +291,17 @@ local registerfunction = function(f,direct) -- either f=code or f=namespace,dire func = resolvestoredfunction(f) end if type(func) ~= "function" then - func = function() report_cld("invalid resolve %A",f) end + func = function() report_cld("invalid resolve %A, case %s",f,1) end end elseif type(f) == "string" then func = loadstring(f) if type(func) ~= "function" then - func = function() report_cld("invalid code %A",f) end + func = function() report_cld("invalid code %A, case %s",f,2) end end elseif type(f) == "function" then func = f else - func = function() report_cld("invalid function %A",f) end + func = function() report_cld("invalid function %A, case %s",f,3) end end knownfunctions[slot] = func return slot @@ -305,53 +350,52 @@ local storedscanners = interfaces.storedscanners storage.register("interfaces/storedscanners", storedscanners, "interfaces.storedscanners") local interfacescanners = setmetatablenewindex(function(t,k,v) + rawset(t,k,v) if storedscanners[k] then - -- report_cld("warning: scanner %a is already set",k) + -- report_cld("warning: scanner %a is already set (mode 1a)",k) -- os.exit() -- \scan_<k> is already in the format -- report_cld("using interface scanner: %s",k) + elseif scannerdefmode then + -- report_cld("installing interface scanner: %s (mode 1b)",k) + -- local n = registerfunction(interfaces.scanners[k],true) + local n = registerfunction("interfaces.scanners."..k,true) + storedscanners[k] = n + local name = "clf_" .. k + setluatoken(name,n,"global") -- todo : protected and "protected" or "" else - -- todo: allocate slot here and pass it + -- report_cld("installing interface scanner: %s (mode 1c)",k) storedscanners[k] = true - -- report_cld("installing interface scanner: %s",k) context("\\installctxscanner{clf_%s}{interfaces.scanners.%s}",k,k) end - rawset(t,k,v) + -- rawset(t,k,v) end) function interfaces.registerscanner(name,action,protected,public,call) + rawset(interfacescanners,name,action) if storedscanners[name] then - -- report_cld("warning: scanner %a is already set",k) + -- report_cld("warning: scanner %a is already set (mode 2a)",name) -- os.exit() -- \scan_<k> is already in the format -- report_cld("using interface scanner: %s",k) + elseif scannerdefmode then + -- report_cld("installing interface scanner: %s (mode 2b)",name) + -- local n = registerfunction(action,true) -- todo + local n = registerfunction("interfaces.scanners."..name,true) + storedscanners[name] = n + local name = public and name or ("clf_" .. name) + setluatoken(name,n,"global",protected and "protected" or "") else storedscanners[name] = true --- if protected then --- -- report_cld("installing expandable interface scanner: %s",k) --- if public then --- context("\\installprotectedctxscanner{%s}{interfaces.scanners.%s}",name,name) --- else --- context("\\installprotectedctxscanner{clf_%s}{interfaces.scanners.%s}",name,name) --- end --- else --- -- report_cld("installing protected interface scanner: %s",k) --- if public then --- context("\\installctxscanner{%s}{interfaces.scanners.%s}",name,name) --- else --- context("\\installctxscanner{clf_%s}{interfaces.scanners.%s}",name,name) --- end --- end - -- report_cld("installing interface scanner: %s",k) - context("\\install%sctxscanner%s{%s%s}{interfaces.scanners.%s}", - protected and "protected" or "", - call and "call" or "", - public and "" or "clf_", - name, - name - ) - end - rawset(interfacescanners,name,action) + -- report_cld("installing interface scanner: %s (mode 2c)",name) + context("\\install%sctxscanner{%s%s}{interfaces.scanners.%s}", + protected and "protected" or "", + public and "" or "clf_", + name, + name + ) + end + -- rawset(interfacescanners,name,action) end interfaces.scanners = storage.mark(interfacescanners) @@ -379,7 +423,7 @@ end local function dummy() end -function commands.ctxresetter(name) +function commands.ctxresetter(name) -- to be checked return function() if storedscanners[name] then rawset(interfacescanners,name,dummy) @@ -388,10 +432,6 @@ function commands.ctxresetter(name) end end -function context.trialtypesetting() - return texgetcount("@@trialtypesetting") ~= 0 -end - -- Should we keep the catcodes with the function? local catcodestack = { } @@ -683,8 +723,17 @@ local s_cldl_argument_e = "}" -- local s_cldl_argument_b = "{" -- local s_cldl_argument_f = "{ " +local t_cldl_luafunction = createtoken("luafunctioncall") +local lua_expandable_call_code = token.command_id and token.command_id("lua_expandable_call") + local function writer(parent,command,...) -- already optimized before call - flush(currentcatcodes,command) -- todo: ctx|prt|texcatcodes + + if type(command) == "string" then -- for now + flush(currentcatcodes,command) -- todo: ctx|prt|texcatcodes + else + flush(command) -- todo: ctx|prt|texcatcodes + end + local direct = false -- local t = { ... } -- for i=1,#t do @@ -751,7 +800,16 @@ local function writer(parent,command,...) -- already optimized before call elseif tn == 1 then -- some 20% faster than the next loop local tj = ti[1] if type(tj) == "function" then - flush(currentcatcodes,s_cldl_option_b,storefunction(tj),s_cldl_option_e) + tj = storefunction(tj) + if tokenflushmode then + if newtoken then + flush(currentcatcodes,"[",newtoken(tj,lua_expandable_call_code),"]") + else + flush(currentcatcodes,"[",t_cldl_luafunction,tj,"]") + end + else + flush(currentcatcodes,s_cldl_option_b,tj,s_cldl_option_e) + end else flush(currentcatcodes,"[",tj,"]") end @@ -760,10 +818,15 @@ local function writer(parent,command,...) -- already optimized before call for j=1,tn do local tj = ti[j] if type(tj) == "function" then - if j == tn then - flush(currentcatcodes,s_cldl_option_s,storefunction(tj),"]") + tj = storefunction(tj) + if tokenflushmode then + if newtoken then + flush(currentcatcodes,"[",newtoken(tj,lua_expandable_call_code),j == tn and "]" or ",") + else + flush(currentcatcodes,"[",t_cldl_luafunction,tj,j == tn and "]" or ",") + end else - flush(currentcatcodes,s_cldl_option_s,storefunction(tj),",") + flush(currentcatcodes,s_cldl_option_s,tj,j == tn and "]" or ",") end else if j == tn then @@ -776,7 +839,16 @@ local function writer(parent,command,...) -- already optimized before call end elseif typ == "function" then -- todo: ctx|prt|texcatcodes - flush(currentcatcodes,s_cldl_argument_f,storefunction(ti),s_cldl_argument_e) + ti = storefunction(ti) + if tokenflushmode then + if newtoken then + flush(currentcatcodes,"{",newtoken(ti,lua_expandable_call_code),"}") + else + flush(currentcatcodes,"{",t_cldl_luafunction,ti,"}") + end + else + flush(currentcatcodes,s_cldl_argument_f,ti,s_cldl_argument_e) + end elseif typ == "boolean" then if ti then flushdirect(currentcatcodes,"\r") @@ -785,62 +857,99 @@ local function writer(parent,command,...) -- already optimized before call end elseif typ == "thread" then report_context("coroutines not supported as we cannot yield across boundaries") - elseif isnode(ti) then -- slow - flush(currentcatcodes,s_cldl_argument_b,storenode(ti),s_cldl_argument_e) + elseif isnode(ti) then -- slow | why {} here ? + if nodeflushmode then + local n = tonut(ti) + if n <= maxflushnodeindex then + flush(currentcatcodes,"{",ti,"}") + else + flush(currentcatcodes,s_cldl_argument_b,storenode(ti),s_cldl_argument_e) + end + else + flush(currentcatcodes,s_cldl_argument_b,storenode(ti),s_cldl_argument_e) + end else report_context("error: %a gets a weird argument %a",command,ti) end +-- else +-- local n = isnode(ti) +-- if n then +-- if nodeflushmode and n <= maxflushnodeindex then +-- flush(ti) +-- else +-- flush(currentcatcodes,s_cldl_argument_b,storenode(ti),s_cldl_argument_e) +-- end +-- else +-- report_context("error: %a gets a weird argument %a",command,ti) +-- end +-- end end end end --- if performance really matters we can consider a compiler but it will never --- pay off - --- local function prtwriter(command,...) -- already optimized before call --- flush(prtcatcodes,command) --- for i=1,select("#",...) do --- local ti = select(i,...) --- if ti == nil then --- -- nothing --- elseif ti == "" then --- flush(prtcatcodes,"{}") --- else --- local tp = type(ti) --- if tp == "string" or tp == "number"then --- flush(prtcatcodes,"{",ti,"}") --- elseif tp == "function" then --- flush(prtcatcodes,s_cldl_argument_f,storefunction(ti),s_cldl_argument_e) --- elseif isnode(ti) then --- flush(prtcatcodes,s_cldl_argument_b,storenode(ti),s_cldl_argument_e) --- else --- report_context("fatal error: prt %a gets a weird argument %a",command,ti) --- end --- end --- end --- end +local core + +if tokenflushmode then -- combine them + + local toks = tokens.cache + + context.tokenizedcs = toks -local core = setmetatableindex(function(parent,k) - local c = "\\" .. k -- tostring(k) - local f = function(first,...) - if first == nil then + core = setmetatableindex(function(parent,k) + local t + local f = function(first,...) + if not t then + t = toks[k] + end + if first == nil then + flush(t) + else + return writer(context,t,first,...) + end + end + parent[k] = f + return f + end) + + core.cs = setmetatableindex(function(parent,k) + local t + local f = function() + if not t then + t = toks[k] + end + flush(t) + end + parent[k] = f + return f + end) + +else + + context.tokenizedcs = false + + core = setmetatableindex(function(parent,k) + local c = "\\" .. k + local f = function(first,...) + if first == nil then + flush(currentcatcodes,c) + else + return writer(context,c,first,...) + end + end + parent[k] = f + return f + end) + + core.cs = setmetatableindex(function(parent,k) + local c = "\\" .. k -- tostring(k) + local f = function() flush(currentcatcodes,c) - else - return writer(context,c,first,...) end - end - parent[k] = f - return f -end) + parent[k] = f + return f + end) -core.cs = setmetatableindex(function(parent,k) - local c = "\\" .. k -- tostring(k) - local f = function() - flush(currentcatcodes,c) - end - parent[k] = f - return f -end) +end local indexer = function(parent,k) if type(k) == "string" then @@ -935,8 +1044,16 @@ local caller = function(parent,f,a,...) end elseif typ == "function" then -- ignored: a ... - flush(currentcatcodes,"{\\cldl",storefunction(f),"}") -- todo: ctx|prt|texcatcodes - -- flush(currentcatcodes,"{",storefunction(f),"}") -- todo: ctx|prt|texcatcodes + f = storefunction(f) + if tokenflushmode then + if newtoken then + flush(currentcatcodes,"{",newtoken(f,lua_expandable_call_code),"}") + else + flush(currentcatcodes,"{",t_cldl_luafunction,f,"}") + end + else + flush(currentcatcodes,s_cldl_argument_b,f,s_cldl_argument_e) -- todo: ctx|prt|texcatcodes + end elseif typ == "boolean" then if f then if a ~= nil then @@ -955,20 +1072,46 @@ local caller = function(parent,f,a,...) elseif typ == "thread" then report_context("coroutines not supported as we cannot yield across boundaries") elseif isnode(f) then -- slow - -- writenode(f) - flush(currentcatcodes,"\\cldl",storenode(f)," ") - -- flush(currentcatcodes,"",storenode(f)," ") + if nodeflushmode then + local n = tonut(f) + if n <= maxflushnodeindex then + flush(f) + else + flush(currentcatcodes,s_cldl_option_s,storenode(f)," ") + end + else + flush(currentcatcodes,s_cldl_option_s,storenode(f)," ") + end else report_context("error: %a gets a weird argument %a","context",f) end +-- else +-- local n = isnode(f) +-- if n then +-- if nodeflushmode and n <= maxflushnodeindex then +-- flush(f) +-- else +-- flush(currentcatcodes,s_cldl_option_s,storenode(f)," ") +-- end +-- else +-- report_context("error: %a gets a weird argument %a","context",f) +-- end +-- end end end -context.nodes = { +context.nodes = { -- todo store = storenode, flush = function(n) - flush(currentcatcodes,"\\cldl",storenode(n)," ") - -- flush(currentcatcodes,"",storenode(n)," ") + if nodeflushmode then + if tonut(n) <= maxflushnodeindex then + flush(n) + else + flush(currentcatcodes,s_cldl_option_s,storenode(n)," ") + end + else + flush(currentcatcodes,s_cldl_option_s,storenode(n)," ") + end end, } @@ -1014,8 +1157,8 @@ local nofflushes = 0 local tracingpermitted = true local visualizer = lpeg.replacer { - { "\n","<<newline>>" }, - { "\r","<<par>>" }, + { "\n", "<<newline>>" }, + { "\r", "<<par>>" }, } statistics.register("traced context", function() @@ -1028,13 +1171,47 @@ statistics.register("traced context", function() end end) +local function userdata(argument) + if isnode(argument) then + return formatters["<< %s node %i>>"](nodes.nodecodes[argument.id],tonut(argument)) + end + if istoken(argument) then + local csname = argument.csname + if csname then + -- return formatters["<<\\%s>>"](csname) + return formatters["\\%s"](csname) + end + local cmdname = argument.cmdname + if cmdname == "lua_expandable_call" or cmdname == "lua_call" then + return "<<function>>" -- argument.mode + end + return "<<token>>" + end + return "<<userdata>>" +end + + local tracedwriter = function(parent,...) -- also catcodes ? nofwriters = nofwriters + 1 local savedflush = flush local savedflushdirect = flushdirect -- unlikely to be used here local t, n = { "w : - : " }, 1 local traced = function(catcodes,...) -- todo: check for catcodes - local s = concat({...}) + local s = type(catcodes) == "number" and { ... } or { catcodes, ... } + for i=1,#s do + local argument = s[i] + local argtype = type(argument) + if argtype == "string" then + s[i] = lpegmatch(visualizer,argument) + elseif argtype == "number" then + s[i] = argument + elseif argtype == "userdata" then + s[i] = userdata(argument) + else + s[i] = formatters["<<%S>>"](argument) + end + end + s = concat(s) s = lpegmatch(visualizer,s) n = n + 1 t[n] = s @@ -1073,6 +1250,8 @@ local traced = function(one,two,...) collapsed[c] = lpegmatch(visualizer,argument) elseif argtype == "number" then collapsed[c] = argument + elseif argtype == "userdata" then + collapsed[c] = userdata(argument) else collapsed[c] = formatters["<<%S>>"](argument) end @@ -1085,6 +1264,8 @@ local traced = function(one,two,...) currenttrace(formatters["f : - : %s"](lpegmatch(visualizer,one))) elseif argtype == "number" then currenttrace(formatters["f : - : %s"](one)) + elseif argtype == "userdata" then + currenttrace(formatters["f : - : %s"](userdata(one))) else currenttrace(formatters["f : - : <<%S>>"](one)) end @@ -1367,14 +1548,22 @@ function context.newindexer(catcodes,cmdcodes) contentcatcodes = savedcatcodes end - handler.cs = setmetatableindex(function(parent,k) - local c = "\\" .. k -- tostring(k) - local f = function() - flush(cmdcodes,c) - end - parent[k] = f - return f - end) + if tokenflushmode then + + handler.cs = core.cs + + else + + handler.cs = setmetatableindex(function(parent,k) + local c = "\\" .. k -- tostring(k) + local f = function() + flush(cmdcodes,c) + end + parent[k] = f + return f + end) + + end setmetatableindex(handler,indexer) setmetatablecall (handler,caller) @@ -1427,21 +1616,51 @@ do end end - local function indexer(parent,k) - if type(k) == "string" then - local c = "\\" .. k - local f = function(first,...) - if first == nil then - flush(currentcatcodes,c) - else - return formattedflush(parent,c,first,...) + local indexer + + if tokenflushmode then -- combine them + + local toks = tokens.cache + + indexer = function(parent,k) + if type(k) == "string" then + local t + local f = function(first,...) + if not t then + t = toks[k] + end + if first == nil then + flush(t) + else + return formattedflush(parent,t,first,...) + end end + parent[k] = f + return f + else + return context -- catch end - parent[k] = f - return f - else - return context -- catch end + + else + + indexer = function(parent,k) + if type(k) == "string" then + local c = "\\" .. k + local f = function(first,...) + if first == nil then + flush(currentcatcodes,c) + else + return formattedflush(parent,c,first,...) + end + end + parent[k] = f + return f + else + return context -- catch + end + end + end -- formatted([catcodes,]format[,...]) |