2018-06-02 22:41:00

1 files changed, 63 insertions, 45 deletions

diff --git a/tex/context/base/mkiv/toks-ini.lua b/tex/context/base/mkiv/toks-ini.lua
index 481858971..dfdf854d0 100644
--- a/tex/context/base/mkiv/toks-ini.lua
+++ b/tex/context/base/mkiv/toks-ini.lua
@@ -18,41 +18,15 @@ local printtable = table.print
 local concat     = table.concat
 local format     = string.format
 
-if setinspector then
+if token.commands then
 
-    local istoken = token.is_token
-    local simple  = { letter = "letter", other_char = "other" }
+    local commands = token.commands()
 
-    local function astable(t)
-        if t and istoken(t) then
-            local cmdname = t.cmdname
-            local simple  = simple[cmdname]
-            if simple then
-                return {
-                    category   = simple,
-                    character  = utfchar(t.mode) or nil,
-                }
-            else
-                return {
-                    command    = t.command,
-                    id         = t.id,
-                    tok        = t.tok,
-                    csname     = t.csname,
-                    active     = t.active,
-                    expandable = t.expandable,
-                    protected  = t.protected,
-                    mode       = t.mode,
-                    index      = t.index,
-                    cmdname    = cmdname,
-                }
-            end
-        end
-    end
+    tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
 
-    tokens.istoken = istoken
-    tokens.astable = astable
+else
 
-    setinspector("token",function(v) if istoken(v) then printtable(astable(v),tostring(v)) return true end end)
+    tokens.commands = { }
 
 end
 
@@ -79,25 +53,26 @@ local get_macro       = token.get_macro
 local get_meaning     = token.get_meaning
 local get_cmdname     = token.get_cmdname
 local set_char        = token.set_char
+local set_lua         = token.set_lua
+
 local create_token    = token.create
+local new_token       = token.new
+local is_defined      = token.is_defined
+local is_token        = token.is_token
 
-if not set_char then -- for a while
-    local contextsprint = context.sprint
-    local ctxcatcodes   = catcodes.numbers.ctxcatcodes
-    set_char = function(n,u) contextsprint(ctxcatcodes,format("\\chardef\\%s=%s",n,u)) end
-end
+if not is_defined then
+
+    is_defined = function(name)
+        return get_cmdname(create_token(name)) ~= "undefined_cs"
+    end
 
-function tokens.defined(name)
-    return get_cmdname(create_token(name)) ~= "undefined_cs"
 end
 
--- set_macro = function(k,v,g)
---     if g == "global" then
---         context.setgvalue(k,v or '')
---     else
---         context.setvalue(k,v or '')
---     end
--- end
+tokens.new            = new_token
+tokens.create         = create_token
+tokens.istoken        = is_token
+tokens.isdefined      = is_defined
+tokens.defined        = is_defined
 
 local bits = {
     escape      = 0x00000001, -- 2^00
@@ -272,6 +247,7 @@ tokens.getters = { -- these don't expand
 tokens.setters = {
     macro = set_macro,
     char  = set_char,
+    lua   = set_lua,
     count = tex.setcount,
     dimen = tex.setdimen,
     skip  = tex.setglue,
@@ -301,3 +277,45 @@ tokens.setters = {
 --  /* unsave_tex_scanner(texstate); */
 --     return 1;
 -- }
+
+if setinspector then
+
+    local simple = { letter = "letter", other_char = "other" }
+
+    local function astable(t)
+        if t and is_token(t) then
+            local cmdname = t.cmdname
+            local simple  = simple[cmdname]
+            if simple then
+                return {
+                    category   = simple,
+                    character  = utfchar(t.mode) or nil,
+                }
+            else
+                return {
+                    command    = t.command,
+                    id         = t.id,
+                    tok        = t.tok,
+                    csname     = t.csname,
+                    active     = t.active,
+                    expandable = t.expandable,
+                    protected  = t.protected,
+                    mode       = t.mode,
+                    index      = t.index,
+                    cmdname    = cmdname,
+                }
+            end
+        end
+    end
+
+    tokens.astable = astable
+
+    setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)
+
+end
+
+tokens.cache = table.setmetatableindex(function(t,k)
+    local v = create_token(k)
+    t[k] = v
+    return v
+end)