From 13ec4b540e0d46c97fd7b089e0b7413da81e0a9f Mon Sep 17 00:00:00 2001
From: Marius <mariausol@gmail.com>
Date: Sun, 19 May 2013 20:40:34 +0300
Subject: beta 2013.05.19 19:27

---
 tex/context/base/toks-ini.lua | 682 +++++++++++++++++++++---------------------
 1 file changed, 341 insertions(+), 341 deletions(-)

(limited to 'tex/context/base/toks-ini.lua')

diff --git a/tex/context/base/toks-ini.lua b/tex/context/base/toks-ini.lua
index ef4b5406b..0136f274f 100644
--- a/tex/context/base/toks-ini.lua
+++ b/tex/context/base/toks-ini.lua
@@ -1,341 +1,341 @@
-if not modules then modules = { } end modules ['toks-ini'] = {
-    version   = 1.001,
-    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
-    copyright = "PRAGMA ADE / ConTeXt Development Team",
-    license   = "see context related readme files"
-}
-
-local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values
-local format, gsub = string.format, string.gsub
-
---[[ldx--
-<p>This code is experimental and needs a cleanup. The visualizers will move to
-a module.</p>
---ldx]]--
-
--- 1 = command, 2 = modifier (char), 3 = controlsequence id
---
--- callback.register('token_filter', token.get_next)
---
--- token.get_next()
--- token.expand()
--- token.create()
--- token.csname_id()
--- token.csname_name(v)
--- token.command_id()
--- token.command_name(v)
--- token.is_expandable()
--- token.is_activechar()
--- token.lookup(v)
-
--- actually, we can use token registers to store tokens
-
-local token, tex = token, tex
-
-local createtoken   = token.create
-local csname_id     = token.csname_id
-local command_id    = token.command_id
-local command_name  = token.command_name
-local get_next      = token.get_next
-local expand        = token.expand
-local is_activechar = token.is_activechar
-local csname_name   = token.csname_name
-
-tokens        = tokens or { }
-local tokens  = tokens
-
-tokens.vbox   = createtoken("vbox")
-tokens.hbox   = createtoken("hbox")
-tokens.vtop   = createtoken("vtop")
-tokens.bgroup = createtoken(utfbyte("{"), 1)
-tokens.egroup = createtoken(utfbyte("}"), 2)
-
-tokens.letter = function(chr) return createtoken(utfbyte(chr), 11) end
-tokens.other  = function(chr) return createtoken(utfbyte(chr), 12) end
-
-tokens.letters = function(str)
-    local t, n = { }, 0
-    for chr in utfvalues(str) do
-        n = n + 1
-        t[n] = createtoken(chr, 11)
-    end
-    return t
-end
-
-tokens.collectors     = tokens.collectors or { }
-local collectors      = tokens.collectors
-
-collectors.data       = collectors.data or { }
-local collectordata   = collectors.data
-
-collectors.registered = collectors.registered or { }
-local registered      = collectors.registered
-
-local function printlist(data)
-    callbacks.push('token_filter', function ()
-       callbacks.pop('token_filter') -- tricky but the nil assignment helps
-       return data
-    end)
-end
-
-tex.printlist = printlist -- will change to another namespace
-
-function collectors.flush(tag)
-    printlist(collectordata[tag])
-end
-
-function collectors.test(tag)
-    printlist(collectordata[tag])
-end
-
-function collectors.register(name)
-    registered[csname_id(name)] = name
-end
-
-local call   = command_id("call")
-local letter = command_id("letter")
-local other  = command_id("other_char")
-
-function collectors.install(tag,end_cs)
-    local data, d = { }, 0
-    collectordata[tag] = data
-    local endcs = csname_id(end_cs)
-    while true do
-        local t = get_next()
-        local a, b = t[1], t[3]
-        if b == endcs then
-            context["end_cs"]()
-            return
-        elseif a == call and registered[b] then
-            expand()
-        else
-            d = d + 1
-            data[d] = t
-        end
-    end
-end
-
-function collectors.handle(tag,handle,flush)
-    collectordata[tag] = handle(collectordata[tag])
-    if flush then
-        collectors.flush(tag)
-    end
-end
-
-local show_methods      = { }
-collectors.show_methods = show_methods
-
-function collectors.show(tag, method)
-    if type(tag) == "table" then
-        show_methods[method or 'a'](tag)
-    else
-        show_methods[method or 'a'](collectordata[tag])
-    end
-end
-
-function collectors.defaultwords(t,str)
-    local n = #t
-    n = n + 1
-    t[n] = tokens.bgroup
-    n = n + 1
-    t[n] = createtoken("red")
-    for i=1,#str do
-        n = n + 1
-        t[n] = tokens.other('*')
-    end
-    n = n + 1
-    t[n] = tokens.egroup
-end
-
-function collectors.dowithwords(tag,handle)
-    local t, w, tn, wn = { }, { }, 0, 0
-    handle = handle or collectors.defaultwords
-    local tagdata = collectordata[tag]
-    for k=1,#tagdata do
-        local v = tagdata[k]
-        if v[1] == letter then
-            wn = wn + 1
-            w[wn] = v[2]
-        else
-            if wn > 0 then
-                handle(t,w)
-                wn = 0
-            end
-            tn = tn + 1
-            t[tn] = v
-        end
-    end
-    if wn > 0 then
-        handle(t,w)
-    end
-    collectordata[tag] = t
-end
-
-local function showtoken(t)
-    if t then
-        local cmd, chr, id, cs, name = t[1], t[2], t[3], nil, command_name(t) or ""
-        if cmd == letter or cmd == other then
-            return format("%s-> %s -> %s", name, chr, utfchar(chr))
-        elseif id > 0 then
-            cs = csname_name(t) or nil
-            if cs then
-                return format("%s-> %s", name, cs)
-            elseif tonumber(chr) < 0 then
-                return format("%s-> %s", name, id)
-            else
-                return format("%s-> (%s,%s)", name, chr, id)
-            end
-        else
-            return format("%s", name)
-        end
-    else
-        return "no node"
-    end
-end
-
-collectors.showtoken = showtoken
-
-function collectors.trace()
-    local t = get_next()
-    logs.report("tokenlist",showtoken(t))
-    return t
-end
-
--- these might move to a runtime module
-
-show_methods.a = function(data) -- no need to store the table, just pass directly
-    local function row(one,two,three,four,five)
-        context.NC() context(one)
-        context.NC() context(two)
-        context.NC() context(three)
-        context.NC() context(four)
-        context.NC() context(five)
-        context.NC() context.NR()
-    end
-    context.starttabulate { "|T|Tr|cT|Tr|T|" }
-    row("cmd","chr","","id","name")
-    context.HL()
-    for _,v in next, data do
-        local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", ""
-        local name = gsub(command_name(v) or "","_","\\_")
-        if id > 0 then
-            cs = csname_name(v) or ""
-            if cs ~= "" then cs = "\\string " .. cs end
-        else
-            id = ""
-        end
-        if cmd == letter or cmd == other then
-            sym = "\\char " .. chr
-        end
-        if tonumber(chr) < 0 then
-            row(name,"",sym,id,cs)
-        else
-            row(name,chr,sym,id,cs)
-        end
-    end
-    context.stoptabulate()
-end
-
-local function show_b_c(data,swap) -- no need to store the table, just pass directly
-    local function row(one,two,three)
-        context.NC() context(one)
-        context.NC() context(two)
-        context.NC() context(three)
-        context.NC() context.NR()
-    end
-    if swap then
-        context.starttabulate { "|Tl|Tl|Tr|" }
-    else
-        context.starttabulate { "|Tl|Tr|Tl|" }
-    end
-    row("cmd","chr","name")
-    context.HL()
-    for _,v in next, data do
-        local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", ""
-        local name = gsub(command_name(v) or "","_","\\_")
-        if id > 0 then
-            cs = csname_name(v) or ""
-        end
-        if cmd == letter or cmd == other then
-            sym = "\\char " .. chr
-        elseif cs == "" then
-            -- okay
-        elseif is_activechar(v) then
-            sym = "\\string " .. cs
-        else
-            sym = "\\string\\" .. cs
-        end
-        if swap then
-            row(name,sym,chr)
-        elseif tonumber(chr) < 0 then
-            row(name,"",sym)
-        else
-            row(name,chr,sym)
-        end
-    end
-    context.stoptabulate()
-end
-
--- Even more experimental ...
-
-show_methods.b = function(data) show_b_c(data,false) end
-show_methods.c = function(data) show_b_c(data,true ) end
-
-local remapper      = { }  -- namespace
-collectors.remapper = remapper
-
-local remapperdata  = { }  -- user mappings
-remapper.data       = remapperdata
-
-function remapper.store(tag,class,key)
-    local s = remapperdata[class]
-    if not s then
-        s = { }
-        remapperdata[class] = s
-    end
-    s[key] = collectordata[tag]
-    collectordata[tag] = nil
-end
-
-function remapper.convert(tag,toks)
-    local data = remapperdata[tag]
-    local leftbracket, rightbracket = utfbyte('['), utfbyte(']')
-    local skipping = 0
-    -- todo: math
-    if data then
-        local t, n = { }, 0
-        for s=1,#toks do
-            local tok = toks[s]
-            local one, two = tok[1], tok[2]
-            if one == 11 or one == 12 then
-                if two == leftbracket then
-                    skipping = skipping + 1
-                    n = n + 1 ; t[n] = tok
-                elseif two == rightbracket then
-                    skipping = skipping - 1
-                    n = n + 1 ; t[n] = tok
-                elseif skipping == 0 then
-                    local new = data[two]
-                    if new then
-                        if #new > 1 then
-                            for n=1,#new do
-                                n = n + 1 ; t[n] = new[n]
-                            end
-                        else
-                            n = n + 1 ; t[n] = new[1]
-                        end
-                    else
-                        n = n + 1 ; t[n] = tok
-                    end
-                else
-                    n = n + 1 ; t[n] = tok
-                end
-            else
-                n = n + 1 ; t[n] = tok
-            end
-        end
-        return t
-    else
-        return toks
-    end
-end
+if not modules then modules = { } end modules ['toks-ini'] = {
+    version   = 1.001,
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values
+local format, gsub = string.format, string.gsub
+
+--[[ldx--
+<p>This code is experimental and needs a cleanup. The visualizers will move to
+a module.</p>
+--ldx]]--
+
+-- 1 = command, 2 = modifier (char), 3 = controlsequence id
+--
+-- callback.register('token_filter', token.get_next)
+--
+-- token.get_next()
+-- token.expand()
+-- token.create()
+-- token.csname_id()
+-- token.csname_name(v)
+-- token.command_id()
+-- token.command_name(v)
+-- token.is_expandable()
+-- token.is_activechar()
+-- token.lookup(v)
+
+-- actually, we can use token registers to store tokens
+
+local token, tex = token, tex
+
+local createtoken   = token.create
+local csname_id     = token.csname_id
+local command_id    = token.command_id
+local command_name  = token.command_name
+local get_next      = token.get_next
+local expand        = token.expand
+local is_activechar = token.is_activechar
+local csname_name   = token.csname_name
+
+tokens        = tokens or { }
+local tokens  = tokens
+
+tokens.vbox   = createtoken("vbox")
+tokens.hbox   = createtoken("hbox")
+tokens.vtop   = createtoken("vtop")
+tokens.bgroup = createtoken(utfbyte("{"), 1)
+tokens.egroup = createtoken(utfbyte("}"), 2)
+
+tokens.letter = function(chr) return createtoken(utfbyte(chr), 11) end
+tokens.other  = function(chr) return createtoken(utfbyte(chr), 12) end
+
+tokens.letters = function(str)
+    local t, n = { }, 0
+    for chr in utfvalues(str) do
+        n = n + 1
+        t[n] = createtoken(chr, 11)
+    end
+    return t
+end
+
+tokens.collectors     = tokens.collectors or { }
+local collectors      = tokens.collectors
+
+collectors.data       = collectors.data or { }
+local collectordata   = collectors.data
+
+collectors.registered = collectors.registered or { }
+local registered      = collectors.registered
+
+local function printlist(data)
+    callbacks.push('token_filter', function ()
+       callbacks.pop('token_filter') -- tricky but the nil assignment helps
+       return data
+    end)
+end
+
+tex.printlist = printlist -- will change to another namespace
+
+function collectors.flush(tag)
+    printlist(collectordata[tag])
+end
+
+function collectors.test(tag)
+    printlist(collectordata[tag])
+end
+
+function collectors.register(name)
+    registered[csname_id(name)] = name
+end
+
+local call   = command_id("call")
+local letter = command_id("letter")
+local other  = command_id("other_char")
+
+function collectors.install(tag,end_cs)
+    local data, d = { }, 0
+    collectordata[tag] = data
+    local endcs = csname_id(end_cs)
+    while true do
+        local t = get_next()
+        local a, b = t[1], t[3]
+        if b == endcs then
+            context["end_cs"]()
+            return
+        elseif a == call and registered[b] then
+            expand()
+        else
+            d = d + 1
+            data[d] = t
+        end
+    end
+end
+
+function collectors.handle(tag,handle,flush)
+    collectordata[tag] = handle(collectordata[tag])
+    if flush then
+        collectors.flush(tag)
+    end
+end
+
+local show_methods      = { }
+collectors.show_methods = show_methods
+
+function collectors.show(tag, method)
+    if type(tag) == "table" then
+        show_methods[method or 'a'](tag)
+    else
+        show_methods[method or 'a'](collectordata[tag])
+    end
+end
+
+function collectors.defaultwords(t,str)
+    local n = #t
+    n = n + 1
+    t[n] = tokens.bgroup
+    n = n + 1
+    t[n] = createtoken("red")
+    for i=1,#str do
+        n = n + 1
+        t[n] = tokens.other('*')
+    end
+    n = n + 1
+    t[n] = tokens.egroup
+end
+
+function collectors.dowithwords(tag,handle)
+    local t, w, tn, wn = { }, { }, 0, 0
+    handle = handle or collectors.defaultwords
+    local tagdata = collectordata[tag]
+    for k=1,#tagdata do
+        local v = tagdata[k]
+        if v[1] == letter then
+            wn = wn + 1
+            w[wn] = v[2]
+        else
+            if wn > 0 then
+                handle(t,w)
+                wn = 0
+            end
+            tn = tn + 1
+            t[tn] = v
+        end
+    end
+    if wn > 0 then
+        handle(t,w)
+    end
+    collectordata[tag] = t
+end
+
+local function showtoken(t)
+    if t then
+        local cmd, chr, id, cs, name = t[1], t[2], t[3], nil, command_name(t) or ""
+        if cmd == letter or cmd == other then
+            return format("%s-> %s -> %s", name, chr, utfchar(chr))
+        elseif id > 0 then
+            cs = csname_name(t) or nil
+            if cs then
+                return format("%s-> %s", name, cs)
+            elseif tonumber(chr) < 0 then
+                return format("%s-> %s", name, id)
+            else
+                return format("%s-> (%s,%s)", name, chr, id)
+            end
+        else
+            return format("%s", name)
+        end
+    else
+        return "no node"
+    end
+end
+
+collectors.showtoken = showtoken
+
+function collectors.trace()
+    local t = get_next()
+    logs.report("tokenlist",showtoken(t))
+    return t
+end
+
+-- these might move to a runtime module
+
+show_methods.a = function(data) -- no need to store the table, just pass directly
+    local function row(one,two,three,four,five)
+        context.NC() context(one)
+        context.NC() context(two)
+        context.NC() context(three)
+        context.NC() context(four)
+        context.NC() context(five)
+        context.NC() context.NR()
+    end
+    context.starttabulate { "|T|Tr|cT|Tr|T|" }
+    row("cmd","chr","","id","name")
+    context.HL()
+    for _,v in next, data do
+        local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", ""
+        local name = gsub(command_name(v) or "","_","\\_")
+        if id > 0 then
+            cs = csname_name(v) or ""
+            if cs ~= "" then cs = "\\string " .. cs end
+        else
+            id = ""
+        end
+        if cmd == letter or cmd == other then
+            sym = "\\char " .. chr
+        end
+        if tonumber(chr) < 0 then
+            row(name,"",sym,id,cs)
+        else
+            row(name,chr,sym,id,cs)
+        end
+    end
+    context.stoptabulate()
+end
+
+local function show_b_c(data,swap) -- no need to store the table, just pass directly
+    local function row(one,two,three)
+        context.NC() context(one)
+        context.NC() context(two)
+        context.NC() context(three)
+        context.NC() context.NR()
+    end
+    if swap then
+        context.starttabulate { "|Tl|Tl|Tr|" }
+    else
+        context.starttabulate { "|Tl|Tr|Tl|" }
+    end
+    row("cmd","chr","name")
+    context.HL()
+    for _,v in next, data do
+        local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", ""
+        local name = gsub(command_name(v) or "","_","\\_")
+        if id > 0 then
+            cs = csname_name(v) or ""
+        end
+        if cmd == letter or cmd == other then
+            sym = "\\char " .. chr
+        elseif cs == "" then
+            -- okay
+        elseif is_activechar(v) then
+            sym = "\\string " .. cs
+        else
+            sym = "\\string\\" .. cs
+        end
+        if swap then
+            row(name,sym,chr)
+        elseif tonumber(chr) < 0 then
+            row(name,"",sym)
+        else
+            row(name,chr,sym)
+        end
+    end
+    context.stoptabulate()
+end
+
+-- Even more experimental ...
+
+show_methods.b = function(data) show_b_c(data,false) end
+show_methods.c = function(data) show_b_c(data,true ) end
+
+local remapper      = { }  -- namespace
+collectors.remapper = remapper
+
+local remapperdata  = { }  -- user mappings
+remapper.data       = remapperdata
+
+function remapper.store(tag,class,key)
+    local s = remapperdata[class]
+    if not s then
+        s = { }
+        remapperdata[class] = s
+    end
+    s[key] = collectordata[tag]
+    collectordata[tag] = nil
+end
+
+function remapper.convert(tag,toks)
+    local data = remapperdata[tag]
+    local leftbracket, rightbracket = utfbyte('['), utfbyte(']')
+    local skipping = 0
+    -- todo: math
+    if data then
+        local t, n = { }, 0
+        for s=1,#toks do
+            local tok = toks[s]
+            local one, two = tok[1], tok[2]
+            if one == 11 or one == 12 then
+                if two == leftbracket then
+                    skipping = skipping + 1
+                    n = n + 1 ; t[n] = tok
+                elseif two == rightbracket then
+                    skipping = skipping - 1
+                    n = n + 1 ; t[n] = tok
+                elseif skipping == 0 then
+                    local new = data[two]
+                    if new then
+                        if #new > 1 then
+                            for n=1,#new do
+                                n = n + 1 ; t[n] = new[n]
+                            end
+                        else
+                            n = n + 1 ; t[n] = new[1]
+                        end
+                    else
+                        n = n + 1 ; t[n] = tok
+                    end
+                else
+                    n = n + 1 ; t[n] = tok
+                end
+            else
+                n = n + 1 ; t[n] = tok
+            end
+        end
+        return t
+    else
+        return toks
+    end
+end
-- 
cgit v1.2.3