if not modules then modules = { } end modules ['toks-scn'] = {
    version   = 1.001,
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

-- Writing this kind of code (and completing the newtoken code base) is fun. I did
-- so with the brilliant film music from The Girl with the Dragon Tattoo running in a
-- loop in the background (three cd's by Trent Reznor and Atticus Ross). An alien
-- feeling helps with alien code.

-- todo: more \let's at the tex end

local type, next, tostring, tonumber = type, next, tostring, tonumber

local formatters     = string.formatters
local concat         = table.concat

local scanners       = tokens.scanners
local tokenbits      = tokens.bits

local scanstring     = scanners.string
local scanargument   = scanners.argument
local scandelimited  = scanners.delimited  -- lmtx
local scanverbatim   = scanners.verbatim
local scantokenlist  = scanners.tokenlist
local scantoks       = scanners.toks
local scaninteger    = scanners.integer
local scancardinal   = scanners.cardinal
local scannumber     = scanners.number
local scankeyword    = scanners.keyword
local scankeywordcs  = scanners.keywordcs
local scanword       = scanners.word
local scanletters    = scanners.letters
local scankey        = scanners.key
local scancode       = scanners.code
local scanboolean    = scanners.boolean
local scandimen      = scanners.dimen
local scanglue       = scanners.glue
local scangluevalues = scanners.gluevalues
local scangluespec   = scanners.gluespec
local scancsname     = scanners.csname

local todimen        = number.todimen
local toboolean      = toboolean

local lpegmatch      = lpeg.match
local p_unquoted     = lpeg.Cs(lpeg.patterns.unquoted)

local trace_compile  = false  trackers.register("tokens.compile", function(v) trace_compile = v end)
local report_compile = logs.reporter("tokens","compile")
local report_scan    = logs.reporter("tokens","scan")

local open  = tokenbits.open
local close = tokenbits.close

local function scanopen()
    while true do
        local c = scancode(open)
        if c == 123 then
            return true
     -- elseif c ~= 32 then
        elseif not c then
            return
        end
    end
end

local function scanclose()
    while true do
        local c = scancode(close)
        if c == 125 then
            return true
     -- elseif c ~= 32 then
        elseif not c then
            return
        end
    end
end

scanners.scanopen  = scanopen
scanners.scanclose = scanclose

local function scanlist()
    local wrapped = scanopen()
    local list    = { }
    local size    = 0
    while true do
        local entry = scanstring()
        if entry then
            size = size + 1
            list[size] = entry
        else
            break
        end
    end
    if wrapped then
        scanclose()
    end
    return list
end

local function scanconditional()
    local kw = scanword()
    if kw == "true" then
        return true
    end
    if kw == "false" then
        return false
    end
    local c = scaninteger()
    if c then
        return c == 0 -- with a conditional 0=true
    end
    return nil
end

local function scantable(t,data)
    if not data then
        data = { }
    end
    if t then
        local wrapped = scanopen()
        while true do
            local key = scanword(true)
            if key then
                local get = t[key]
                if get then
                    data[key] = get()
                else
                    -- catch all we can get
                end
            else
                break
            end
        end
        if wrapped then
            scanclose()
        end
    end
    return data
end

function tokens.constant(s)
    if type(s) == "string" then
        return "'" .. s .. "'"
    else
        return s
    end
end

scanners.list        = scanlist
scanners.table       = scantable
scanners.conditional = scanconditional

function scanners.whd()
    local width, height, depth
    while true do
        if scankeyword("width") then
            width = scandimen()
        elseif scankeyword("height") then
            height = scandimen()
        elseif scankeyword("depth") then
            depth = scandimen()
        else
            break
        end
    end
    if width or height or depth then
        return width or 0, height or 0, depth or 0
    else
        -- we inherit
    end
end

-- begin lmtx

local l = utf.byte("[")
local r = utf.byte("]")

local function scanbracketed()
    local s = scandelimited(l, r)
    if s then
        return s
    else
        report_scan("missing argument in line %i of %a", status.linenumber, status.filename)
        return ""
    end
end

local function scanoptional()
    return scandelimited(l, r) or ""
end

local function scanbracketedasis()
    return scandelimited(l, r, false)
end

local function scanargumentasis()
    return scanargument(false)
end

scanners.bracketed     = scanbracketed
scanners.optional      = scanoptional
scanners.bracketedasis = scanbracketedasis
scanners.argumentasis  = scanargumentasis

-- end lmtx

local shortcuts = {
    tokens            = tokens,
    bits              = tokenbits,
    open              = open,
    close             = close,
    scanners          = scanners,
    scanstring        = scanstring,
    scanargument      = scanargument,
    scanverbatim      = scanverbatim,
    scantokenlist     = scantokenlist,
    scantoks          = scantoks,
    scaninteger       = scaninteger,
    scancardinal      = scancardinal,
    scannumber        = scannumber,
    scantable         = scantable, -- not directly useable
    scankeyword       = scankeyword,
    scankeywordcs     = scankeywordcs,
    scanword          = scanword,
    scanletters       = scanletters,
 -- scankey           = scankey,
    scancode          = scancode,
    scanboolean       = scanboolean,
    scanglue          = scanglue, -- list
    scangluespec      = scangluespec,
    scangluevalues    = scangluevalues,
    scandimen         = scandimen,
    scandimension     = scandimen,
    scanbox           = scanners.box,
    scanhbox          = scanners.hbox,
    scanvbox          = scanners.vbox,
    scanvtop          = scanners.vtop,
    scanconditional   = scanconditional,
    scanopen          = scanopen,
    scanclose         = scanclose,
    scanlist          = scanlist,
    scancsname        = scancsname,
    todimen           = todimen,
    tonumber          = tonumber,
    tostring          = tostring,
    toboolean         = toboolean,
    inspect           = inspect,
    report            = report_scan,
    -- lmtx
    scandelimited     = scandelimited, -- not directly useable
    scanbracketed     = scanbracketed,
    scanoptional      = scanoptional,
    scanbracketedasis = scanbracketedasis,
    scanargumentasis  = scanargumentasis,
}

tokens.shortcuts = shortcuts

local load = load
local dump = string.dump

local function loadstripped(code)
     return load(code,nil,nil,shortcuts)
  -- return load(dump(load(code),true),nil,nil,shortcuts)
end

tokens.converters = {
    tonumber  = "tonumber",
    tostring  = "tostring",
    toboolean = "toboolean",
    todimen   = "todimen",
    toglue    = "todimen",
}

-- We could just pickup a keyword but then we really need to make sure that no number
-- follows it when that is the assignment and adding an optional = defeats the gain
-- in speed. Currently we have sources with no spaces (\startcontextdefinitioncode
-- ...) so it fails there.
--
-- Another drawback is that we then need to use { } instead of ending with \relax (as
-- we can do now) but that is no big deal. It's just that I then need to check the TeX
-- end. More pain than gain and a bit risky too. Using scanletters works better, but
-- the gain is only some 10 percent but if we don't have keywords with numbers it might
-- make sense in the end, some day.

local f_if       = formatters[    "  if scankeywordcs('%s') then data['%s'] = scan%s()"]
local f_elseif   = formatters["  elseif scankeywordcs('%s') then data['%s'] = scan%s()"]

----- f_if_x     = formatters[    "  if not data['%s'] and scankeywordcs('%s') then data['%s'] = scan%s()"]
----- f_elseif_x = formatters["  elseif not data['%s'] and scankeywordcs('%s') then data['%s'] = scan%s()"]

-- if CONTEXTLMTXMODE > 0 then
--     f_if       = formatters["  local key = scanletters() if key == '' then break elseif key == '%s' then data['%s'] = scan%s()"]
--     f_elseif   = formatters["  elseif key == '%s' then data['%s'] = scan%s()"]
-- end

local f_local    = formatters["local scan%s = scanners.%s"]
local f_scan     = formatters["scan%s()"]
local f_shortcut = formatters["local %s = scanners.converters.%s"]

local f_if_c     = formatters[    "  if scankeywordcs('%s') then data['%s'] = %s(scan%s())"]
local f_elseif_c = formatters["  elseif scankeywordcs('%s') then data['%s'] = %s(scan%s())"]
local f_scan_c   = formatters["%s(scan%s())"]

-- see above

-- if CONTEXTLMTXMODE > 0 then
--     f_if_c     = formatters["  local key = scanletters() if key == '' then break elseif key == '%s' then data['%s'] = %s(scan%s())"]
--     f_elseif_c = formatters["  elseif k == '%s' then data['%s'] = %s(scan%s())"]
-- end

local f_any      = formatters["  else local key = scanword(true) if key then data[key] = scan%s() else break end end"]
local f_any_c    = formatters["  else local key = scanword(true) if key then data[key] = %s(scan%s()) else break end end"]
local s_done     = "  else break end"

local f_any_all  = formatters["  local key = scanword(true) if key then data[key] = scan%s() else break end"]
local f_any_all_c= formatters["  local key = scanword(true) if key then data[key] = %s(scan%s()) else break end"]

local f_table    = formatters["%\nt\nreturn function()\n  local data = { }\n%s\n  return %s\nend\n"]
local f_sequence = formatters["%\nt\n%\nt\n%\nt\nreturn function()\n    return %s\nend\n"]
local f_singular = formatters["%\nt\n%\nt\n\nreturn function(%s)\n    return %s\nend\n"]
local f_simple   = formatters["%\nt\nreturn function()\n    return %s\nend\n"]
local f_string   = formatters["%q"]
local f_action_f = formatters["action%s(%s)"]
local f_action_s = formatters["local action%s = tokens._action[%s]"]
local f_nested   = formatters["local function scan%s()\n  local data = { }\n%s\n  return data\nend\n"]

local f_check = formatters[ [[
  local wrapped = scanopen()
  while true do
    ]] .. "%\nt\n" .. [[
    %s
  end
  if wrapped then
    scanclose()
  end
]] ]

-- using these shortcuts saves temporary small tables (okay, it looks uglier)

local presets = {
    ["1 string" ] = { "string" },
    ["2 strings"] = { "string", "string" },
    ["3 strings"] = { "string", "string", "string" },
    ["4 strings"] = { "string", "string", "string", "string" },
    ["5 strings"] = { "string", "string", "string", "string", "string" },
    ["6 strings"] = { "string", "string", "string", "string", "string", "string" },
    ["7 strings"] = { "string", "string", "string", "string", "string", "string", "string" },
    ["8 strings"] = { "string", "string", "string", "string", "string", "string", "string", "string" },

    ["1 argument" ] = { "argument" },
    ["2 arguments"] = { "argument", "argument" },
    ["3 arguments"] = { "argument", "argument", "argument" },
    ["4 arguments"] = { "argument", "argument", "argument", "argument" },
}

tokens.presets = presets

function tokens.compile(specification)
    local f = { }
    local n = 0
    local c = { }
    local t = specification.arguments or specification
    local a = specification.actions or nil
    if type(a) == "function" then
        a = { a }
    end
    local code
    local args
    local function compile(t,nested)
        local done = s_done
        local r = { }
        local m = 0
        for i=1,#t do
            local ti = t[i]
            if ti == "*" and i == 1 then
                done = f_any_all("string")
            else
                local t1 = ti[1]
                local t2 = ti[2] or "string"
                if type(t2) == "table" then
                    n = n + 1
                    f[n] = compile(t2,n)
                    t2 = n
                end
                local t3 = ti[3]
                if type(t3) == "function" then
                    -- todo: also create shortcut
                elseif t3 then
                    c[t3] = f_shortcut(t3,t3)
                    if t1 == "*" then
                        if i == 1 then
                            done = f_any_all_c(t3,t2)
                            break
                        else
                            done = f_any_c(t3,t2)
                        end
                    else
                        m = m + 1
                        r[m] = (m > 1 and f_elseif_c or f_if_c)(t1,t1,t3,t2)
                    end
                else
                    if t1 == "*" then
                        if i == 1 then
                            done = f_any_all(t2)
                            break
                        else
                            done = f_any(t2)
                        end
                    else
                        m = m + 1
                        r[m] = (m > 1 and f_elseif   or f_if  )(t1,t1,t2)
                     -- r[m] = (m > 1 and f_elseif_x or f_if_x)(t1,t1,t1,t2)
                    end
                end
            end
        end
        local c = f_check(r,done)
        if nested then
            return f_nested(nested,c)
        else
            return c
        end
    end
    local p = t and presets[t] -- already done in implement
    if p then
        t = p
    end
    local tt = type(t)
    if tt == "string" then
        if a then
            local s = lpegmatch(p_unquoted,t)
            if s and t ~= s then
                code = t
            else
                code = f_scan(t)
            end
            tokens._action = a
            for i=1,#a do
                code = f_action_f(i,code)
                n    = n + 1
                f[n] = f_action_s(i,i)
            end
            code = f_simple(f,code)
        else
            return scanners[t]
        end
    elseif tt ~= "table" then
        return
    elseif #t == 1 then
        local ti = t[1]
        if type(ti) == "table" then
            ti = compile(ti)
            code = "data"
            if a then
                tokens._action = a
                for i=1,#a do
                    code = f_action_f(i,code)
                    n    = n + 1
                    f[n] = f_action_s(i,i)
                end
            end
            code = f_table(f,ti,code)
        elseif a then
            code = f_scan(ti)
            tokens._action = a
            for i=1,#a do
                code = f_action_f(i,code)
                n    = n + 1
                f[n] = f_action_s(i,i)
            end
            code = f_simple(f,code)
        else
            return scanners[ti]
        end
    elseif #t == 0 then
        if specification.value then
            code = "b"
            args = "_,b"
        else
            code = ""
            args = ""
        end
        if a then
            tokens._action = a
            for i=1,#a do
                code = f_action_f(i,code)
                n    = n + 1
                f[n] = f_action_s(i,i)
            end
        end
        code = f_singular(c,f,args,code)
    else
        local r = { }
        local p = { }
        local m = 0
        for i=1,#t do
            local ti = t[i]
            local tt = type(ti)
            if tt == "table" then
                if ti[1] == "_constant_" then
                    local v = ti[2]
                    if type(v) == "string" then
                        r[i] = f_string(v)
                    else
                        r[i] = tostring(v)
                    end
                else
                    m = m + 1
                    p[m] = compile(ti,100+m)
                    r[i] = f_scan(100+m)
                end
            elseif tt == "number" then
                r[i] = tostring(ti)
            elseif tt == "boolean" then
                r[i] = tostring(ti)
            else
                local s = lpegmatch(p_unquoted,ti)
                if s and ti ~= s then
                    r[i] = ti -- a string, given as "'foo'" or '"foo"'
                elseif scanners[ti] then
                    r[i] = f_scan(ti)
                else
                    report_compile("unknown scanner %a",ti)
                    r[i] = ti
                end
            end
        end
        code = concat(r,",")
        if a then
            tokens._action = a
            for i=1,#a do
                code = f_action_f(i,code)
                n    = n + 1
                f[n] = f_action_s(i,i)
            end
        end
        code = f_sequence(c,f,p,code)
    end
    if not code then
        return
    end
    if trace_compile then
        report_compile("code: %s",code)
    end
    local code, message = loadstripped(code)
    if code then
        code = code() -- sets action
    else
        report_compile("error in code: %s",code)
        report_compile("error message: %s",message)
    end
    if a then
        tokens._action = nil
    end
    if code then
        return code
    end
end

-- local fetch = tokens.compile {
--     "string",
--     "string",
--     {
--         { "data",    "string" },
--         { "tab",     "string" },
--         { "method",  "string" },
--         { "foo", {
--             { "method", "integer" },
--             { "compact", "number" },
--             { "nature" },
--             { "*" }, -- any key
--         } },
--         { "compact", "string", "tonumber" },
--         { "nature",  "boolean" },
--         { "escape",  "string" },
--         { "escape"  },
--     },
--     "boolean",
-- }
--
-- os.exit()