path: root/tex/context/base/mkxl/toks-ini.lmt

if not modules then modules = { } end modules ['toks-ini'] = {
    version   = 1.001,
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

tokens = tokens or { }

local tokens     = tokens
local token      = token -- the built in one
local next       = next
local tonumber   = tonumber
local tostring   = tostring
local utfchar    = utf.char
local char       = string.char
local printtable = table.print
local concat     = table.concat
local format     = string.format

local commands  = token.getcommandvalues()  -- tex.functioncode
local values    = token.getfunctionvalues() -- tex.functioncode
values.dimen    = values.dimension
values.count    = values.integer
tokens.values   = utilities.storage.allocate(table.swapped(values, values))
tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))

local scantoks            = token.scantoks
local scanstring          = token.scanstring
local scanargument        = token.scanargument
local scandelimited       = token.scandelimited
local scantokenlist       = token.scantokenlist
local scaninteger         = token.scaninteger
local scancardinal        = token.scancardinal
local scancode            = token.scancode
local scantokencode       = token.scantokencode
local scandimen           = token.scandimen
local scanglue            = token.scanglue
local scanskip            = token.scanskip
local scankeyword         = token.scankeyword
local scankeywordcs       = token.scankeywordcs
local scantoken           = token.scantoken
local scanbox             = token.scanbox
local scanword            = token.scanword
local scanletters         = token.scanletters
local scankey             = token.scankey
local scanvalue           = token.scanvalue
local scanchar            = token.scanchar
local scancsname          = token.scancsname
local scannextchar        = token.scannextchar
local scanposit           = token.scanposit
local scanreal            = token.scanreal
local scanfloat           = token.scanfloat
local scanluanumber       = token.scanluanumber
local scanluainteger      = token.scanluainteger
local scanluacardinal     = token.scanluacardinal
local scanintegerargument = token.scanintegerargument
local scandimenargument   = token.scandimenargument
local scandetokened       = token.scandetokened

local scannumber          = token.scannumber
local scanboolean         = token.scanboolean

local setmacro     = token.setmacro
local setchar      = token.setchar
local setlua       = token.setlua

local createtoken  = token.create
local newtoken     = token.new
local isdefined    = token.isdefined
local istoken      = token.istoken

tokens.new         = newtoken
tokens.create      = createtoken
tokens.istoken     = istoken
tokens.isdefined   = isdefined
tokens.defined     = isdefined

tokens.gobble      = token.gobble

tokens.getinteger  = token.getinteger
tokens.setinteger  = token.setinteger

local bits = {
    escape      = 0x00000001, -- 2^00
    begingroup  = 0x00000002, -- 2^01
    endgroup    = 0x00000004, -- 2^02
    mathshift   = 0x00000008, -- 2^03
    alignment   = 0x00000010, -- 2^04
    endofline   = 0x00000020, -- 2^05
    parameter   = 0x00000040, -- 2^06
    superscript = 0x00000080, -- 2^07
    subscript   = 0x00000100, -- 2^08
    ignore      = 0x00000200, -- 2^09
    space       = 0x00000400, -- 2^10 -- 1024
    letter      = 0x00000800, -- 2^11
    other       = 0x00001000, -- 2^12
    active      = 0x00002000, -- 2^13
    comment     = 0x00004000, -- 2^14
    invalid     = 0x00008000, -- 2^15
    --
    character   = 0x00001800, -- 2^11 + 2^12
    whitespace  = 0x00002400, -- 2^13 + 2^10 --    / needs more checking
    --
    open        = 0x00000402, -- 2^10 + 2^01 -- space + begingroup
    close       = 0x00000404, -- 2^10 + 2^02 -- space + endgroup
}

-- for k, v in next, bits do bits[v] = k end

tokens.bits = bits

-- words are space or \relax terminated and the trailing space is gobbled; a word
-- can contain any non-space letter/other (see archive for implementation in lua)

if not scannumber then -- we do have float and real .. this is actually scanluanumber

    scannumber = function(base)
        local s = scanword()
        if not s then
            return nil
        elseif base then
            return tonumber(s,base)
        else
            return tonumber(s)
        end
    end

end

if not scanboolean then

    scanboolean = function()
        local kw = scanword()
        if kw == "true" then
            return true
        elseif kw == "false" then
            return false
        else
            return nil
        end
    end

end

local function scanverbatim() -- check
    return scanargument(false)
end

tokens.scanners = { -- these expand
    token           = scantoken,
    toks            = scantoks,
    tokens          = scantoks,
    box             = scanbox,
    hbox            = function() return scanbox("hbox") end,
    vbox            = function() return scanbox("vbox") end,
    vtop            = function() return scanbox("vtop") end,
    dimen           = scandimen,
    dimension       = scandimen,
    glue            = scanglue,
    gluevalues      = function() return scanglue(false,false,true) end,
    gluespec        = scanskip,
    integer         = scaninteger,
    cardinal        = scancardinal,
    posit           = scanposit,
    real            = scanreal,
    float           = scanfloat,
    luanumber       = scanluanumber,
    luainteger      = scanluainteger,
    luacardinal     = scanluacardinal,
    count           = scaninteger,
    string          = scanstring,
    argument        = scanargument,
    delimited       = scandelimited,
    tokenlist       = scantokenlist,
    verbatim        = scanverbatim, -- detokenize
    code            = scancode,
    tokencode       = scantokencode,
    word            = scanword,
    letters         = scanletters,
    key             = scankey,
    value           = scanvalue,
    char            = scanchar,
    number          = scannumber,
    boolean         = scanboolean,
    keyword         = scankeyword,
    keywordcs       = scankeywordcs,
    csname          = scancsname,
    nextchar        = scannextchar,
    detokened       = scandetokened,

    next            = token.scannext,
    nextexpanded    = token.scannextexpanded,

    peek            = token.peeknext,
    peekexpanded    = token.peeknextexpanded,
    peekchar        = token.peeknextchar,

    skip            = token.skipnext,
    skipexpanded    = token.skipnextexpanded,

    cmdchr          = token.scancmdchr,
    cmdchrexpanded  = token.scancmdchrexpanded,

    ischar          = token.isnextchar,

    integerargument = scanintegerargument,
    dimenargument   = scandimenargument,
}

tokens.getters = { -- these don't expand
    meaning = token.getmeaning,
    macro   = token.getmacro,
    token   = token.scannext,
    cstoken = token.getcstoken,
    count   = tex.getcount,
    dimen   = tex.getdimen,
    skip    = tex.getglue,
    glue    = tex.getglue,
    skip    = tex.getmuglue,
    glue    = tex.getmuglue,
    box     = tex.getbox,
}

tokens.setters = {
    macro = setmacro,
    char  = setchar,
    lua   = setlua,
    count = tex.setcount,
    dimen = tex.setdimen,
    skip  = tex.setglue,
    glue  = tex.setglue,
    skip  = tex.setmuglue,
    glue  = tex.setmuglue,
    box   = tex.setbox,
}

tokens.accessors = {
    command    = token.getcommand,
    cmd        = token.getcommand,
    cmdname    = token.getcmdname,
    name       = token.getcmdname,
    csname     = token.getcsname,
    index      = token.getindex,
    active     = token.getactive,
    frozen     = token.getfrozen,
    protected  = token.getprotected,
    expandable = token.getprotected,
    user       = token.getuser,
    cmdchrcs   = token.getcmdchrcs,
    active     = token.getactive,
    range      = token.getrange,
}

if setinspector then -- can best be true

    local simple = { letter = "letter", other_char = "other" }

    local astable = function(t)
        if t and istoken(t) then
            local cmdname = t.cmdname
            local simple  = simple[cmdname]
            if simple then
                return {
                    id         = t.id,
                    category   = simple,
                    character  = utfchar(t.index) or nil,
                }
            else
                return {
                    id         = t.id,
                    command    = t.command,
                    index      = t.index,
                    csname     = t.csname,
                    cmdname    = cmdname,
                    active     = t.active,
                    expandable = t.expandable,
                    frozen     = t.frozen,
                    tolerant   = t.tolerant,
                    protected  = t.protected,
                    primitive  = t.primitive,
                    permanent  = t.permanent,
                    noaligned  = t.noaligned,
                    instance   = t.instance,
                    immutable  = t.imutable,
                    user       = t.user,
                }
            end
        end
    end

    tokens.astable = astable

    setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)

end

tokens.cache = table.setmetatableindex(function(t,k)
    if not isdefined(k) then
        setmacro(k,"","global") -- So we default to nothing! Maybe some message as option?
    end
    local v = createtoken(k)
    t[k] = v
    return v
end)

-- This will go, although ... we use some old names in styles. Check what we
-- actually use.

token.is_token              = token.istoken
token.is_defined            = token.isdefined
token.scan_next             = token.scannext
token.scan_next_expanded    = token.scannextexpanded
token.scan_next_char        = token.scannextchar
token.skip_next             = token.skipnext
token.skip_next_expanded    = token.skipnextexpanded
token.peek_next             = token.peeknext
token.peek_next_expanded    = token.peeknextexpanded
token.peek_next_char        = token.peeknextchar
token.scan_cmdchr           = token.scancmdchr
token.scan_cmdchr_expanded  = token.scancmdchrexpanded
token.scan_keyword          = token.scankeyword
token.scan_keyword_cs       = token.scankeywordcs
token.scan_integer          = token.scaninteger
token.scan_integer_argument = token.scanintegerargument
token.scan_dimen_argument   = token.scandimenargument
token.scan_cardinal         = token.scancardinal
token.scan_float            = token.scanfloat
token.scan_real             = token.scanreal
token.scan_luanumber        = token.scanluanumber
token.scan_luainteger       = token.scanluainteger
token.scan_luacardinal      = token.scanluacardinal
token.scan_scale            = token.scanscale
token.scan_dimen            = token.scandimen
token.scan_skip             = token.scanskip
token.scan_glue             = token.scanglue
token.scan_toks             = token.scantoks
token.scan_tokenlist        = token.scantokenlist
token.scan_code             = token.scancode
token.scan_token_code       = token.scantokencode
token.scan_string           = token.scanstring
token.scan_argument         = token.scanargument
token.scan_delimited        = token.scandelimited
token.scan_word             = token.scanword
token.scan_letters          = token.scanletters
token.scan_key              = token.scankey
token.scan_value            = token.scanvalue
token.scan_char             = token.scanchar
token.scan_csname           = token.scancsname
token.scan_token            = token.scantoken
token.scan_box              = token.scanbox
token.is_next_char          = token.isnextchar
token.put_next              = token.putnext
token.put_back              = token.putback
token.get_command           = token.getcommand
token.get_index             = token.getindex
token.get_range             = token.getrange
token.get_cmdname           = token.getcmdname
token.get_csname            = token.getcsname
token.get_id                = token.getid
token.get_tok               = token.gettok
token.get_active            = token.getactive
token.get_expandable        = token.getexpandable
token.get_protected         = token.getprotected
token.get_frozen            = token.getfrozen
token.get_tolerant          = token.gettolerant
token.get_noaligned         = token.getnoaligned
token.get_primitive         = token.getprimitive
token.get_permanent         = token.getpermanent
token.get_immutable         = token.getimmutable
token.get_instance          = token.getinstance
token.get_flags             = token.getflags
token.get_parameters        = token.getparameters
token.get_macro             = token.getmacro
token.get_meaning           = token.getmeaning
token.get_cmdchrcs          = token.getcmdchrcs
token.get_cstoken           = token.getcstoken
token.get_fields            = token.getfields
token.set_macro             = token.setmacro
token.undefine_macro        = token.undefinemacro
token.expand_macro          = token.expandmacro
token.set_char              = token.setchar
token.set_lua               = token.setlua
token.set_integer           = token.setinteger
token.get_integer           = token.getinteger
token.set_dimension         = token.setdimension
token.get_dimension         = token.getdimension
token.gobble_integer        = token.gobbleinteger
token.gobble_dimen          = token.gobbledimen
token.future_expand         = token.futureexpand
token.push_macro            = token.pushmacro
token.pop_macro             = token.popmacro
token.save_lua              = token.savelua