path: root/tex/context/base/mkiv/toks-ini.lua

if not modules then modules = { } end modules ['toks-ini'] = {
    version   = 1.001,
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

tokens = tokens or { }

local tokens     = tokens
local token      = token -- the built in one
local next       = next
local tonumber   = tonumber
local tostring   = tostring
local utfchar    = utf.char
local char       = string.char
local printtable = table.print
local concat     = table.concat
local format     = string.format

local commands  = token.commands()
tokens.commands = utilities.storage.allocate(table.swapped(commands,commands))
tokens.values   = { }

local scantoks        = token.scan_toks
local scanstring      = token.scan_string
local scanargument    = token.scan_argument
local scandelimited   = token.scan_delimited
local scantokenlist   = token.scan_tokenlist or scanstring
local scaninteger     = token.scan_integer or token.scan_int
local scancardinal    = token.scan_cardinal
local scancode        = token.scan_code
local scantokencode   = token.scan_token_code
local scandimen       = token.scan_dimen
local scanglue        = token.scan_glue
local scanskip        = token.scan_skip
local scankeyword     = token.scan_keyword
local scankeywordcs   = token.scan_keyword_cs or scankeyword
local scantoken       = token.scan_token
local scanbox         = token.scan_box
local scanword        = token.scan_word
local scanletters     = token.scan_letters or scanword -- lmtx
local scankey         = token.scan_key
local scanvalue       = token.scan_value
local scanchar        = token.scan_char
local scannumber      = token.scan_number -- not defined
local scancsname      = token.scan_csname
local scanreal        = token.scan_real
local scanfloat       = token.scan_float
local scanluanumber   = token.scan_luanumber   or scanfloat    -- only lmtx
local scanluainteger  = token.scan_luainteger  or scaninteger  -- only lmtx
local scanluacardinal = token.scan_luacardinal or scancardinal -- only lmtx

local setmacro        = token.set_macro
local setchar         = token.set_char
local setlua          = token.set_lua

local createtoken     = token.create
local newtoken        = token.new
local isdefined       = token.is_defined
local istoken         = token.is_token

tokens.new            = newtoken
tokens.create         = createtoken
tokens.istoken        = istoken
tokens.isdefined      = isdefined
tokens.defined        = isdefined

local bits = {
    escape      = 0x00000001, -- 2^00
    begingroup  = 0x00000002, -- 2^01
    endgroup    = 0x00000004, -- 2^02
    mathshift   = 0x00000008, -- 2^03
    alignment   = 0x00000010, -- 2^04
    endofline   = 0x00000020, -- 2^05
    parameter   = 0x00000040, -- 2^06
    superscript = 0x00000080, -- 2^07
    subscript   = 0x00000100, -- 2^08
    ignore      = 0x00000200, -- 2^09
    space       = 0x00000400, -- 2^10 -- 1024
    letter      = 0x00000800, -- 2^11
    other       = 0x00001000, -- 2^12
    active      = 0x00002000, -- 2^13
    comment     = 0x00004000, -- 2^14
    invalid     = 0x00008000, -- 2^15
    --
    character   = 0x00001800, -- 2^11 + 2^12
    whitespace  = 0x00002400, -- 2^13 + 2^10 --    / needs more checking
    --
    open        = 0x00000402, -- 2^10 + 2^01 -- space + begingroup
    close       = 0x00000404, -- 2^10 + 2^02 -- space + endgroup
}

-- for k, v in next, bits do bits[v] = k end

tokens.bits = bits

-- words are space or \relax terminated and the trailing space is gobbled; a word
-- can contain any non-space letter/other (see archive for implementation in lua)

if not scannumber then

    scannumber = function(base)
        local s = scanword()
        if not s then
            return nil
        elseif base then
            return tonumber(s,base)
        else
            return tonumber(s)
        end
    end

end

local function scanboolean()
    local kw = scanword()
    if kw == "true" then
        return true
    elseif kw == "false" then
        return false
    else
        return nil
    end
end

local function scanverbatim()
    return scanargument(false)
end

if not scanbox then

    local scanlist = token.scan_list
    local putnext  = token.put_next

    scanbox = function(s)
        if s == "hbox" or s == "vbox" or s == "vtop" then
            putnext(createtoken(s))
        end
        return scanlist()
    end

    token.scanbox = scanbox

end

tokens.scanners = { -- these expand
    token          = scantoken,
    toks           = scantoks,
    tokens         = scantoks,
    box            = scanbox,
    hbox           = function() return scanbox("hbox") end,
    vbox           = function() return scanbox("vbox") end,
    vtop           = function() return scanbox("vtop") end,
    dimen          = scandimen,
    dimension      = scandimen,
    glue           = scanglue,
    gluevalues     = function() return scanglue(false,false,true) end,
    gluespec       = scanskip,
    integer        = scaninteger,
    cardinal       = scancardinal,
    real           = scanreal,
    float          = scanfloat,
    luanumber      = scanluanumber,
    luainteger     = scanluainteger,
    luacardinal    = scanluacardinal,
    count          = scaninteger,
    string         = scanstring,
    argument       = scanargument,
    delimited      = scandelimited,
    tokenlist      = scantokenlist,
    verbatim       = scanverbatim, -- detokenize
    code           = scancode,
    tokencode      = scantokencode,
    word           = scanword,
    letters        = scanletters,
    key            = scankey,
    value          = scanvalue,
    char           = scanchar,
    number         = scannumber,
    boolean        = scanboolean,
    keyword        = scankeyword,
    keywordcs      = scankeywordcs,
    csname         = scancsname,

    next           = token.scan_next,
    nextexpanded   = token.scan_next_expanded,

    peek           = token.peek_next,
    peekexpanded   = token.peek_next_expanded,
    peekchar       = token.peek_next_char,

    skip           = token.skip_next,
    skipexpanded   = token.skip_next_expanded,

    cmdchr         = token.scan_cmdchr,
    cmdchrexpanded = token.scan_cmdchr_expanded,

    ischar         = token.is_next_char,
}

tokens.getters = { -- these don't expand
    meaning = token.get_meaning,
    macro   = token.get_macro,
    token   = token.scan_next or token.get_next, -- not here, use scanners.next or token
    cstoken = token.get_cstoken,
    count   = tex.getcount,
    dimen   = tex.getdimen,
    skip    = tex.getglue,
    glue    = tex.getglue,
    skip    = tex.getmuglue,
    glue    = tex.getmuglue,
    box     = tex.getbox,
}

tokens.setters = {
    macro = setmacro,
    char  = setchar,
    lua   = setlua,
    count = tex.setcount,
    dimen = tex.setdimen,
    skip  = tex.setglue,
    glue  = tex.setglue,
    skip  = tex.setmuglue,
    glue  = tex.setmuglue,
    box   = tex.setbox,
}

tokens.accessors = {
    command    = token.get_command,
    cmd        = token.get_command,
    cmdname    = token.get_cmdname,
    name       = token.get_cmdname,
    csname     = token.get_csname,
    index      = token.get_index,
    active     = token.get_active,
    frozen     = token.get_frozen,
    protected  = token.get_protected,
    expandable = token.get_protected,
    user       = token.get_user,
    cmdchrcs   = token.get_cmdchrcs,
    active     = token.get_active,
    range      = token.get_range,
}

if setinspector then

    local simple = { letter = "letter", other_char = "other" }

    local astable = function(t)
        if t and istoken(t) then
            local cmdname = t.cmdname
            local simple  = simple[cmdname]
            if simple then
                return {
                    id         = t.id,
                    category   = simple,
                    character  = utfchar(t.mode) or nil,
                }
            else
                return {
                    command    = t.command,
                    id         = t.id,
                    tok        = t.tok,
                    csname     = t.csname,
                    active     = t.active,
                    expandable = t.expandable,
                    protected  = t.protected,
                    frozen     = t.frozen,
                    mode       = t.mode,
                    index      = t.index,
                    user       = t.user,
                    cmdname    = cmdname,
                }
            end
        end
    end

    tokens.astable = astable

    setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end)

end

tokens.cache = table.setmetatableindex(function(t,k)
    if not isdefined(k) then
        setmacro(k,"","global")
    end
    local v = createtoken(k)
    t[k] = v
    return v
end)

if LUATEXVERSION < 114 then

    local d = tokens.defined
    local c = tokens.create

    function tokens.defined(s,b)
        if b then
            return d(s)
        else
            return c(s).cmd_name == "undefined_cmd"
        end
    end

end