if not modules then modules = { } end modules ['toks-ini'] = { version = 1.001, author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } tokens = tokens or { } local tokens = tokens local token = token -- the built in one local next = next local tonumber = tonumber local tostring = tostring local utfchar = utf.char local char = string.char local printtable = table.print local concat = table.concat local format = string.format if token.commands then local commands = token.commands() tokens.commands = utilities.storage.allocate(table.swapped(commands,commands)) else tokens.commands = { } end local scan_toks = token.scan_toks local scan_string = token.scan_string local scan_argument = token.scan_argument local scan_tokenlist = token.scan_tokenlist local scan_int = token.scan_int local scan_code = token.scan_code local scan_token_code = token.scan_token_code local scan_dimen = token.scan_dimen local scan_glue = token.scan_glue local scan_keyword = token.scan_keyword local scan_keyword_cs = token.scan_keyword_cs or scan_keyword local scan_token = token.scan_token local scan_list = token.scan_list local scan_word = token.scan_word local scan_key = token.scan_key local scan_value = token.scan_value local scan_char = token.scan_char local scan_number = token.scan_number local scan_csname = token.scan_csname local scan_real = token.scan_real local scan_float = token.scan_float local get_next = token.get_next local get_next_token = token.get_next_token local skip_next = token.skip_next local peek_next_char = token.peek_next_char local is_next_char = token.is_next_char local set_macro = token.set_macro local get_macro = token.get_macro local get_meaning = token.get_meaning local get_cmdname = token.get_cmdname local set_char = token.set_char local set_lua = token.set_lua local create_token = token.create local new_token = token.new local is_defined = token.is_defined local is_token = token.is_token tokens.new = new_token tokens.create = create_token tokens.istoken = is_token tokens.isdefined = is_defined tokens.defined = is_defined local bits = { escape = 0x00000001, -- 2^00 begingroup = 0x00000002, -- 2^01 endgroup = 0x00000004, -- 2^02 mathshift = 0x00000008, -- 2^03 alignment = 0x00000010, -- 2^04 endofline = 0x00000020, -- 2^05 parameter = 0x00000040, -- 2^06 superscript = 0x00000080, -- 2^07 subscript = 0x00000100, -- 2^08 ignore = 0x00000200, -- 2^09 space = 0x00000400, -- 2^10 -- 1024 letter = 0x00000800, -- 2^11 other = 0x00001000, -- 2^12 active = 0x00002000, -- 2^13 comment = 0x00004000, -- 2^14 invalid = 0x00008000, -- 2^15 -- character = 0x00001800, -- 2^11 + 2^12 whitespace = 0x00002400, -- 2^13 + 2^10 -- / needs more checking -- open = 0x00000402, -- 2^10 + 2^01 -- space + begingroup close = 0x00000404, -- 2^10 + 2^02 -- space + endgroup } -- for k, v in next, bits do bits[v] = k end tokens.bits = bits -- words are space or \relax terminated and the trailing space is gobbled; a word -- can contain any non-space letter/other (see archive for implementation in lua) if not scan_number then scan_number = function(base) local s = scan_word() if not s then return nil elseif base then return tonumber(s,base) else return tonumber(s) end end end local function scan_boolean() local kw = scan_word() if kw == "true" then return true elseif kw == "false" then return false else return nil end end local function scan_verbatim() return scan_argument(false) end tokens.scanners = { -- these expand token = scan_token, toks = scan_toks, tokens = scan_toks, list = scan_list, dimen = scan_dimen, dimension = scan_dimen, glue = scan_glue, skip = scan_glue, integer = scan_int, real = scan_real, float = scan_float, count = scan_int, string = scan_string, argument = scan_argument, tokenlist = scan_tokenlist, verbatim = scan_verbatim, code = scan_code, tokencode = scan_token_code, word = scan_word, key = scan_key, value = scan_value, char = scan_char, number = scan_number, boolean = scan_boolean, keyword = scan_keyword, keywordcs = scan_keyword_cs, csname = scan_csname, peek = peek_next_char, skip = skip_next, ischar = is_next_char, } tokens.getters = { -- these don't expand meaning = get_meaning, macro = get_macro, token = get_next, count = tex.getcount, dimen = tex.getdimen, skip = tex.getglue, glue = tex.getglue, skip = tex.getmuglue, glue = tex.getmuglue, box = tex.getbox, } tokens.setters = { macro = set_macro, char = set_char, lua = set_lua, count = tex.setcount, dimen = tex.setdimen, skip = tex.setglue, glue = tex.setglue, skip = tex.setmuglue, glue = tex.setmuglue, box = tex.setbox, } -- static int run_scan_token(lua_State * L) -- { -- saved_tex_scanner texstate; -- save_tex_scanner(texstate); -- get_x_token(); -- make_new_token(L, cur_cmd, cur_chr, cur_cs); -- unsave_tex_scanner(texstate); -- return 1; -- } -- -- static int run_get_future(lua_State * L) -- { -- /* saved_tex_scanner texstate; */ -- /* save_tex_scanner(texstate); */ -- get_token(); -- make_new_token(L, cur_cmd, cur_chr, cur_cs); -- back_input(); -- /* unsave_tex_scanner(texstate); */ -- return 1; -- } if setinspector then local simple = { letter = "letter", other_char = "other" } local function astable(t) if t and is_token(t) then local cmdname = t.cmdname local simple = simple[cmdname] if simple then return { category = simple, character = utfchar(t.mode) or nil, } else return { command = t.command, id = t.id, tok = t.tok, csname = t.csname, active = t.active, expandable = t.expandable, protected = t.protected, frozen = t.frozen, mode = t.mode, index = t.index, cmdname = cmdname, } end end end tokens.astable = astable setinspector("token",function(v) local t = astable(v) if t then printtable(t,tostring(v)) return true end end) end tokens.cache = table.setmetatableindex(function(t,k) if not is_defined(k) then set_macro(k,"","global") end local v = create_token(k) t[k] = v return v end)