diff options
Diffstat (limited to 'tex/context/base/toks-ini.lua')
| -rw-r--r-- | tex/context/base/toks-ini.lua | 396 |
1 files changed, 207 insertions, 189 deletions
diff --git a/tex/context/base/toks-ini.lua b/tex/context/base/toks-ini.lua index 642f85d94..a6c465d50 100644 --- a/tex/context/base/toks-ini.lua +++ b/tex/context/base/toks-ini.lua @@ -8,232 +8,250 @@ if not modules then modules = { } end modules ['toks-ini'] = { tokens = tokens or { } local tokens = tokens +local newtoken = newtoken or token local tostring = tostring local utfchar = utf.char local char = string.char local printtable = table.print local concat = table.concat -if newtoken then - - if setinspector then - - local istoken = newtoken.is_token - local simple = { letter = "letter", other_char = "other" } - - local function astable(t) - if t and istoken(t) then - local cmdname = t.cmdname - local simple = simple[cmdname] - if simple then - return { - category = simple, - character = utfchar(t.mode) or nil, - } - else - return { - command = t.command, - id = t.id, - tok = t.tok, - csname = t.csname, - active = t.active, - expandable = t.expandable, - protected = t.protected, - mode = t.mode, - cmdname = cmdname, - } - end +if setinspector then + + local istoken = newtoken.is_token + local simple = { letter = "letter", other_char = "other" } + + local function astable(t) + if t and istoken(t) then + local cmdname = t.cmdname + local simple = simple[cmdname] + if simple then + return { + category = simple, + character = utfchar(t.mode) or nil, + } + else + return { + command = t.command, + id = t.id, + tok = t.tok, + csname = t.csname, + active = t.active, + expandable = t.expandable, + protected = t.protected, + mode = t.mode, + index = t.index, + cmdname = cmdname, + } end end + end - tokens.istoken = istoken - tokens.astable = astable + tokens.istoken = istoken + tokens.astable = astable - setinspector("token",function(v) if istoken(v) then printtable(astable(v),tostring(v)) return true end end) + setinspector("token",function(v) if istoken(v) then printtable(astable(v),tostring(v)) return true end end) +end + +local scan_toks = newtoken.scan_toks +local scan_string = newtoken.scan_string +local scan_int = newtoken.scan_int +local scan_code = newtoken.scan_code +local scan_dimen = newtoken.scan_dimen +local scan_glue = newtoken.scan_glue +local scan_keyword = newtoken.scan_keyword +local scan_token = newtoken.scan_token +local scan_word = newtoken.scan_word +local scan_number = newtoken.scan_number +local scan_csname = newtoken.scan_csname + +local get_next = newtoken.get_next + +local set_macro = newtoken.set_macro + +set_macro = function(k,v,g) + if g == "global" then + context.setgvalue(k,v or '') + else + context.setvalue(k,v or '') end +end + +local bits = { + escape = 2^ 0, + begingroup = 2^ 1, + endgroup = 2^ 2, + mathshift = 2^ 3, + alignment = 2^ 4, + endofline = 2^ 5, + parameter = 2^ 6, + superscript = 2^ 7, + subscript = 2^ 8, + ignore = 2^ 9, + space = 2^10, -- 1024 + letter = 2^11, + other = 2^12, + active = 2^13, + comment = 2^14, + invalid = 2^15, + -- + character = 2^11 + 2^12, + whitespace = 2^13 + 2^10, -- / needs more checking + -- + open = 2^10 + 2^1, -- space + begingroup + close = 2^10 + 2^2, -- space + endgroup +} - local scan_toks = newtoken.scan_toks - local scan_string = newtoken.scan_string - local scan_int = newtoken.scan_int - local scan_code = newtoken.scan_code - local scan_dimen = newtoken.scan_dimen - local scan_glue = newtoken.scan_glue - local scan_keyword = newtoken.scan_keyword - local scan_token = newtoken.scan_token - local scan_word = newtoken.scan_word - local scan_number = newtoken.scan_number +-- for k, v in next, bits do bits[v] = k end - local get_next = newtoken.get_next +tokens.bits = bits - local set_macro = newtoken.set_macro +local space_bits = bits.space - set_macro = function(k,v,g) - if g == "global" then - context.setgvalue(k,v or '') +-- words are space or \relax terminated and the trailing space is gobbled; a word +-- can contain any non-space letter/other + +local t = { } -- small optimization, a shared variable that is not reset + +if scan_word then + + scan_number = function(base) + local s = scan_word() + if not s then + return nil + elseif base then + return tonumber(s,base) else - context.setvalue(k,v or '') + return tonumber(s) end end - local bits = { - escape = 2^ 0, - begingroup = 2^ 1, - endgroup = 2^ 2, - mathshift = 2^ 3, - alignment = 2^ 4, - endofline = 2^ 5, - parameter = 2^ 6, - superscript = 2^ 7, - subscript = 2^ 8, - ignore = 2^ 9, - space = 2^10, -- 1024 - letter = 2^11, - other = 2^12, - active = 2^13, - comment = 2^14, - invalid = 2^15, - -- - character = 2^11 + 2^12, - whitespace = 2^13 + 2^10, -- / needs more checking - -- - open = 2^10 + 2^1, -- space + begingroup - close = 2^10 + 2^2, -- space + endgroup - } - - -- for k, v in next, bits do bits[v] = k end - - tokens.bits = bits - - local space_bits = bits.space - - -- words are space or \relax terminated and the trailing space is gobbled; a word - -- can contain any non-space letter/other - - local t = { } -- small optimization, a shared variable that is not reset - - if scan_word then - - scan_number = function(base) - local s = scan_word() - if not s then - return nil - elseif base then - return tonumber(s,base) - else - return tonumber(s) - end - end - - else - - scan_word = function() - local n = 0 - while true do - local c = scan_code() - if c then - n = n + 1 - t[n] = utfchar(c) - elseif scan_code(space_bits) then - if n > 0 then - break - end - elseif n > 0 then +else + + scan_word = function() + local n = 0 + while true do + local c = scan_code() + if c then + n = n + 1 + t[n] = utfchar(c) + elseif scan_code(space_bits) then + if n > 0 then break - else - return end + elseif n > 0 then + break + else + return end - return concat(t,"",1,n) end + return concat(t,"",1,n) + end - -- so we gobble the space (like scan_int) (number has to be space or non-char terminated - -- as we accept 0xabcd and such so there is no clear separator for a keyword - - scan_number = function(base) - local n = 0 - while true do - local c = scan_code() - if c then - n = n + 1 - t[n] = char(c) - elseif scan_code(space_bits) then - if n > 0 then - break - end - elseif n > 0 then + -- so we gobble the space (like scan_int) (number has to be space or non-char terminated + -- as we accept 0xabcd and such so there is no clear separator for a keyword + + scan_number = function(base) + local n = 0 + while true do + local c = scan_code() + if c then + n = n + 1 + t[n] = char(c) + elseif scan_code(space_bits) then + if n > 0 then break - else - return end - end - local s = concat(t,"",1,n) - if base then - return tonumber(s,base) + elseif n > 0 then + break else - return tonumber(s) + return end end - - end - - -- -- the next one cannot handle \iftrue true\else false\fi - -- - -- local function scan_boolean() - -- if scan_keyword("true") then - -- return true - -- elseif scan_keyword("false") then - -- return false - -- else - -- return nil - -- end - -- end - - local function scan_boolean() - local kw = scan_word() - if kw == "true" then - return true - elseif kw == "false" then - return false + local s = concat(t,"",1,n) + if base then + return tonumber(s,base) else - return nil + return tonumber(s) end end - tokens.scanners = { -- these expand - token = scan_token or get_next, - toks = scan_toks, - tokens = scan_toks, - dimen = scan_dimen, - dimension = scan_dimen, - glue = scan_glue, - skip = scan_glue, - integer = scan_int, - count = scan_int, - string = scan_string, - code = scan_code, - word = scan_word, - number = scan_number, - boolean = scan_boolean, - keyword = scan_keyword, - } - - tokens.getters = { -- these don't expand - token = get_next, - count = tex.getcount, - dimen = tex.getdimen, - box = tex.getbox, - } - - tokens.setters = { - macro = set_macro, - count = tex.setcount, - dimen = tex.setdimen, - box = tex.setbox, - } +end +-- -- the next one cannot handle \iftrue true\else false\fi +-- +-- local function scan_boolean() +-- if scan_keyword("true") then +-- return true +-- elseif scan_keyword("false") then +-- return false +-- else +-- return nil +-- end +-- end + +local function scan_boolean() + local kw = scan_word() + if kw == "true" then + return true + elseif kw == "false" then + return false + else + return nil + end end +if not scan_csname then + + scan_csname = function() + local t = get_next() + local n = t.csname + return n ~= "" and n or nil + end + +end + +tokens.scanners = { -- these expand + token = scan_token, + toks = scan_toks, + tokens = scan_toks, + dimen = scan_dimen, + dimension = scan_dimen, + glue = scan_glue, + skip = scan_glue, + integer = scan_int, + count = scan_int, + string = scan_string, + code = scan_code, + word = scan_word, + number = scan_number, + boolean = scan_boolean, + keyword = scan_keyword, + csname = scan_csname, +} + +tokens.getters = { -- these don't expand + token = get_next, + count = tex.getcount, + dimen = tex.getdimen, + skip = tex.getglue, + glue = tex.getglue, + skip = tex.getmuglue, + glue = tex.getmuglue, + box = tex.getbox, +} + +tokens.setters = { + macro = set_macro, + count = tex.setcount, + dimen = tex.setdimen, + skip = tex.setglue, + glue = tex.setglue, + skip = tex.setmuglue, + glue = tex.setmuglue, + box = tex.setbox, +} + -- static int run_scan_token(lua_State * L) -- { -- saved_tex_scanner texstate; |