diff options
Diffstat (limited to 'tex/context/base/toks-ini.lua')
| -rw-r--r-- | tex/context/base/toks-ini.lua | 524 |
1 files changed, 219 insertions, 305 deletions
diff --git a/tex/context/base/toks-ini.lua b/tex/context/base/toks-ini.lua index 0f0c016f8..f8b945890 100644 --- a/tex/context/base/toks-ini.lua +++ b/tex/context/base/toks-ini.lua @@ -5,338 +5,252 @@ if not modules then modules = { } end modules ['toks-ini'] = { license = "see context related readme files" } -local context, commands = context, commands -local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values -local format, gsub = string.format, string.gsub - ---[[ldx-- -<p>This code is experimental and needs a cleanup. The visualizers will move to -a module.</p> ---ldx]]-- - --- 1 = command, 2 = modifier (char), 3 = controlsequence id --- --- callback.register('token_filter', token.get_next) --- --- token.get_next() --- token.expand() --- token.create() --- token.csname_id() --- token.csname_name(v) --- token.command_id() --- token.command_name(v) --- token.is_expandable() --- token.is_activechar() --- token.lookup(v) - --- actually, we can use token registers to store tokens - -local token, tex = token, tex - -local createtoken = token.create -local csname_id = token.csname_id -local command_id = token.command_id -local command_name = token.command_name -local get_next = token.get_next -local expand = token.expand -local is_activechar = token.is_activechar -local csname_name = token.csname_name - -tokens = tokens or { } -local tokens = tokens - -tokens.vbox = createtoken("vbox") -tokens.hbox = createtoken("hbox") -tokens.vtop = createtoken("vtop") -tokens.bgroup = createtoken(utfbyte("{"), 1) -tokens.egroup = createtoken(utfbyte("}"), 2) - -tokens.letter = function(chr) return createtoken(utfbyte(chr), 11) end -tokens.other = function(chr) return createtoken(utfbyte(chr), 12) end - -tokens.letters = function(str) - local t, n = { }, 0 - for chr in utfvalues(str) do - n = n + 1 - t[n] = createtoken(chr, 11) - end - return t -end - -tokens.collectors = tokens.collectors or { } -local collectors = tokens.collectors - -collectors.data = collectors.data or { } -local collectordata = collectors.data +tokens = tokens or { } + +local tokens = tokens +local tostring = tostring +local utfchar = utf.char +local char = string.char +local printtable = table.print +local concat = table.concat + +if newtoken then + + if setinspector then + + local istoken = newtoken.is_token + local simple = { letter = "letter", other_char = "other" } + + local function astable(t) + if t and istoken(t) then + local cmdname = t.cmdname + local simple = simple[cmdname] + if simple then + return { + category = simple, + character = utfchar(t.mode) or nil, + } + else + return { + command = t.command, + id = t.id, + tok = t.tok, + csname = t.csname, + active = t.active, + expandable = t.expandable, + protected = t.protected, + mode = t.mode, + cmdname = cmdname, + } + end + end + end -collectors.registered = collectors.registered or { } -local registered = collectors.registered + tokens.istoken = istoken + tokens.astable = astable -local function printlist(data) - callbacks.push('token_filter', function () - callbacks.pop('token_filter') -- tricky but the nil assignment helps - return data - end) -end + setinspector(function(v) if istoken(v) then printtable(astable(v),tostring(v)) return true end end) -tex.printlist = printlist -- will change to another namespace + end -function collectors.flush(tag) - printlist(collectordata[tag]) -end + local scan_toks = newtoken.scan_toks + local scan_string = newtoken.scan_string + local scan_int = newtoken.scan_int + local scan_code = newtoken.scan_code + local scan_dimen = newtoken.scan_dimen + local scan_glue = newtoken.scan_glue + local scan_keyword = newtoken.scan_keyword + local scan_token = newtoken.scan_token + local scan_word = newtoken.scan_word + local scan_number = newtoken.scan_number -function collectors.test(tag) - printlist(collectordata[tag]) -end + local get_next = newtoken.get_next -function collectors.register(name) - registered[csname_id(name)] = name -end + local set_macro = newtoken.set_macro -local call = command_id("call") -local letter = command_id("letter") -local other = command_id("other_char") - -function collectors.install(tag,end_cs) - local data, d = { }, 0 - collectordata[tag] = data - local endcs = csname_id(end_cs) - while true do - local t = get_next() - local a, b = t[1], t[3] - if b == endcs then - context["end_cs"]() - return - elseif a == call and registered[b] then - expand() + set_macro = function(k,v,g) + if g == "global" then + context.setgvalue(k,v or '') else - d = d + 1 - data[d] = t + context.setvalue(k,v or '') end end -end - -function collectors.handle(tag,handle,flush) - collectordata[tag] = handle(collectordata[tag]) - if flush then - collectors.flush(tag) - end -end -local show_methods = { } -collectors.show_methods = show_methods + local bits = { + escape = 2^ 0, + begingroup = 2^ 1, + endgroup = 2^ 2, + mathshift = 2^ 3, + alignment = 2^ 4, + endofline = 2^ 5, + parameter = 2^ 6, + superscript = 2^ 7, + subscript = 2^ 8, + ignore = 2^ 9, + space = 2^10, -- 1024 + letter = 2^11, + other = 2^12, + active = 2^13, + comment = 2^14, + invalid = 2^15, + -- + character = 2^11 + 2^12, + whitespace = 2^13 + 2^10, -- / needs more checking + -- + open = 2^10 + 2^1, -- space + begingroup + close = 2^10 + 2^2, -- space + endgroup + } + + -- for k, v in next, bits do bits[v] = k end + + tokens.bits = bits + + local space_bits = bits.space + + -- words are space or \relax terminated and the trailing space is gobbled; a word + -- can contain any non-space letter/other + + local t = { } -- small optimization, a shared variable that is not reset + + if scan_word then + + scan_number = function(base) + local s = scan_word() + if not s then + return nil + elseif base then + return tonumber(s,base) + else + return tonumber(s) + end + end -function collectors.show(tag, method) - if type(tag) == "table" then - show_methods[method or 'a'](tag) else - show_methods[method or 'a'](collectordata[tag]) - end -end - -function collectors.defaultwords(t,str) - local n = #t - n = n + 1 - t[n] = tokens.bgroup - n = n + 1 - t[n] = createtoken("red") - for i=1,#str do - n = n + 1 - t[n] = tokens.other('*') - end - n = n + 1 - t[n] = tokens.egroup -end -function collectors.dowithwords(tag,handle) - local t, w, tn, wn = { }, { }, 0, 0 - handle = handle or collectors.defaultwords - local tagdata = collectordata[tag] - for k=1,#tagdata do - local v = tagdata[k] - if v[1] == letter then - wn = wn + 1 - w[wn] = v[2] - else - if wn > 0 then - handle(t,w) - wn = 0 + scan_word = function() + local n = 0 + while true do + local c = scan_code() + if c then + n = n + 1 + t[n] = utfchar(c) + elseif scan_code(space_bits) then + if n > 0 then + break + end + elseif n > 0 then + break + else + return + end end - tn = tn + 1 - t[tn] = v + return concat(t,"",1,n) end - end - if wn > 0 then - handle(t,w) - end - collectordata[tag] = t -end -local function showtoken(t) - if t then - local cmd, chr, id, cs, name = t[1], t[2], t[3], nil, command_name(t) or "" - if cmd == letter or cmd == other then - return format("%s-> %s -> %s", name, chr, utfchar(chr)) - elseif id > 0 then - cs = csname_name(t) or nil - if cs then - return format("%s-> %s", name, cs) - elseif tonumber(chr) < 0 then - return format("%s-> %s", name, id) + -- so we gobble the space (like scan_int) (number has to be space or non-char terminated + -- as we accept 0xabcd and such so there is no clear separator for a keyword + + scan_number = function(base) + local n = 0 + while true do + local c = scan_code() + if c then + n = n + 1 + t[n] = char(c) + elseif scan_code(space_bits) then + if n > 0 then + break + end + elseif n > 0 then + break + else + return + end + end + local s = concat(t,"",1,n) + if base then + return tonumber(s,base) else - return format("%s-> (%s,%s)", name, chr, id) + return tonumber(s) end - else - return format("%s", name) end - else - return "no node" - end -end - -collectors.showtoken = showtoken - -function collectors.trace() - local t = get_next() - logs.report("tokenlist",showtoken(t)) - return t -end - --- these might move to a runtime module -show_methods.a = function(data) -- no need to store the table, just pass directly - local function row(one,two,three,four,five) - context.NC() context(one) - context.NC() context(two) - context.NC() context(three) - context.NC() context(four) - context.NC() context(five) - context.NC() context.NR() end - context.starttabulate { "|T|Tr|cT|Tr|T|" } - row("cmd","chr","","id","name") - context.HL() - for _,v in next, data do - local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", "" - local name = gsub(command_name(v) or "","_","\\_") - if id > 0 then - cs = csname_name(v) or "" - if cs ~= "" then cs = "\\string " .. cs end - else - id = "" - end - if cmd == letter or cmd == other then - sym = "\\char " .. chr - end - if tonumber(chr) < 0 then - row(name,"",sym,id,cs) - else - row(name,chr,sym,id,cs) - end - end - context.stoptabulate() -end -local function show_b_c(data,swap) -- no need to store the table, just pass directly - local function row(one,two,three) - context.NC() context(one) - context.NC() context(two) - context.NC() context(three) - context.NC() context.NR() - end - if swap then - context.starttabulate { "|Tl|Tl|Tr|" } - else - context.starttabulate { "|Tl|Tr|Tl|" } - end - row("cmd","chr","name") - context.HL() - for _,v in next, data do - local cmd, chr, id, cs, sym = v[1], v[2], v[3], "", "" - local name = gsub(command_name(v) or "","_","\\_") - if id > 0 then - cs = csname_name(v) or "" - end - if cmd == letter or cmd == other then - sym = "\\char " .. chr - elseif cs == "" then - -- okay - elseif is_activechar(v) then - sym = "\\string " .. cs + -- -- the next one cannot handle \iftrue true\else false\fi + -- + -- local function scan_boolean() + -- if scan_keyword("true") then + -- return true + -- elseif scan_keyword("false") then + -- return false + -- else + -- return nil + -- end + -- end + + local function scan_boolean() + local kw = scan_word() + if kw == "true" then + return true + elseif kw == "false" then + return false else - sym = "\\string\\" .. cs - end - if swap then - row(name,sym,chr) - elseif tonumber(chr) < 0 then - row(name,"",sym) - else - row(name,chr,sym) + return nil end end - context.stoptabulate() -end --- Even more experimental ... + tokens.scanners = { -- these expand + token = scan_token or get_next, + toks = scan_toks, + tokens = scan_toks, + dimen = scan_dimen, + dimension = scan_dimen, + glue = scan_glue, + skip = scan_glue, + integer = scan_int, + count = scan_int, + string = scan_string, + code = scan_code, + word = scan_word, + number = scan_number, + boolean = scan_boolean, + keyword = scan_keyword, + } + + tokens.getters = { -- these don't expand + token = get_next, + count = tex.getcount, + dimen = tex.getdimen, + box = tex.getbox, + } + + tokens.setters = { + macro = set_macro, + count = tex.setcount, + dimen = tex.setdimen, + box = tex.setbox, + } -show_methods.b = function(data) show_b_c(data,false) end -show_methods.c = function(data) show_b_c(data,true ) end - -local remapper = { } -- namespace -collectors.remapper = remapper - -local remapperdata = { } -- user mappings -remapper.data = remapperdata - -function remapper.store(tag,class,key) - local s = remapperdata[class] - if not s then - s = { } - remapperdata[class] = s - end - s[key] = collectordata[tag] - collectordata[tag] = nil end -function remapper.convert(tag,toks) - local data = remapperdata[tag] - local leftbracket, rightbracket = utfbyte('['), utfbyte(']') - local skipping = 0 - -- todo: math - if data then - local t, n = { }, 0 - for s=1,#toks do - local tok = toks[s] - local one, two = tok[1], tok[2] - if one == 11 or one == 12 then - if two == leftbracket then - skipping = skipping + 1 - n = n + 1 ; t[n] = tok - elseif two == rightbracket then - skipping = skipping - 1 - n = n + 1 ; t[n] = tok - elseif skipping == 0 then - local new = data[two] - if new then - if #new > 1 then - for n=1,#new do - n = n + 1 ; t[n] = new[n] - end - else - n = n + 1 ; t[n] = new[1] - end - else - n = n + 1 ; t[n] = tok - end - else - n = n + 1 ; t[n] = tok - end - else - n = n + 1 ; t[n] = tok - end - end - return t - else - return toks - end -end +-- static int run_scan_token(lua_State * L) +-- { +-- saved_tex_scanner texstate; +-- save_tex_scanner(texstate); +-- get_x_token(); +-- make_new_token(L, cur_cmd, cur_chr, cur_cs); +-- unsave_tex_scanner(texstate); +-- return 1; +-- } +-- +-- static int run_get_future(lua_State * L) +-- { +-- /* saved_tex_scanner texstate; */ +-- /* save_tex_scanner(texstate); */ +-- get_token(); +-- make_new_token(L, cur_cmd, cur_chr, cur_cs); +-- back_input(); +-- /* unsave_tex_scanner(texstate); */ +-- return 1; +-- } |