diff options
Diffstat (limited to 'tex/context/base/mkiv/toks-scn.lmt')
| -rw-r--r-- | tex/context/base/mkiv/toks-scn.lmt | 592 |
1 files changed, 0 insertions, 592 deletions
diff --git a/tex/context/base/mkiv/toks-scn.lmt b/tex/context/base/mkiv/toks-scn.lmt deleted file mode 100644 index 93e0af09a..000000000 --- a/tex/context/base/mkiv/toks-scn.lmt +++ /dev/null @@ -1,592 +0,0 @@ -if not modules then modules = { } end modules ['toks-scn'] = { - version = 1.001, - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- Writing this kind of code (and completing the newtoken code base) is fun. I did --- so with the brilliant film music from The Girl with the Dragon Tattoo running in a --- loop in the background (three cd's by Trent Reznor and Atticus Ross). An alien --- feeling helps with alien code. - --- todo: more \let's at the tex end - -local type, next, tostring, tonumber = type, next, tostring, tonumber - -local formatters = string.formatters -local concat = table.concat - -local scanners = tokens.scanners -local tokenbits = tokens.bits - -local scanstring = scanners.string -local scanargument = scanners.argument -local scandelimited = scanners.delimited -- lmtx -local scanverbatim = scanners.verbatim -local scantokenlist = scanners.tokenlist -local scantoks = scanners.toks -local scaninteger = scanners.integer -local scancardinal = scanners.cardinal -local scannumber = scanners.number -local scankeyword = scanners.keyword -local scankeywordcs = scanners.keywordcs -local scanword = scanners.word -local scanletters = scanners.letters -local scankey = scanners.key -local scancode = scanners.code -local scanboolean = scanners.boolean -local scandimen = scanners.dimen -local scanglue = scanners.glue -local scangluevalues = scanners.gluevalues -local scangluespec = scanners.gluespec -local scancsname = scanners.csname -local scanintegerargument = scanners.integerargument -local scandimenargument = scanners.dimenargument - -local todimen = number.todimen -local toboolean = toboolean - -local lpegmatch = lpeg.match -local p_unquoted = lpeg.Cs(lpeg.patterns.unquoted) - -local trace_compile = false trackers.register("tokens.compile", function(v) trace_compile = v end) -local report_compile = logs.reporter("tokens","compile") -local report_scan = logs.reporter("tokens","scan") - -local open = tokenbits.open -local close = tokenbits.close - -local function scanopen() - while true do - local c = scancode(open) - if c == 123 then - return true - -- elseif c ~= 32 then - elseif not c then - return - end - end -end - -local function scanclose() - while true do - local c = scancode(close) - if c == 125 then - return true - -- elseif c ~= 32 then - elseif not c then - return - end - end -end - -scanners.scanopen = scanopen -scanners.scanclose = scanclose - -local function scanlist() - local wrapped = scanopen() - local list = { } - local size = 0 - while true do - local entry = scanstring() - if entry then - size = size + 1 - list[size] = entry - else - break - end - end - if wrapped then - scanclose() - end - return list -end - -local function scanconditional() - local kw = scanword() - if kw == "true" then - return true - end - if kw == "false" then - return false - end - local c = scaninteger() - if c then - return c == 0 -- with a conditional 0=true - end - return nil -end - -local function scantable(t,data) - if not data then - data = { } - end - if t then - local wrapped = scanopen() - while true do - local key = scanword(true) - if key then - local get = t[key] - if get then - data[key] = get() - else - -- catch all we can get - end - else - break - end - end - if wrapped then - scanclose() - end - end - return data -end - -function tokens.constant(s) - if type(s) == "string" then - return "'" .. s .. "'" - else - return s - end -end - -scanners.list = scanlist -scanners.table = scantable -scanners.conditional = scanconditional - -function scanners.whd() - local width, height, depth - while true do - if scankeyword("width") then - width = scandimen() - elseif scankeyword("height") then - height = scandimen() - elseif scankeyword("depth") then - depth = scandimen() - else - break - end - end - if width or height or depth then - return width or 0, height or 0, depth or 0 - else - -- we inherit - end -end - --- begin lmtx - -local l = utf.byte("[") -local r = utf.byte("]") - -local function scanbracketed() - local s = scandelimited(l, r) - if s then - return s - else - local readstate = status.getreadstate() - report_scan("missing argument in line %i of %a", readstate.linenumber, readstate.filename) - return "" - end -end - -local function scanoptional() - return scandelimited(l, r) or "" -end - -local function scanbracketedasis() - return scandelimited(l, r, false) -end - -local function scanargumentasis() - return scanargument(false) -end - -scanners.bracketed = scanbracketed -scanners.optional = scanoptional -scanners.bracketedasis = scanbracketedasis -scanners.argumentasis = scanargumentasis - --- end lmtx - -local shortcuts = { - tokens = tokens, - bits = tokenbits, - open = open, - close = close, - scanners = scanners, - scanstring = scanstring, - scanargument = scanargument, - scanverbatim = scanverbatim, - scantokenlist = scantokenlist, - scantoks = scantoks, - scaninteger = scaninteger, - scancardinal = scancardinal, - scannumber = scannumber, - scantable = scantable, -- not directly useable - scankeyword = scankeyword, - scankeywordcs = scankeywordcs, - scanword = scanword, - scanletters = scanletters, - -- scankey = scankey, - scancode = scancode, - scanboolean = scanboolean, - scanglue = scanglue, -- list - scangluespec = scangluespec, - scangluevalues = scangluevalues, - scandimen = scandimen, - scandimension = scandimen, - scanbox = scanners.box, - scanhbox = scanners.hbox, - scanvbox = scanners.vbox, - scanvtop = scanners.vtop, - scanconditional = scanconditional, - scanopen = scanopen, - scanclose = scanclose, - scanlist = scanlist, - scancsname = scancsname, - todimen = todimen, - tonumber = tonumber, - tostring = tostring, - toboolean = toboolean, - inspect = inspect, - report = report_scan, - -- lmtx - scandelimited = scandelimited, -- not directly useable - scanbracketed = scanbracketed, - scanoptional = scanoptional, - scanbracketedasis = scanbracketedasis, - scanargumentasis = scanargumentasis, - -- - scanintegerargument = scanintegerargument, - scandimenargument = scandimenargument, -} - -tokens.shortcuts = shortcuts - -local load = load -local dump = string.dump - -local function loadstripped(code) - return load(code,nil,nil,shortcuts) - -- return load(dump(load(code),true),nil,nil,shortcuts) -end - -tokens.converters = { - tonumber = "tonumber", - tostring = "tostring", - toboolean = "toboolean", - todimen = "todimen", - toglue = "todimen", -} - --- We could just pickup a keyword but then we really need to make sure that no number --- follows it when that is the assignment and adding an optional = defeats the gain --- in speed. Currently we have sources with no spaces (\startcontextdefinitioncode --- ...) so it fails there. --- --- Another drawback is that we then need to use { } instead of ending with \relax (as --- we can do now) but that is no big deal. It's just that I then need to check the TeX --- end. More pain than gain and a bit risky too. Using scanletters works better, but --- the gain is only some 10 percent but if we don't have keywords with numbers it might --- make sense in the end, some day. - -local f_if = formatters[ " if scankeywordcs('%s') then data['%s'] = scan%s()"] -local f_elseif = formatters[" elseif scankeywordcs('%s') then data['%s'] = scan%s()"] - ------ f_if_x = formatters[ " if not data['%s'] and scankeywordcs('%s') then data['%s'] = scan%s()"] ------ f_elseif_x = formatters[" elseif not data['%s'] and scankeywordcs('%s') then data['%s'] = scan%s()"] - --- if CONTEXTLMTXMODE > 0 then --- f_if = formatters[" local key = scanletters() if key == '' then break elseif key == '%s' then data['%s'] = scan%s()"] --- f_elseif = formatters[" elseif key == '%s' then data['%s'] = scan%s()"] --- end - -local f_local = formatters["local scan%s = scanners.%s"] -local f_scan = formatters["scan%s()"] -local f_shortcut = formatters["local %s = scanners.converters.%s"] - -local f_if_c = formatters[ " if scankeywordcs('%s') then data['%s'] = %s(scan%s())"] -local f_elseif_c = formatters[" elseif scankeywordcs('%s') then data['%s'] = %s(scan%s())"] -local f_scan_c = formatters["%s(scan%s())"] - --- see above - --- if CONTEXTLMTXMODE > 0 then --- f_if_c = formatters[" local key = scanletters() if key == '' then break elseif key == '%s' then data['%s'] = %s(scan%s())"] --- f_elseif_c = formatters[" elseif k == '%s' then data['%s'] = %s(scan%s())"] --- end - -local f_any = formatters[" else local key = scanword(true) if key then data[key] = scan%s() else break end end"] -local f_any_c = formatters[" else local key = scanword(true) if key then data[key] = %s(scan%s()) else break end end"] -local s_done = " else break end" - -local f_any_all = formatters[" local key = scanword(true) if key then data[key] = scan%s() else break end"] -local f_any_all_c= formatters[" local key = scanword(true) if key then data[key] = %s(scan%s()) else break end"] - -local f_table = formatters["%\nt\nreturn function()\n local data = { }\n%s\n return %s\nend\n"] -local f_sequence = formatters["%\nt\n%\nt\n%\nt\nreturn function()\n return %s\nend\n"] -local f_singular = formatters["%\nt\n%\nt\n\nreturn function(%s)\n return %s\nend\n"] -local f_simple = formatters["%\nt\nreturn function()\n return %s\nend\n"] -local f_string = formatters["%q"] -local f_action_f = formatters["action%s(%s)"] -local f_action_s = formatters["local action%s = tokens._action[%s]"] -local f_nested = formatters["local function scan%s()\n local data = { }\n%s\n return data\nend\n"] - -local f_check = formatters[ [[ - local wrapped = scanopen() - while true do - ]] .. "%\nt\n" .. [[ - %s - end - if wrapped then - scanclose() - end -]] ] - --- using these shortcuts saves temporary small tables (okay, it looks uglier) - -local presets = { - ["1 string" ] = { "string" }, - ["2 strings"] = { "string", "string" }, - ["3 strings"] = { "string", "string", "string" }, - ["4 strings"] = { "string", "string", "string", "string" }, - ["5 strings"] = { "string", "string", "string", "string", "string" }, - ["6 strings"] = { "string", "string", "string", "string", "string", "string" }, - ["7 strings"] = { "string", "string", "string", "string", "string", "string", "string" }, - ["8 strings"] = { "string", "string", "string", "string", "string", "string", "string", "string" }, - - ["1 argument" ] = { "argument" }, - ["2 arguments"] = { "argument", "argument" }, - ["3 arguments"] = { "argument", "argument", "argument" }, - ["4 arguments"] = { "argument", "argument", "argument", "argument" }, -} - -tokens.presets = presets - -function tokens.compile(specification) - local f = { } - local n = 0 - local c = { } - local t = specification.arguments or specification - local a = specification.actions or nil - if type(a) == "function" then - a = { a } - end - local code - local args - local function compile(t,nested) - local done = s_done - local r = { } - local m = 0 - for i=1,#t do - local ti = t[i] - if ti == "*" and i == 1 then - done = f_any_all("string") - else - local t1 = ti[1] - local t2 = ti[2] or "string" - if type(t2) == "table" then - n = n + 1 - f[n] = compile(t2,n) - t2 = n - end - local t3 = ti[3] - if type(t3) == "function" then - -- todo: also create shortcut - elseif t3 then - c[t3] = f_shortcut(t3,t3) - if t1 == "*" then - if i == 1 then - done = f_any_all_c(t3,t2) - break - else - done = f_any_c(t3,t2) - end - else - m = m + 1 - r[m] = (m > 1 and f_elseif_c or f_if_c)(t1,t1,t3,t2) - end - else - if t1 == "*" then - if i == 1 then - done = f_any_all(t2) - break - else - done = f_any(t2) - end - else - m = m + 1 - r[m] = (m > 1 and f_elseif or f_if )(t1,t1,t2) - -- r[m] = (m > 1 and f_elseif_x or f_if_x)(t1,t1,t1,t2) - end - end - end - end - local c = f_check(r,done) - if nested then - return f_nested(nested,c) - else - return c - end - end - local p = t and presets[t] -- already done in implement - if p then - t = p - end - local tt = type(t) - if tt == "string" then - if a then - local s = lpegmatch(p_unquoted,t) - if s and t ~= s then - code = t - else - code = f_scan(t) - end - tokens._action = a - for i=1,#a do - code = f_action_f(i,code) - n = n + 1 - f[n] = f_action_s(i,i) - end - code = f_simple(f,code) - else - return scanners[t] - end - elseif tt ~= "table" then - return - elseif #t == 1 then - local ti = t[1] - if type(ti) == "table" then - ti = compile(ti) - code = "data" - if a then - tokens._action = a - for i=1,#a do - code = f_action_f(i,code) - n = n + 1 - f[n] = f_action_s(i,i) - end - end - code = f_table(f,ti,code) - elseif a then - code = f_scan(ti) - tokens._action = a - for i=1,#a do - code = f_action_f(i,code) - n = n + 1 - f[n] = f_action_s(i,i) - end - code = f_simple(f,code) - else - return scanners[ti] - end - elseif #t == 0 then - if specification.usage == "value" then - code = "b" - args = "_,b" - else - code = "" - args = "" - end - if a then - tokens._action = a - for i=1,#a do - code = f_action_f(i,code) - n = n + 1 - f[n] = f_action_s(i,i) - end - end - code = f_singular(c,f,args,code) - else - local r = { } - local p = { } - local m = 0 - for i=1,#t do - local ti = t[i] - local tt = type(ti) - if tt == "table" then - if ti[1] == "_constant_" then - local v = ti[2] - if type(v) == "string" then - r[i] = f_string(v) - else - r[i] = tostring(v) - end - else - m = m + 1 - p[m] = compile(ti,100+m) - r[i] = f_scan(100+m) - end - elseif tt == "number" then - r[i] = tostring(ti) - elseif tt == "boolean" then - r[i] = tostring(ti) - else - local s = lpegmatch(p_unquoted,ti) - if s and ti ~= s then - r[i] = ti -- a string, given as "'foo'" or '"foo"' - elseif scanners[ti] then - r[i] = f_scan(ti) - else - report_compile("unknown scanner %a",ti) - r[i] = ti - end - end - end - code = concat(r,",") - if a then - tokens._action = a - for i=1,#a do - code = f_action_f(i,code) - n = n + 1 - f[n] = f_action_s(i,i) - end - end - code = f_sequence(c,f,p,code) - end - if not code then - return - end - if trace_compile then - report_compile("code: %s",code) - end - local code, message = loadstripped(code) - if code then - code = code() -- sets action - else - report_compile("error in code: %s",code) - report_compile("error message: %s",message) - end - if a then - tokens._action = nil - end - if code then - return code - end -end - --- local fetch = tokens.compile { --- "string", --- "string", --- { --- { "data", "string" }, --- { "tab", "string" }, --- { "method", "string" }, --- { "foo", { --- { "method", "integer" }, --- { "compact", "number" }, --- { "nature" }, --- { "*" }, -- any key --- } }, --- { "compact", "string", "tonumber" }, --- { "nature", "boolean" }, --- { "escape", "string" }, --- { "escape" }, --- }, --- "boolean", --- } --- --- os.exit() |