beta 2011.09.27 20:05

9 files changed, 389 insertions, 65 deletions

diff --git a/context/data/scite/lexers/scite-context-lexer-lua.lua b/context/data/scite/lexers/scite-context-lexer-lua.lua
index 62577d4a9..1b55be55f 100644
--- a/context/data/scite/lexers/scite-context-lexer-lua.lua
+++ b/context/data/scite/lexers/scite-context-lexer-lua.lua
@@ -100,10 +100,10 @@ local shortcomment  = token("comment", dashes * lexer.nonnewline^0)
 local longcomment   = token("comment", dashes * longcomment)
 
 local shortstring   = token("quote",  dquote)
-                    * token("string", (escaped + (1-dquote))^0 )
+                    * token("string", (escaped + (1-dquote))^0)
                     * token("quote",  dquote)
                     + token("quote",  squote)
-                    * token("string", (escaped + (1-squote))^0 )
+                    * token("string", (escaped + (1-squote))^0)
                     * token("quote",  squote)
 
 local longstring    = token("quote",  longonestart)
diff --git a/context/data/scite/lexers/scite-context-lexer-mps.lua b/context/data/scite/lexers/scite-context-lexer-mps.lua
index afde63bcc..d374f7f82 100644
--- a/context/data/scite/lexers/scite-context-lexer-mps.lua
+++ b/context/data/scite/lexers/scite-context-lexer-mps.lua
@@ -8,13 +8,15 @@ local info = {
 
 local lexer = lexer
 local global, string, table, lpeg = _G, string, table, lpeg
-local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
+local token, exact_match = lexer.token, lexer.exact_match
 local P, R, S, V, C, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt
-local type, next, pcall, loadfile = type, next, pcall, loadfile
+local type = type
 
 module(...)
 
-local metafunlexer = _M
+local metafunlexer      = _M
+
+local context           = lexer.context
 
 local metafunhelpers    = { }
 local metafunconstants  = { }
@@ -23,14 +25,14 @@ local primitivecommands = { }
 
 do
 
-    local definitions = lexer.context.loaddefinitions("scite-context-data-metapost")
+    local definitions = context.loaddefinitions("scite-context-data-metapost")
 
     if definitions then
         plaincommands     = definitions.plain      or { }
         primitivecommands = definitions.primitives or { }
     end
 
-    local definitions = lexer.context.loaddefinitions("scite-context-data-metafun")
+    local definitions = context.loaddefinitions("scite-context-data-metafun")
 
     if definitions then
         metafunhelpers   = definitions.helpers   or { }
@@ -44,16 +46,9 @@ local whitespace = metafunlexer.WHITESPACE -- triggers states
 local space      = lexer.space -- S(" \n\r\t\f\v")
 local any        = lexer.any
 
-local digit      = R("09")
-local sign       = S("+-")
-local period     = P(".")
 local dquote     = P('"')
 local cstoken    = R("az","AZ") + P("_")
-local number     = sign^-1 * (                     -- at most one
-                        digit^1 * period * digit^0 -- 10.0 10.
-                      + digit^0 * period * digit^1 -- 0.10 .10
-                      + digit^1                    -- 10
-                   )
+local number     = context.patterns.real
 
 local cstokentex = R("az","AZ","\127\255") + S("@!?_")
 
@@ -66,10 +61,10 @@ local constant   = token('data',      exact_match(metafunconstants))
 local helper     = token('command',   exact_match(metafunhelpers))
 local plain      = token('plain',     exact_match(plaincommands))
 local quoted     = token('quote',     dquote)
-                 * token('string',    P(1-dquote)^1)
+                 * token('string',    P(1-dquote)^0)
                  * token('quote',     dquote)
 local primitive  = token('primitive', exact_match(primitivecommands))
-local identifier = token('default',   cstoken)
+local identifier = token('default',   cstoken^1)
 local number     = token('number',    number)
 local special    = token('special',   S("#()[]<>=:\"")) -- or else := <> etc split
 local texlike    = token('string',    P("\\") * cstokentex^1)
@@ -86,12 +81,12 @@ _rules = {
     { 'number',     number     },
     { 'quoted',     quoted     },
     { 'special',    special    },
---     { 'texlike',    texlike    },
+ -- { 'texlike',    texlike    },
     { 'extra',      extra      },
     { 'rest',       rest       },
 }
 
-_tokenstyles = lexer.context.styleset
+_tokenstyles = context.styleset
 
 _foldsymbols = {
     _patterns = {
diff --git a/context/data/scite/lexers/scite-context-lexer-pdf-object.lua b/context/data/scite/lexers/scite-context-lexer-pdf-object.lua
new file mode 100644
index 000000000..1de006813
--- /dev/null
+++ b/context/data/scite/lexers/scite-context-lexer-pdf-object.lua
@@ -0,0 +1,115 @@
+local info = {
+    version   = 1.002,
+    comment   = "scintilla lpeg lexer for pdf",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+}
+
+local lexer = lexer
+local token = lexer.token
+local P, R, S, C, V = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.V
+local global = _G
+
+module(...)
+
+local objectlexer       = _M
+
+local context           = lexer.context
+local patterns          = context.patterns
+
+local whitespace        = objectlexer.WHITESPACE -- triggers states
+
+local space             = lexer.space
+local somespace         = space^1
+
+local newline           = S("\n\r")
+local real              = patterns.real
+local cardinal          = patterns.cardinal
+
+local lparent           = P("(")
+local rparent           = P(")")
+local langle            = P("<")
+local rangle            = P(">")
+local escape            = P("\\")
+local anything          = P(1)
+local unicodetrigger    = P("feff")
+
+local nametoken         = 1 - space - S("<>/[]()")
+local name              = P("/") * nametoken^1
+
+local p_string          = P { ( escape * anything + lparent * V(1) * rparent + (1 - rparent) )^0 }
+
+local t_spacing         = token(whitespace, space^1)
+local t_spaces          = token(whitespace, space^1)^0
+
+local p_stream          = P("stream")
+local p_endstream       = P("endstream")
+----- p_obj             = P("obj")
+local p_endobj          = P("endobj")
+local p_reference       = P("R")
+
+local p_objectnumber    = patterns.cardinal
+local p_comment         = P('%') * (1-S("\n\r"))^0
+
+local string            = token("quote",    lparent)
+                        * token("string",   p_string)
+                        * token("quote",    rparent)
+local unicode           = token("quote",    langle)
+                        * token("plain",    unicodetrigger)
+                        * token("string",   (1-rangle)^1)
+                        * token("quote",    rangle)
+local whatsit           = token("quote",    langle)
+                        * token("string",   (1-rangle)^1)
+                        * token("quote",    rangle)
+local keyword           = token("command",  name)
+local constant          = token("constant", name)
+local number            = token('number',   real)
+local reference         = token("number",   cardinal)
+                        * t_spacing
+                        * token("number",   cardinal)
+                        * t_spacing
+                        * token("keyword",  p_reference)
+local t_comment         = token("comment",  p_comment)
+
+--    t_openobject      = token("number",  p_objectnumber)
+--                      * t_spacing
+--                      * token("number",  p_objectnumber)
+--                      * t_spacing
+--                      * token("keyword", p_obj)
+local t_closeobject     = token("keyword", p_endobj)
+
+local t_opendictionary  = token("grouping", P("<<"))
+local t_closedictionary = token("grouping", P(">>"))
+
+local t_openarray       = token("grouping", P("["))
+local t_closearray      = token("grouping", P("]"))
+
+local t_stream          = token("keyword", p_stream)
+--                         * token("default", newline * (1-newline*p_endstream*newline)^1 * newline)
+                        * token("default", (1 - p_endstream)^1)
+                        * token("keyword", p_endstream)
+
+local t_dictionary      = { "dictionary",
+                            dictionary = t_opendictionary * (t_spaces * keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
+                            array      = t_openarray * (t_spaces * V("whatever"))^0 * t_spaces * t_closearray,
+                            whatever   = V("dictionary") + V("array") + constant + reference + string + unicode + number + whatsit,
+                        }
+
+local t_object          = { "object", -- weird that we need to catch the end here (probably otherwise an invalid lpeg)
+                            object     = t_spaces * (V("dictionary") * t_spaces * t_stream^-1 + V("array") + t_spaces) * t_spaces * t_closeobject,
+                            dictionary = t_opendictionary * (t_spaces * keyword * t_spaces * V("whatever"))^0 * t_spaces * t_closedictionary,
+                            array      = t_openarray * (t_spaces * V("whatever"))^0 * t_spaces * t_closearray,
+                            whatever   = V("dictionary") + V("array") + constant + reference + string + unicode + number + whatsit,
+                        }
+
+_shared = {
+    dictionary = t_dictionary,
+}
+
+_rules = {
+    { 'whitespace', t_spacing },
+    { 'object',     t_object  },
+}
+
+_tokenstyles = context.styleset
diff --git a/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua b/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua
new file mode 100644
index 000000000..8988fbbb4
--- /dev/null
+++ b/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua
@@ -0,0 +1,41 @@
+local info = {
+    version   = 1.002,
+    comment   = "scintilla lpeg lexer for pdf",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+}
+
+local lexer = lexer
+local token = lexer.token
+local P = lpeg.P
+local global = _G
+
+module(...)
+
+local pdflexer          = _M
+local objectlexer       = lexer.load("scite-context-lexer-pdf-object")
+
+local context           = lexer.context
+local patterns          = context.patterns
+
+local whitespace        = pdflexer.WHITESPACE -- triggers states
+
+local space             = patterns.space
+local spacing           = patterns.spacing
+
+local t_spacing         = token(whitespace, spacing)
+
+local p_trailer         = P("trailer")
+
+local t_xref            = token("default", (1-p_trailer)^1)
+                        * token("keyword", p_trailer)
+                        * t_spacing
+                        * objectlexer._shared.dictionary
+
+_rules = {
+    { 'whitespace', t_spacing },
+    { 'xref',       t_xref    },
+}
+
+_tokenstyles = context.styleset
diff --git a/context/data/scite/lexers/scite-context-lexer-pdf.lua b/context/data/scite/lexers/scite-context-lexer-pdf.lua
new file mode 100644
index 000000000..ebc3fba4f
--- /dev/null
+++ b/context/data/scite/lexers/scite-context-lexer-pdf.lua
@@ -0,0 +1,63 @@
+local info = {
+    version   = 1.002,
+    comment   = "scintilla lpeg lexer for pdf",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+}
+
+local lexer = lexer
+local token = lexer.token
+local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local global = _G
+
+module(...)
+
+local pdflexer          = _M
+local objectlexer       = lexer.load("scite-context-lexer-pdf-object")
+local xreflexer         = lexer.load("scite-context-lexer-pdf-xref")
+
+local context           = lexer.context
+local patterns          = context.patterns
+
+local whitespace        = pdflexer.WHITESPACE -- triggers states
+
+local space             = patterns.space
+local spacing           = patterns.spacing
+local nospacing         = patterns.nospacing
+local anything          = patterns.anything
+local restofline        = patterns.restofline
+
+local t_spacing         = token(whitespace, spacing)
+local t_rest            = token("default",  nospacing) -- anything
+
+local p_obj             = P("obj")
+local p_endobj          = P("endobj")
+local p_xref            = P("xref")
+local p_startxref       = P("startxref")
+
+local p_objectnumber    = patterns.cardinal
+local p_comment         = P('%') * restofline
+
+local t_comment         = token("comment", p_comment)
+local t_openobject      = token("number",  p_objectnumber)
+                        * t_spacing
+                        * token("number",  p_objectnumber)
+                        * t_spacing
+                        * token("keyword", p_obj)
+                        * t_spacing^0
+local t_closeobject     = token("keyword", p_endobj)
+
+local t_openxref        = token("keyword", p_xref)
+local t_closexref       = token("keyword", p_startxref)
+
+lexer.embed_lexer(pdflexer, objectlexer, t_openobject, t_closeobject)
+lexer.embed_lexer(pdflexer, xreflexer,   t_openxref,   t_closexref)
+
+_rules = {
+    { 'whitespace', t_spacing },
+    { 'comment',    t_comment },
+    { 'rest',       t_rest    },
+}
+
+_tokenstyles = context.styleset
diff --git a/context/data/scite/lexers/scite-context-lexer-tex.lua b/context/data/scite/lexers/scite-context-lexer-tex.lua
index cea0aa544..8204ae3a0 100644
--- a/context/data/scite/lexers/scite-context-lexer-tex.lua
+++ b/context/data/scite/lexers/scite-context-lexer-tex.lua
@@ -34,9 +34,9 @@ local info = {
 
 local lexer = lexer
 local global, string, table, lpeg = _G, string, table, lpeg
-local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
+local token, exact_match = lexer.token, lexer.exact_match
 local P, R, S, V, C, Cmt, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt, lpeg.Cp, lpeg.Cc, lpeg.Ct
-local type, next, pcall, loadfile, setmetatable = type, next, pcall, loadfile, setmetatable
+local type, next = type, next
 local find, match, lower = string.find, string.match, string.lower
 
 module(...)
@@ -45,14 +45,16 @@ local contextlexer = _M
 local cldlexer     = lexer.load('scite-context-lexer-cld')
 local mpslexer     = lexer.load('scite-context-lexer-mps')
 
-local commands   = { en = { } }
-local primitives = { }
-local helpers    = { }
-local constants  = { }
+local commands     = { en = { } }
+local primitives   = { }
+local helpers      = { }
+local constants    = { }
+
+local context      = lexer.context
 
 do -- todo: only once, store in global
 
-    local definitions = lexer.context.loaddefinitions("scite-context-data-interfaces")
+    local definitions = context.loaddefinitions("scite-context-data-interfaces")
 
     if definitions then
         for interface, list in next, definitions do
@@ -72,14 +74,14 @@ do -- todo: only once, store in global
         end
     end
 
-    local definitions = lexer.context.loaddefinitions("scite-context-data-context")
+    local definitions = context.loaddefinitions("scite-context-data-context")
 
     if definitions then
         helpers   = definitions.helpers   or { }
         constants = definitions.constants or { }
     end
 
-    local definitions = lexer.context.loaddefinitions("scite-context-data-tex")
+    local definitions = context.loaddefinitions("scite-context-data-tex")
 
     if definitions then
         local function add(data)
@@ -109,9 +111,10 @@ local knowncommand = Cmt(cstoken^1, function(_,i,s)
     return currentcommands[s] and i
 end)
 
-local wordpattern = lexer.context.wordpattern
-local checkedword = lexer.context.checkedword
-local setwordlist = lexer.context.setwordlist
+local wordtoken   = context.patterns.wordtoken
+local wordpattern = context.patterns.wordpattern
+local checkedword = context.checkedword
+local setwordlist = context.setwordlist
 local validwords  = false
 
 -- % language=uk
@@ -121,11 +124,11 @@ local knownpreamble = Cmt(#P("% "), function(input,i,_) -- todo : utfbomb
         validwords = false
         local s, e, word = find(input,'^(.+)[\n\r]',i) -- combine with match
         if word then
-            local interface = match(word,"interface=(..)")
+            local interface = match(word,"interface=([a-z]+)")
             if interface then
                 currentcommands  = commands[interface] or commands.en or { }
             end
-            local language = match(word,"language=(..)")
+            local language = match(word,"language=([a-z]+)")
             validwords = language and setwordlist(language)
         end
     end
@@ -170,6 +173,8 @@ end)
 
 -- experiment: keep space with whatever ... less tables
 
+-- 10pt
+
 local commentline            = P('%') * (1-S("\n\r"))^0
 local endline                = S("\n\r")^1
 
@@ -194,7 +199,10 @@ local p_csname               = backslash * (cstoken^1 + P(1))
 local p_grouping             = S("{$}")
 local p_special              = S("#()[]<>=\"")
 local p_extra                = S("`~%^&_-+/\'|")
-local p_text                 = cstoken^1 --maybe add punctuation and space
+local p_text                 = wordtoken^1 --maybe add punctuation and space
+
+local p_number               = context.patterns.real
+local p_unit                 = P("pt") + P("bp") + P("sp") + P("mm") + P("cm") + P("cc") + P("dd")
 
 -- no looking back           = #(1-S("[=")) * cstoken^3 * #(1-S("=]"))
 
@@ -283,10 +291,11 @@ local primitive              = token('primitive', p_primitive  )
 local ifprimitive            = token('primitive', p_ifprimitive)
 local csname                 = token('user',      p_csname     )
 local grouping               = token('grouping',  p_grouping   )
+local number                 = token('number',    p_number     )
+                             * token('constant',  p_unit       )
 local special                = token('special',   p_special    )
 local extra                  = token('extra',     p_extra      )
------ text                   = token('default',   p_text       )
------ word                   = token("okay",      p_word       )
+local text                   = token('default',   p_text       )
 local word                   = p_word
 
 ----- startluacode           = token("grouping", P("\\startluacode"))
@@ -306,7 +315,7 @@ end
 local function stopdisplaylua(_,i,s)
     local ok = luatag == s
     if ok then
-cldlexer._directives.cld_inline = false
+        cldlexer._directives.cld_inline = false
         luastatus = false
     end
     return ok
@@ -344,7 +353,7 @@ local function stopinlinelua_e(_,i,s) -- }
         lualevel = lualevel - 1
         local ok = lualevel <= 0
         if ok then
-cldlexer._directives.cld_inline = false
+            cldlexer._directives.cld_inline = false
             luastatus = false
         end
         return ok
@@ -395,7 +404,7 @@ _rules = {
     { "whitespace",  spacing     },
     { "preamble",    preamble    },
     { "word",        word        },
- -- { "text",        text        },
+    { "text",        text        }, -- non words
     { "comment",     comment     },
     { "constant",    constant    },
     { "helper",      helper      },
@@ -405,12 +414,13 @@ _rules = {
     { "csname",      csname      },
  -- { "whatever",    specialword }, -- not yet, crashes
     { "grouping",    grouping    },
+ -- { "number",      number      },
     { "special",     special     },
     { "extra",       extra       },
     { "rest",        rest        },
 }
 
-_tokenstyles = lexer.context.styleset
+_tokenstyles = context.styleset
 
 local folds = {
     ["\\start"] = 1, ["\\stop" ] = -1,
diff --git a/context/data/scite/lexers/scite-context-lexer-txt.lua b/context/data/scite/lexers/scite-context-lexer-txt.lua
new file mode 100644
index 000000000..07dff2970
--- /dev/null
+++ b/context/data/scite/lexers/scite-context-lexer-txt.lua
@@ -0,0 +1,69 @@
+local lexer = lexer
+local token = lexer.token
+local P, S, Cmt = lpeg.P, lpeg.S, lpeg.Cmt
+local find, match = string.find, string.match
+
+module(...)
+
+local textlexer   = _M
+
+local context     = lexer.context
+
+local whitespace  = textlexer.WHITESPACE -- triggers states
+
+local space       = lexer.space
+local any         = lexer.any
+
+local wordtoken   = context.patterns.wordtoken
+local wordpattern = context.patterns.wordpattern
+local checkedword = context.checkedword
+local setwordlist = context.setwordlist
+local validwords  = false
+
+-- [#!-%] language=uk
+
+local p_preamble = Cmt(#(S("#!-%") * P(" ")), function(input,i,_) -- todo: utf bomb
+    if i == 1 then -- < 10 then
+        validwords = false
+        local s, e, line = find(input,'^[#!%-%%](.+)[\n\r]',i)
+        if line then
+            local language = match(line,"language=([a-z]+)")
+            if language then
+                validwords = setwordlist(language)
+            end
+        end
+    end
+    return false
+end)
+
+local t_preamble =
+    token('preamble', p_preamble)
+
+local t_word =
+    Cmt(wordpattern, function(_,i,s)
+        if validwords then
+            return checkedword(validwords,s,i)
+        else
+            return true, { "text", i }
+        end
+    end)
+
+local t_text =
+    token("default", wordtoken^1)
+
+local t_rest =
+    token("default", (1-wordtoken-space)^1)
+
+local t_spacing =
+    token(whitespace, space^1)
+
+_rules = {
+    { "whitespace", t_spacing  },
+    { "preamble",   t_preamble },
+    { "word",       t_word     }, -- words >= 3
+    { "text",       t_text     }, -- non words
+    { "rest",       t_rest     },
+}
+
+_tokenstyles = lexer.context.styleset
+
diff --git a/context/data/scite/lexers/scite-context-lexer-xml.lua b/context/data/scite/lexers/scite-context-lexer-xml.lua
index 0441585c1..f8892bbce 100644
--- a/context/data/scite/lexers/scite-context-lexer-xml.lua
+++ b/context/data/scite/lexers/scite-context-lexer-xml.lua
@@ -14,15 +14,17 @@ local info = {
 
 local lexer = lexer
 local global, string, table, lpeg = _G, string, table, lpeg
-local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
+local token, exact_match = lexer.token, lexer.exact_match
 local P, R, S, V, C, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt
-local type, setmetatable = type, setmetatable
+local type = type
 local match, find = string.match, string.find
 
 module(...)
 
 local examplelexer     = _M
 
+local context          = lexer.context
+
 local whitespace       = examplelexer.WHITESPACE -- triggers states
 
 local space            = lexer.space -- S(" \t\n\r\v\f")
@@ -49,9 +51,9 @@ local closecdata       = P("]]>")
 
 local entity           = ampersand * (1-semicolon)^1 * semicolon
 
-local wordpattern = lexer.context.wordpattern
-local checkedword = lexer.context.checkedword
-local setwordlist = lexer.context.setwordlist
+local wordpattern = context.patterns.wordpattern
+local checkedword = context.patterns.checkedword
+local setwordlist = context.setwordlist
 local validwords  = false
 
 -- <?xml version="1.0" encoding="UTF-8" language="uk" ?>
@@ -72,7 +74,6 @@ local p_preamble = Cmt(#P("<?xml "), function(input,i,_) -- todo: utf bomb
     return false
 end)
 
-
 local p_word =
     Cmt(wordpattern, function(_,i,s)
         if validwords then
@@ -190,7 +191,7 @@ _rules = {
     { "rest",        p_rest        },
 }
 
-_tokenstyles = lexer.context.styleset
+_tokenstyles = context.styleset
 
 _foldsymbols = { -- somehow doesn't work yet
     _patterns = {
diff --git a/context/data/scite/lexers/scite-context-lexer.lua b/context/data/scite/lexers/scite-context-lexer.lua
index a2bb35a57..13b8d77a8 100644
--- a/context/data/scite/lexers/scite-context-lexer.lua
+++ b/context/data/scite/lexers/scite-context-lexer.lua
@@ -41,7 +41,11 @@ local type, next, setmetatable, rawset = type, next, setmetatable, rawset
 
 dofile(_LEXERHOME .. '/lexer.lua')
 
-lexer.context = lexer.context or { }
+lexer.context    = lexer.context or { }
+local context    = lexer.context
+
+context.patterns = context.patterns or { }
+local patterns   = context.patterns
 
 local locations = {
  -- lexer.context.path,
@@ -87,7 +91,7 @@ end
 --     end
 -- end
 
-function lexer.context.loaddefinitions(name)
+function context.loaddefinitions(name)
     for i=1,#locations do
         local data = collect(locations[i] .. "/" .. name)
         if data then
@@ -98,7 +102,7 @@ end
 
 -- maybe more efficient:
 
-function lexer.context.word_match(words,word_chars,case_insensitive)
+function context.word_match(words,word_chars,case_insensitive)
     local chars = '%w_' -- maybe just "" when word_chars
     if word_chars then
         chars = '^([' .. chars .. gsub(word_chars,'([%^%]%-])', '%%%1') ..']+)'
@@ -128,16 +132,42 @@ end
 
 -- nicer (todo: utf):
 
-local defaults = R("az","AZ","\127\255","__")
+local idtoken = R("az","AZ","\127\255","__")
+local digit   = R("09")
+local sign    = S("+-")
+local period  = P(".")
+local space   = S(" \n\r\t\f\v")
+
+patterns.idtoken  = idtoken
+
+patterns.digit    = digit
+patterns.sign     = sign
+patterns.period   = period
+
+patterns.cardinal = digit^1
+patterns.integer  = sign^-1 * digit^1
 
-function lexer.context.exact_match(words,word_chars,case_insensitive)
+patterns.real     =
+    sign^-1 * (                    -- at most one
+        digit^1 * period * digit^0 -- 10.0 10.
+      + digit^0 * period * digit^1 -- 0.10 .10
+      + digit^1                    -- 10
+   )
+
+patterns.restofline = (1-S("\n\r"))^1
+patterns.space      = space
+patterns.spacing    = space^1
+patterns.nospacing  = (1-space)^1
+patterns.anything   = P(1)
+
+function context.exact_match(words,word_chars,case_insensitive)
     local characters = concat(words)
     local pattern -- the concat catches _ etc
     if word_chars == true or word_chars == false or word_chars == nil then
         word_chars = ""
     end
     if type(word_chars) == "string" then
-        pattern = S(characters) + defaults
+        pattern = S(characters) + idtoken
         if case_insensitive then
             pattern = pattern + S(upper(characters)) + S(lower(characters))
         end
@@ -166,7 +196,6 @@ function lexer.context.exact_match(words,word_chars,case_insensitive)
     end
 end
 
-
 -- spell checking (we can only load lua files)
 
 -- return {
@@ -185,13 +214,13 @@ local function splitwords(words)
     return lpegmatch(splitter,words)
 end
 
-function lexer.context.setwordlist(tag,limit) -- returns hash (lowercase keys and original values)
+function context.setwordlist(tag,limit) -- returns hash (lowercase keys and original values)
     if not tag or tag == "" then
         return false
     elseif lists[tag] ~= nil then
         return lists[tag]
     else
-        local list = lexer.context.loaddefinitions("spell-" .. tag)
+        local list = context.loaddefinitions("spell-" .. tag)
         if not list or type(list) ~= "table" then
             lists[tag] = false
             return nil
@@ -207,9 +236,10 @@ function lexer.context.setwordlist(tag,limit) -- returns hash (lowercase keys an
     end
 end
 
-lexer.context.wordpattern = R("az","AZ","\127\255")^3 -- todo: if limit and #s < limit then
+patterns.wordtoken   = R("az","AZ","\127\255")
+patterns.wordpattern = patterns.wordtoken^3 -- todo: if limit and #s < limit then
 
-function lexer.context.checkedword(validwords,s,i) -- ,limit
+function context.checkedword(validwords,s,i) -- ,limit
     if not validwords then
         return true, { "text", i }
     else
@@ -302,7 +332,7 @@ local splitlines = ( (
   + (                           newline   ) / function()    action_n()    end
 ) )^0
 
-function lexer.context.fold(text, start_pos, start_line, start_level)
+function context.fold(text, start_pos, start_line, start_level)
     if text == '' then
         return { }
     end
@@ -413,7 +443,7 @@ function lexer.context.fold(text, start_pos, start_line, start_level)
     return folds
 end
 
-function lexer.context.lex(text,init_style)
+function context.lex(text,init_style)
     local lexer = global._LEXER
     local grammar = lexer._GRAMMAR
     if not grammar then
@@ -499,11 +529,11 @@ end
 
 -- todo: keywords: one lookup and multiple matches
 
-function lexer.context.token(name, patt)
+function context.token(name, patt)
     return Ct(patt * Cc(name) * Cp())
 end
 
-lexer.fold        = lexer.context.fold
-lexer.lex         = lexer.context.lex
-lexer.token       = lexer.context.token
-lexer.exact_match = lexer.context.exact_match
+lexer.fold        = context.fold
+lexer.lex         = context.lex
+lexer.token       = context.token
+lexer.exact_match = context.exact_match