2021-10-08 20:07:00

1 files changed, 52 insertions, 147 deletions

diff --git a/context/data/scite/context/lexers/scite-context-lexer-lua.lua b/context/data/scite/context/lexers/scite-context-lexer-lua.lua
index 0e54d56ba..41219b66e 100644
--- a/context/data/scite/context/lexers/scite-context-lexer-lua.lua
+++ b/context/data/scite/context/lexers/scite-context-lexer-lua.lua
@@ -6,33 +6,23 @@ local info = {
     license   = "see context related readme files",
 }
 
--- beware: all multiline is messy, so even if it's no lexer, it should be an embedded lexer
--- we probably could use a local whitespace variant but this is cleaner
-
 local P, R, S, C, Cmt, Cp = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cmt, lpeg.Cp
 local match, find = string.match, string.find
 local setmetatable = setmetatable
 
-local lexer       = require("scite-context-lexer")
-local context     = lexer.context
-local patterns    = context.patterns
-
-local token       = lexer.token
-local exact_match = lexer.exact_match
-local just_match  = lexer.just_match
+local lexers        = require("scite-context-lexer")
 
-local lualexer    = lexer.new("lua","scite-context-lexer-lua")
-local whitespace  = lualexer.whitespace
+local patterns      = lexers.patterns
+local token         = lexers.token
 
-local stringlexer = lexer.load("scite-context-lexer-lua-longstring")
------ labellexer  = lexer.load("scite-context-lexer-lua-labelstring")
+local lualexer      = lexers.new("lua","scite-context-lexer-lua")
 
-local directives  = { } -- communication channel
+local luawhitespace = lualexer.whitespace
 
--- this will be extended
+local stringlexer   = lexers.load("scite-context-lexer-lua-longstring")
+----- labellexer    = lexers.load("scite-context-lexer-lua-labelstring")
 
--- we could combine some in a hash that returns the class that then makes the token
--- this can save time on large files
+local directives = { } -- communication channel
 
 local keywords = {
     "and", "break", "do", "else", "elseif", "end", "false", "for", "function", -- "goto",
@@ -61,12 +51,6 @@ local constants = {
    "<const>", "<toclose>",
 }
 
--- local tokenmappings = { }
---
--- for i=1,#keywords  do tokenmappings[keywords [i]] = "keyword"  }
--- for i=1,#functions do tokenmappings[functions[i]] = "function" }
--- for i=1,#constants do tokenmappings[constants[i]] = "constant" }
-
 local internals = { -- __
     "add", "call", "concat", "div", "idiv", "eq", "gc", "index",
     "le", "lt", "metatable", "mode", "mul", "newindex",
@@ -113,16 +97,16 @@ local longtwostring = P(function(input,index)
     end
 end)
 
-    local longtwostring_body = longtwostring
+local longtwostring_body = longtwostring
 
-    local longtwostring_end = P(function(input,index)
-        if level then
-         -- local sentinel = "]" .. level .. "]"
-            local sentinel = sentinels[level]
-            local _, stop = find(input,sentinel,index,true)
-            return stop and stop + 1 or #input + 1
-        end
-    end)
+local longtwostring_end = P(function(input,index)
+    if level then
+     -- local sentinel = "]" .. level .. "]"
+        local sentinel = sentinels[level]
+        local _, stop = find(input,sentinel,index,true)
+        return stop and stop + 1 or #input + 1
+    end
+end)
 
 local longcomment = Cmt(#("[[" + ("[" * C(equals) * "[")), function(input,index,level)
  -- local sentinel = "]" .. level .. "]"
@@ -134,14 +118,16 @@ end)
 local space         = patterns.space -- S(" \n\r\t\f\v")
 local any           = patterns.any
 local eol           = patterns.eol
+local exactmatch    = patterns.exactmatch
+local justmatch     = patterns.justmatch
 
 local squote        = P("'")
 local dquote        = P('"')
 local escaped       = P("\\") * P(1)
 local dashes        = P("--")
 
-local spacing       = token(whitespace, space^1)
-local rest          = token("default",  any)
+local spacing       = token(luawhitespace, space^1)
+local rest          = token("default", any)
 
 local shortcomment  = token("comment", dashes * (1-eol)^0)
 local longcomment   = token("comment", dashes * longcomment)
@@ -166,7 +152,7 @@ local shortstring   = token("quote",  dquote)
 local string        = shortstring
 -----               + longstring
 
-lexer.embed_lexer(lualexer, stringlexer, token("quote",longtwostart), token("string",longtwostring_body) * token("quote",longtwostring_end))
+lexers.embed(lualexer, stringlexer, token("quote",longtwostart), token("string",longtwostring_body) * token("quote",longtwostring_end))
 
 local integer       = P("-")^-1 * (patterns.hexadecimal + patterns.decimal)
 local number        = token("number", patterns.float + integer)
@@ -187,8 +173,6 @@ local identifier    = token("default",validword)
 ----- operator      = token("special", S('+-*/%^#=<>;:,{}[]().') + P('~=') ) -- no ^1 because of nested lexers
 local operator      = token("special", S('+-*/%^#=<>;:,{}[]().|~')) -- no ^1 because of nested lexers
 
-local structure     = token("special", S('{}[]()'))
-
 local optionalspace = spacing^0
 local hasargument   = #S("{([")
 
@@ -203,20 +187,15 @@ local gotolabel     = token("keyword", P("::"))
                     * (spacing + shortcomment)^0
                     * token("keyword", P("::"))
 
------ p_keywords    = exact_match(keywords)
------ p_functions   = exact_match(functions)
------ p_constants   = exact_match(constants)
------ p_internals   = P("__")
------               * exact_match(internals)
+local p_keywords    = exactmatch(keywords)
+local p_functions   = exactmatch(functions)
+local p_constants   = exactmatch(constants)
+local p_internals   = P("__")
+                    * exactmatch(internals)
 
 local p_finish      = #(1-R("az","AZ","__"))
-local p_keywords    = lexer.helpers.utfchartabletopattern(keywords)  * p_finish -- exact_match(keywords)
-local p_functions   = lexer.helpers.utfchartabletopattern(functions) * p_finish -- exact_match(functions)
-local p_constants   = lexer.helpers.utfchartabletopattern(constants) * p_finish -- exact_match(constants)
-local p_internals   = P("__")
-                    * lexer.helpers.utfchartabletopattern(internals) * p_finish -- exact_match(internals)
 
-local p_csnames     = lexer.helpers.utfchartabletopattern(csnames) -- * p_finish -- just_match(csnames)
+local p_csnames     = justmatch(csnames)
 local p_ctnames     = P("ctx") * R("AZ","az","__")^0
 local keyword       = token("keyword", p_keywords)
 local builtin       = token("plain",   p_functions)
@@ -237,23 +216,11 @@ local identifier    = token("default", validword)
                             token("default", validword )
                     ) )^0
 
--- local t = { } for k, v in next, tokenmappings do t[#t+1] = k end t = table.concat(t)
--- -- local experimental =  (S(t)^1) / function(s) return tokenmappings[s] end * Cp()
---
--- local experimental =  Cmt(S(t)^1, function(_,i,s)
---     local t = tokenmappings[s]
---     if t then
---         return true, t, i
---     end
--- end)
-
-lualexer._rules = {
+lualexer.rules = {
     { "whitespace",   spacing      },
     { "keyword",      keyword      }, -- can be combined
- -- { "structure",    structure    },
     { "function",     builtin      }, -- can be combined
     { "constant",     constant     }, -- can be combined
- -- { "experimental", experimental }, -- works but better split
     { "csname",       csname       },
     { "goto",         gotokeyword  },
     { "identifier",   identifier   },
@@ -266,82 +233,29 @@ lualexer._rules = {
     { "rest",         rest         },
 }
 
--- -- experiment
---
--- local idtoken = R("az","AZ","__")
---
--- function context.one_of_match(specification)
---     local pattern = idtoken -- the concat catches _ etc
---     local list = { }
---     for i=1,#specification do
---        local style = specification[i][1]
---        local words = specification[i][2]
---        pattern = pattern + S(table.concat(words))
---        for i=1,#words do
---            list[words[i]] = style
---        end
---    end
---    return Cmt(pattern^1, function(_,i,s)
---         local style = list[s]
---         if style then
---             return true, { style, i } -- and i or nil
---         else
---             -- fail
---         end
---    end)
--- end
---
--- local whatever = context.one_of_match {
---     { "keyword", keywords  }, -- keyword
---     { "plain",   functions }, -- builtin
---     { "data",    constants }, -- constant
--- }
---
--- lualexer._rules = {
---     { "whitespace",   spacing      },
---     { "whatever",     whatever     },
---     { "csname",       csname       },
---     { "goto",         gotokeyword  },
---     { "identifier",   identifier   },
---     { "string",       string       },
---     { "number",       number       },
---     { "longcomment",  longcomment  },
---     { "shortcomment", shortcomment },
---     { "label",        gotolabel    },
---     { "operator",     operator     },
---     { "rest",         rest         },
--- }
-
-lualexer._tokenstyles = context.styleset
-
--- lualexer._foldpattern = R("az")^2 + S("{}[]") -- separate entry else interference
-
-lualexer._foldpattern = (P("end") + P("if") + P("do") + P("function") + P("repeat") + P("until")) * P(#(1 - R("az")))
-                      + S("{}[]")
-
-lualexer._foldsymbols = {
-    _patterns = {
-        "[a-z][a-z]+",
-        "[{}%[%]]",
+lualexer.folding = {
+    -- challenge:  if=0  then=1  else=-1  elseif=-1
+    ["if"]       = { ["keyword"] =  1 }, -- if .. [then|else] .. end
+    ["do"]       = { ["keyword"] =  1 }, -- [while] do .. end
+    ["function"] = { ["keyword"] =  1 }, -- function .. end
+    ["repeat"]   = { ["keyword"] =  1 }, -- repeat .. until
+    ["until"]    = { ["keyword"] = -1 },
+    ["end"]      = { ["keyword"] = -1 },
+ -- ["else"]     = { ["keyword"] =  1 },
+ -- ["elseif"]   = { ["keyword"] =  1 }, -- already catched by if
+ -- ["elseif"]   = { ["keyword"] =  0 },
+    ["["] = {
+        ["comment"] =  1,
+     -- ["quote"]   =  1, -- confusing
     },
-    ["keyword"] = { -- challenge:  if=0  then=1  else=-1  elseif=-1
-        ["if"]       =  1, -- if .. [then|else] .. end
-        ["do"]       =  1, -- [while] do .. end
-        ["function"] =  1, -- function .. end
-        ["repeat"]   =  1, -- repeat .. until
-        ["until"]    = -1,
-        ["end"]      = -1,
-      },
-    ["comment"] = {
-        ["["] = 1, ["]"] = -1,
-    },
- -- ["quote"] = { -- confusing
- --     ["["] = 1, ["]"] = -1,
- -- },
-    ["special"] = {
-     -- ["("] = 1, [")"] = -1,
-        ["{"] = 1, ["}"] = -1,
+    ["]"] = {
+        ["comment"] = -1
+     -- ["quote"]   = -1, -- confusing
     },
+ -- ["("] = { ["special"] =  1 },
+ -- [")"] = { ["special"] = -1 },
+    ["{"] = { ["special"] =  1 },
+    ["}"] = { ["special"] = -1 },
 }
 
 -- embedded in tex:
@@ -360,24 +274,15 @@ local texstring       = token("quote",  longthreestart)
                       * token("string", longthreestring)
                       * token("quote",  longthreestop)
 
------ texcommand      = token("user", texcsname)
 local texcommand      = token("warning", texcsname)
 
--- local texstring    = token("quote", longthreestart)
---                    * (texcommand + token("string",P(1-texcommand-longthreestop)^1) - longthreestop)^0 -- we match long non-\cs sequences
---                    * token("quote", longthreestop)
-
--- local whitespace    = "whitespace"
--- local spacing       = token(whitespace, space^1)
-
-lualexer._directives = directives
+lualexer.directives = directives
 
-lualexer._rules_cld = {
+lualexer.rules_cld = {
     { "whitespace",   spacing      },
     { "texstring",    texstring    },
     { "texcomment",   texcomment   },
     { "texcommand",   texcommand   },
- -- { "structure",    structure    },
     { "keyword",      keyword      },
     { "function",     builtin      },
     { "csname",       csname       },