beta 2011.09.17 09:40

4 files changed, 376 insertions, 206 deletions

diff --git a/context/data/scite/lexers/data/scite-context-data-context.lua b/context/data/scite/lexers/data/scite-context-data-context.lua
index ba7500e01..4142d2a29 100644
--- a/context/data/scite/lexers/data/scite-context-data-context.lua
+++ b/context/data/scite/lexers/data/scite-context-data-context.lua
@@ -1,4 +1,4 @@
 return {
- ["constants"]={ "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive", "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred", "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "zeropoint", "onepoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint", "points", "zeroskip", "pluscxxvii", "pluscxxviii", "pluscclv", "pluscclvi", "endoflinetoken", "outputnewlinechar", "emptytoks", "empty", "undefined", "voidbox", "emptybox", "emptyvbox", "emptyhbox", "bigskipamount", "medskipamount", "smallskipamount", "fmtname", "fmtversion", "texengine", "texenginename", "texengineversion", "luatexengine", "pdftexengine", "xetexengine", "unknownengine", "etexversion", "pdftexversion", "xetexversion", "xetexrevision", "activecatcode", "bgroup", "egroup", "endline", "attributeunsetvalue", "uprotationangle", "rightrotatioangle", "downrotatioangle", "leftrotatioangle" },
- ["helpers"]={ "newcount", "newdimen", "newskip", "newmuskip", "newbox", "newtoks", "newread", "newwrite", "newmarks", "newinsert", "newattribute", "newif", "newlanguage", "newfamily", "newfam", "newhelp", "htdp", "unvoidbox", "scratchcounter", "globalscratchcounter", "scratchdimen", "globalscratchdimen", "scratchskip", "globalscratchskip", "scratchmuskip", "globalscratchmuskip", "scratchtoks", "globalscratchtoks", "scratchbox", "globalscratchbox", "scratchwidth", "scratchheight", "scratchdepth", "scratchcounterone", "scratchcountertwo", "scratchcounterthree", "scratchdimenone", "scratchdimentwo", "scratchdimenthree", "scratchskipone", "scratchskiptwo", "scratchskipthree", "scratchmuskipone", "scratchmuskiptwo", "scratchmuskipthree", "scratchtoksone", "scratchtokstwo", "scratchtoksthree", "scratchboxone", "scratchboxtwo", "scratchboxthree", "doif", "doifnot", "doifelse", "doifinset", "doifnotinset", "doifinsetelse", "doifnextcharelse", "doifnextoptionalelse", "doifnextparenthesiselse", "doiffastoptionalcheckelse", "doifundefinedelse", "doifdefinedelse", "doifundefined", "doifdefined", "doifelsevalue", "doifvalue", "doifnotvalue", "doifnothing", "doifsomething", "doifelsenothing", "doifsomethingelse", "doifvaluenothing", "doifvaluesomething", "doifelsevaluenothing", "doifdimensionelse", "doifnumberelse", "doifcommonelse", "doifcommon", "doifnotcommon", "doifinstring", "doifnotinstring", "doifinstringelse", "tracingall", "tracingnone", "loggingall", "appendtoks", "prependtoks", "appendtotoks", "prependtotoks", "endgraf", "empty", "null", "space", "obeyspaces", "obeylines", "normalspace", "executeifdefined", "singleexpandafter", "doubleexpandafter", "tripleexpandafter", "dontleavehmode", "wait", "writestatus", "define", "redefine", "setmeasure", "setemeasure", "setgmeasure", "setxmeasure", "definemeasure", "measure", "getvalue", "setvalue", "setevalue", "setgvalue", "setxvalue", "letvalue", "letgvalue", "resetvalue", "undefinevalue", "ignorevalue", "setuvalue", "setuevalue", "setugvalue", "setuxvalue", "globallet", "glet", "getparameters", "geteparameters", "getgparameters", "getxparameters", "forgetparameters", "processcommalist", "processcommacommand", "quitcommalist", "quitprevcommalist", "processaction", "processallactions", "processfirstactioninset", "processallactionsinset", "startsetups", "stopsetups", "startxmlsetups", "stopxmlsetups", "starttexdefinition", "stoptexdefinition", "starttexcode", "stoptexcode", "unexpanded", "expanded", "startexpanded", "stopexpanded", "protected", "protect", "unprotect", "firstofoneargument", "firstoftwoarguments", "secondoftwoarguments", "firstofthreearguments", "secondofthreearguments", "thirdofthreearguments", "firstoffourarguments", "secondoffourarguments", "thirdoffourarguments", "fourthoffourarguments", "firstoffivearguments", "secondoffivearguments", "thirdoffivearguments", "fourthoffivearguments", "fifthoffivearguments", "firstofsixarguments", "secondofsixarguments", "thirdofsixarguments", "fourthofsixarguments", "fifthofsixarguments", "sixthofsixarguments", "gobbleoneargument", "gobbletwoarguments", "gobblethreearguments", "gobblefourarguments", "gobblefivearguments", "gobblesixarguments", "gobblesevenarguments", "gobbleeightarguments", "gobbleninearguments", "gobbletenarguments", "gobbleoneoptional", "gobbletwooptionals", "gobblethreeoptionals", "gobblefouroptionals", "gobblefiveoptionals", "dorecurse", "doloop", "exitloop", "dostepwiserecurse", "recurselevel", "recursedepth", "newconstant", "setnewconstant", "newconditional", "settrue", "setfalse", "dosingleempty", "dodoubleempty", "dotripleempty", "doquadrupleempty", "doquintupleempty", "dosixtupleempty", "doseventupleempty", "dosinglegroupempty", "dodoublegroupempty", "dotriplegroupempty", "doquadruplegroupempty", "doquintuplegroupempty", "nopdfcompression", "maximumpdfcompression", "normalpdfcompression", "modulonumber", "dividenumber", "getfirstcharacter", "doiffirstcharelse" },
+ ["constants"]={ "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive", "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred", "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "zeropoint", "onepoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint", "points", "zeroskip", "pluscxxvii", "pluscxxviii", "pluscclv", "pluscclvi", "endoflinetoken", "outputnewlinechar", "emptytoks", "empty", "undefined", "voidbox", "emptybox", "emptyvbox", "emptyhbox", "bigskipamount", "medskipamount", "smallskipamount", "fmtname", "fmtversion", "texengine", "texenginename", "texengineversion", "luatexengine", "pdftexengine", "xetexengine", "unknownengine", "etexversion", "pdftexversion", "xetexversion", "xetexrevision", "activecatcode", "bgroup", "egroup", "endline", "attributeunsetvalue", "uprotationangle", "rightrotatioangle", "downrotatioangle", "leftrotatioangle", "startmode", "stopmode", "startnotmode", "stopnotmode", "doifmode", "doifmodeelse", "doifnotmode", "startenvironment", "stopenvironment", "environment", "startcomponent", "stopcomponent", "component", "startproduct", "stopproduct", "product", "startproject", "stopproject", "project", "starttext", "stoptext" },
+ ["helpers"]={ "startsetups", "stopsetups", "startxmlsetups", "stopxmlsetups", "starttexdefinition", "stoptexdefinition", "starttexcode", "stoptexcode", "newcount", "newdimen", "newskip", "newmuskip", "newbox", "newtoks", "newread", "newwrite", "newmarks", "newinsert", "newattribute", "newif", "newlanguage", "newfamily", "newfam", "newhelp", "htdp", "unvoidbox", "scratchcounter", "globalscratchcounter", "scratchdimen", "globalscratchdimen", "scratchskip", "globalscratchskip", "scratchmuskip", "globalscratchmuskip", "scratchtoks", "globalscratchtoks", "scratchbox", "globalscratchbox", "scratchwidth", "scratchheight", "scratchdepth", "scratchcounterone", "scratchcountertwo", "scratchcounterthree", "scratchdimenone", "scratchdimentwo", "scratchdimenthree", "scratchskipone", "scratchskiptwo", "scratchskipthree", "scratchmuskipone", "scratchmuskiptwo", "scratchmuskipthree", "scratchtoksone", "scratchtokstwo", "scratchtoksthree", "scratchboxone", "scratchboxtwo", "scratchboxthree", "doif", "doifnot", "doifelse", "doifinset", "doifnotinset", "doifinsetelse", "doifnextcharelse", "doifnextoptionalelse", "doifnextparenthesiselse", "doiffastoptionalcheckelse", "doifundefinedelse", "doifdefinedelse", "doifundefined", "doifdefined", "doifelsevalue", "doifvalue", "doifnotvalue", "doifnothing", "doifsomething", "doifelsenothing", "doifsomethingelse", "doifvaluenothing", "doifvaluesomething", "doifelsevaluenothing", "doifdimensionelse", "doifnumberelse", "doifcommonelse", "doifcommon", "doifnotcommon", "doifinstring", "doifnotinstring", "doifinstringelse", "tracingall", "tracingnone", "loggingall", "appendtoks", "prependtoks", "appendtotoks", "prependtotoks", "endgraf", "empty", "null", "space", "obeyspaces", "obeylines", "normalspace", "executeifdefined", "singleexpandafter", "doubleexpandafter", "tripleexpandafter", "dontleavehmode", "wait", "writestatus", "define", "redefine", "setmeasure", "setemeasure", "setgmeasure", "setxmeasure", "definemeasure", "measure", "getvalue", "setvalue", "setevalue", "setgvalue", "setxvalue", "letvalue", "letgvalue", "resetvalue", "undefinevalue", "ignorevalue", "setuvalue", "setuevalue", "setugvalue", "setuxvalue", "globallet", "glet", "getparameters", "geteparameters", "getgparameters", "getxparameters", "forgetparameters", "processcommalist", "processcommacommand", "quitcommalist", "quitprevcommalist", "processaction", "processallactions", "processfirstactioninset", "processallactionsinset", "unexpanded", "expanded", "startexpanded", "stopexpanded", "protected", "protect", "unprotect", "firstofoneargument", "firstoftwoarguments", "secondoftwoarguments", "firstofthreearguments", "secondofthreearguments", "thirdofthreearguments", "firstoffourarguments", "secondoffourarguments", "thirdoffourarguments", "fourthoffourarguments", "firstoffivearguments", "secondoffivearguments", "thirdoffivearguments", "fourthoffivearguments", "fifthoffivearguments", "firstofsixarguments", "secondofsixarguments", "thirdofsixarguments", "fourthofsixarguments", "fifthofsixarguments", "sixthofsixarguments", "gobbleoneargument", "gobbletwoarguments", "gobblethreearguments", "gobblefourarguments", "gobblefivearguments", "gobblesixarguments", "gobblesevenarguments", "gobbleeightarguments", "gobbleninearguments", "gobbletenarguments", "gobbleoneoptional", "gobbletwooptionals", "gobblethreeoptionals", "gobblefouroptionals", "gobblefiveoptionals", "dorecurse", "doloop", "exitloop", "dostepwiserecurse", "recurselevel", "recursedepth", "newconstant", "setnewconstant", "newconditional", "settrue", "setfalse", "dosingleempty", "dodoubleempty", "dotripleempty", "doquadrupleempty", "doquintupleempty", "dosixtupleempty", "doseventupleempty", "dosinglegroupempty", "dodoublegroupempty", "dotriplegroupempty", "doquadruplegroupempty", "doquintuplegroupempty", "nopdfcompression", "maximumpdfcompression", "normalpdfcompression", "modulonumber", "dividenumber", "getfirstcharacter", "doiffirstcharelse" },
 }
\ No newline at end of file
diff --git a/context/data/scite/lexers/scite-context-lexer-cld.lua b/context/data/scite/lexers/scite-context-lexer-cld.lua
index 1abc55f91..632a7672c 100644
--- a/context/data/scite/lexers/scite-context-lexer-cld.lua
+++ b/context/data/scite/lexers/scite-context-lexer-cld.lua
@@ -6,166 +6,14 @@ local info = {
     license   = "see context related readme files",
 }
 
--- Adapted from lua.lua by Mitchell who based it on a lexer by Peter Odding.
-
 local lexer = lexer
-local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
-local P, R, S, C, Cg, Cb, Cs, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt
-local match, find = string.match, string.find
-local global = _G
 
 module(...)
 
-local cldlexer = _M
-
-local keywords = {
-  'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function',
-  'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true',
-  'until', 'while',
-}
-
-local functions = {
-  'assert', 'collectgarbage', 'dofile', 'error', 'getfenv', 'getmetatable',
-  'ipairs', 'load', 'loadfile', 'loadstring', 'module', 'next', 'pairs',
-  'pcall', 'print', 'rawequal', 'rawget', 'rawset', 'require', 'setfenv',
-  'setmetatable', 'tonumber', 'tostring', 'type', 'unpack', 'xpcall',
-}
-
-local constants = {
-  '_G', '_VERSION',
-}
-
-local csnames = {
-    "context",
-    "metafun",
-}
-
-local level         = nil
-local setlevel      = function(_,i,s) level = s return i end
-
-local equals        = P("=")^0
-
-local longonestart  = P("[[")
-local longonestop   = P("]]")
-local longonestring = (1-longonestop)^0
-
-local longtwostart  = P('[') * Cmt(equals,setlevel) * P('[')
-local longtwostop   = P(']') *     equals           * P(']')
-
-local longtwostring = P(function(input,index)
-    if level then
-        local sentinel = ']' .. level .. ']'
-        local _, stop = find(input,sentinel,index,true)
-        return stop and stop + 1 - #sentinel or #input + 1
-    end
-end)
-
--- local longtwostart  = P("[") * Cg(equals, "init") * P("[")
--- local longtwostop   = P("]") * C(equals) * P("]")
--- local longtwocheck  = Cmt(longtwostop * Cb("init"), function(s,i,a,b) return a == b end)
--- local longtwostring = (P(1) - longtwocheck)^0
-
-local longcomment = Cmt(#('[[' + ('[' * P('=')^0 * '[')), function(input,index)
-    local level = match(input,'^%[(=*)%[',index)
-    level = "=="
-    if level then
-        local _, stop = find(input,']' .. level .. ']',index,true)
-        return stop and stop + 1 or #input + 1
-    end
-end)
+local cldlexer = lexer.load('scite-context-lexer-lua')
 
-local longcomment =  Cmt(#('[[' + ('[' * C(P('=')^0) * '[')), function(input,index,level)
-    local _, stop = find(input,']' .. level .. ']',index,true)
-    return stop and stop + 1 or #input + 1
-end)
+_rules       = cldlexer._rules_cld
+_tokenstyles = cldlexer._tokenstyles
+_foldsymbols = cldlexer._foldsymbols
 
-local whitespace    = cldlexer.WHITESPACE -- triggers states
-
-local space         = lexer.space -- S(" \n\r\t\f\v")
-local any           = lexer.any
-
-local squote        = P("'")
-local dquote        = P('"')
-local escaped       = P("\\") * P(1)
-local dashes        = P('--')
-
-local spacing       = token(whitespace, space^1)
-local rest          = token("default",  any)
-
-local shortcomment  = dashes * lexer.nonnewline^0
-local longcomment   = dashes * longcomment
-local comment       = token("comment", longcomment + shortcomment)
-
-local shortstring   = token("quote",  squote)
-                    * token("string", (escaped + (1-squote))^0 )
-                    * token("quote",  squote)
-                    + token("quote",  dquote)
-                    * token("string", (escaped + (1-dquote))^0 )
-                    * token("quote",  dquote)
-
-local longstring    = token("quote",  longonestart)
-                    * token("string", longonestring)
-                    * token("quote",  longonestop)
-                    + token("quote",  longtwostart)
-                    * token("string", longtwostring)
-                    * token("quote",  longtwostop)
-
-local string        = shortstring
-                    + longstring
-
-local integer       = P('-')^-1 * (lexer.hex_num + lexer.dec_num)
-local number        = token("number", lexer.float + integer)
-
-local word          = R('AZ','az','__','\127\255') * (lexer.alnum + '_')^0
-local identifier    = token("default", word)
-
-local operator      = token("special", P('~=') + S('+-*/%^#=<>;:,.{}[]()')) -- maybe split of {}[]()
-
-local keyword       = token("keyword", exact_match(keywords))
-local builtin       = token("plain", exact_match(functions))
-local constant      = token("data", exact_match(constants))
-local csname        = token("user",         exact_match(csnames)) * (
-                        spacing^0 * #S("{(")
-                        + ( spacing^0 * token("special", P(".")) * spacing^0 * token("csname",word) )^1
-                    )
-
-_rules = {
-    { 'whitespace', spacing    },
-    { 'keyword',    keyword    },
-    { 'function',   builtin    },
-    { 'csname',     csname     },
-    { 'constant',   constant   },
-    { 'identifier', identifier },
-    { 'string',     string     },
-    { 'comment',    comment    },
-    { 'number',     number     },
-    { 'operator',   operator   },
-    { 'rest',       rest       },
-}
-
-_tokenstyles = lexer.context.styleset
-
-_foldsymbols = {
-    _patterns = {
-        '%l+',
-        '[%({%)}%[%]]',
-    },
-    ['keyword'] = {
-        ['if']       =  1,
-        ['end']      = -1,
-        ['do']       =  1,
-        ['function'] =  1,
-        ['repeat']   =  1,
-        ['until']    = -1,
-      },
-    ['comment'] = {
-        ['['] = 1, [']'] = -1,
-    },
-    ['quote'] = { -- to be tested
-        ['['] = 1, [']'] = -1,
-    },
-    ['special'] = {
-        ['('] = 1, [')'] = -1,
-        ['{'] = 1, ['}'] = -1,
-    },
-}
+_directives  = cldlexer._directives
diff --git a/context/data/scite/lexers/scite-context-lexer-lua.lua b/context/data/scite/lexers/scite-context-lexer-lua.lua
new file mode 100644
index 000000000..2a0f48026
--- /dev/null
+++ b/context/data/scite/lexers/scite-context-lexer-lua.lua
@@ -0,0 +1,219 @@
+local info = {
+    version   = 1.002,
+    comment   = "scintilla lpeg lexer for cld/lua",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files",
+}
+
+-- Adapted from lua.lua by Mitchell who based it on a lexer by Peter Odding.
+
+local lexer = lexer
+local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing
+local P, R, S, C, Cg, Cb, Cs, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt
+local match, find = string.match, string.find
+local global = _G
+
+module(...)
+
+local cldlexer = _M
+
+_directives = { } -- communication channel
+
+local keywords = {
+  'and', 'break', 'do', 'else', 'elseif', 'end', 'false', 'for', 'function',
+  'if', 'in', 'local', 'nil', 'not', 'or', 'repeat', 'return', 'then', 'true',
+  'until', 'while',
+}
+
+local functions = {
+  'assert', 'collectgarbage', 'dofile', 'error', 'getfenv', 'getmetatable',
+  'ipairs', 'load', 'loadfile', 'loadstring', 'module', 'next', 'pairs',
+  'pcall', 'print', 'rawequal', 'rawget', 'rawset', 'require', 'setfenv',
+  'setmetatable', 'tonumber', 'tostring', 'type', 'unpack', 'xpcall',
+}
+
+local constants = {
+  '_G', '_VERSION', '_M',
+}
+
+local csnames = { -- todo: option
+    "context",
+    "metafun",
+}
+
+local level         = nil
+local setlevel      = function(_,i,s) level = s return i end
+
+local equals        = P("=")^0
+
+local longonestart  = P("[[")
+local longonestop   = P("]]")
+local longonestring = (1-longonestop)^0
+
+local longtwostart  = P('[') * Cmt(equals,setlevel) * P('[')
+local longtwostop   = P(']') *     equals           * P(']')
+
+local longtwostring = P(function(input,index)
+    if level then
+        local sentinel = ']' .. level .. ']'
+        local _, stop = find(input,sentinel,index,true)
+        return stop and stop + 1 - #sentinel or #input + 1
+    end
+end)
+
+-- local longtwostart  = P("[") * Cg(equals, "init") * P("[")
+-- local longtwostop   = P("]") * C(equals) * P("]")
+-- local longtwocheck  = Cmt(longtwostop * Cb("init"), function(s,i,a,b) return a == b end)
+-- local longtwostring = (P(1) - longtwocheck)^0
+
+local longcomment = Cmt(#('[[' + ('[' * P('=')^0 * '[')), function(input,index)
+    local level = match(input,'^%[(=*)%[',index)
+    level = "=="
+    if level then
+        local _, stop = find(input,']' .. level .. ']',index,true)
+        return stop and stop + 1 or #input + 1
+    end
+end)
+
+local longcomment =  Cmt(#('[[' + ('[' * C(P('=')^0) * '[')), function(input,index,level)
+    local _, stop = find(input,']' .. level .. ']',index,true)
+    return stop and stop + 1 or #input + 1
+end)
+
+local whitespace    = cldlexer.WHITESPACE -- triggers states
+
+local space         = lexer.space -- S(" \n\r\t\f\v")
+local any           = lexer.any
+
+local squote        = P("'")
+local dquote        = P('"')
+local escaped       = P("\\") * P(1)
+local dashes        = P('--')
+
+local spacing       = token(whitespace, space^1)
+local rest          = token("default",  any)
+
+local shortcomment  = token("comment", dashes * lexer.nonnewline^0)
+local longcomment   = token("comment", dashes * longcomment)
+
+local shortstring   = token("quote",  dquote)
+                    * token("string", (escaped + (1-dquote))^0 )
+                    * token("quote",  dquote)
+                    + token("quote",  squote)
+                    * token("string", (escaped + (1-squote))^0 )
+                    * token("quote",  squote)
+
+local longstring    = token("quote",  longonestart)
+                    * token("string", longonestring)
+                    * token("quote",  longonestop)
+                    + token("quote",  longtwostart)
+                    * token("string", longtwostring)
+                    * token("quote",  longtwostop)
+
+local string        = shortstring
+                    + longstring
+
+local integer       = P('-')^-1 * (lexer.hex_num + lexer.dec_num)
+local number        = token("number", lexer.float + integer)
+
+-- officially 127-255 are ok but not utf so useless
+
+local validword     = R("AZ","az","__") * R("AZ","az","__","09")^0
+
+local identifier    = token("default",validword)
+
+local operator      = token("special", P('..') + P('~=') + S('+-*/%^#=<>;:,.{}[]()')) -- maybe split off {}[]()
+
+local optionalspace = spacing^0
+local hasargument   = #S("{(")
+
+local keyword       = token("keyword", exact_match(keywords ))
+local builtin       = token("plain",   exact_match(functions))
+local constant      = token("data",    exact_match(constants))
+local csname        = token("user",    exact_match(csnames  ))
+                    * (
+                        optionalspace * hasargument
+                      + ( optionalspace * token("special", P(".")) * optionalspace * token("user", validword) )^1
+                    )
+
+_rules = {
+    { 'whitespace',   spacing      },
+    { 'keyword',      keyword      },
+    { 'function',     builtin      },
+    { 'csname',       csname       },
+    { 'constant',     constant     },
+    { 'identifier',   identifier   },
+    { 'string',       string       },
+    { 'longcomment',  longcomment  },
+    { 'shortcomment', shortcomment },
+    { 'number',       number       },
+    { 'operator',     operator     },
+    { 'rest',         rest         },
+}
+
+_tokenstyles = lexer.context.styleset
+
+_foldsymbols = {
+    _patterns = {
+        '%l+',
+        '[%({%)}%[%]]',
+    },
+    ['keyword'] = {
+        ['if']       =  1,
+        ['end']      = -1,
+        ['do']       =  1,
+        ['function'] =  1,
+        ['repeat']   =  1,
+        ['until']    = -1,
+      },
+    ['comment'] = {
+        ['['] = 1, [']'] = -1,
+    },
+    ['quote'] = { -- to be tested
+        ['['] = 1, [']'] = -1,
+    },
+    ['special'] = {
+        ['('] = 1, [')'] = -1,
+        ['{'] = 1, ['}'] = -1,
+    },
+}
+
+-- embedded in tex:
+
+local cstoken         = R("az","AZ","\127\255") + S("@!?_")
+local csnametex       = P("\\") * cstoken^1
+local commentline     = P('%') * (1-S("\n\r"))^0
+
+local texcomment      = token('comment', Cmt(commentline, function() return _directives.cld_inline end))
+
+local longthreestart  = P("\\!!bs")
+local longthreestop   = P("\\!!es")
+local longthreestring = (1-longthreestop)^0
+
+local texstring       = token("quote",  longthreestart)
+                      * token("string", longthreestring)
+                      * token("quote",  longthreestop)
+
+-- local texcommand   = token("user", csnametex)
+--
+-- local texstring    = token("quote", longthreestart)
+--                    * (texcommand + token("string",P(1-texcommand-longthreestop)^1) - longthreestop)^0 -- we match long non-\cs sequences
+--                    * token("quote", longthreestop)
+
+_rules_cld = {
+    { 'whitespace',   spacing      },
+    { 'texstring',    texstring    },
+    { 'texcomment',   texcomment   },
+    { 'keyword',      keyword      },
+    { 'function',     builtin      },
+    { 'csname',       csname       },
+    { 'constant',     constant     },
+    { 'identifier',   identifier   },
+    { 'string',       string       },
+    { 'longcomment',  longcomment  },
+    { 'shortcomment', shortcomment }, -- should not be used inline so best signal it as comment (otherwise complex state till end of inline)
+    { 'number',       number       },
+    { 'operator',     operator     },
+    { 'rest',         rest         },
+}
diff --git a/context/data/scite/lexers/scite-context-lexer-tex.lua b/context/data/scite/lexers/scite-context-lexer-tex.lua
index 01000657b..dac3ada63 100644
--- a/context/data/scite/lexers/scite-context-lexer-tex.lua
+++ b/context/data/scite/lexers/scite-context-lexer-tex.lua
@@ -42,6 +42,8 @@ local find, match = string.find, string.match
 module(...)
 
 local contextlexer = _M
+local cldlexer     = lexer.load('scite-context-lexer-cld')
+local mpslexer     = lexer.load('scite-context-lexer-mps')
 
 local basepath     = lexer.context and lexer.context.path or _LEXERHOME
 
@@ -158,62 +160,172 @@ end)
 --     end
 -- end)
 
+-- experiment: keep space with whatever ... less tables
+
+local commentline            = P('%') * (1-S("\n\r"))^0
+local endline                = S("\n\r")^1
+
 local whitespace             = contextlexer.WHITESPACE -- triggers states
 
 local space                  = lexer.space -- S(" \n\r\t\f\v")
 local any                    = lexer.any
 
-local spacing                = token(whitespace,  space^1)
-local rest                   = token('default',   any)
-local preamble               = token('preamble',  knownpreamble)
-local comment                = token('comment',   P('%') * (1-S("\n\r"))^0)
-local command                = token('command',   P('\\') * knowncommand)
-local constant               = token('data',      P('\\') * exact_match(constants))
-local helper                 = token('plain',     P('\\') * exact_match(helpers))
-local primitive              = token('primitive', P('\\') * exact_match(primitives))
-local ifprimitive            = token('primitive', P('\\if') * cstoken^1)
-local csname                 = token('user',      P('\\') * (cstoken^1 + P(1)))
-local grouping               = token('grouping',  S("{$}")) -- maybe also \bgroup \egroup \begingroup \endgroup
-local special                = token('special',   S("#()[]<>=\""))
-local extra                  = token('extra',     S("`~%^&_-+/\'|"))
-
-local text                   = token('default',   cstoken^1 )
+local p_spacing              = space^1
+local p_rest                 = any
+
+local p_preamble             = knownpreamble
+local p_comment              = commentline
+local p_command              = P('\\')   * knowncommand
+local p_constant             = P('\\')   * exact_match(constants)
+local p_helper               = P('\\')   * exact_match(helpers)
+local p_primitive            = P('\\')   * exact_match(primitives)
+local p_ifprimitive          = P('\\if') * cstoken^1
+local p_csname               = P('\\')   * (cstoken^1 + P(1))
+local p_grouping             = S("{$}")
+local p_special              = S("#()[]<>=\"")
+local p_extra                = S("`~%^&_-+/\'|")
+local p_text                 = cstoken^1
+
+-- keep key pressed at end-of syst-aux.mkiv:
+--
+-- 0 : 15 sec
+-- 1 : 13 sec
+-- 2 : 10 sec
+--
+-- the problem is that quite some style subtables get generated so collapsing ranges helps
+
+local option = 1
+
+if option == 1 then
+
+    p_comment                = p_comment^1
+    p_grouping               = p_grouping^1
+    p_special                = p_special^1
+    p_extra                  = p_extra^1
+    p_text                   = p_text^1
+
+    p_command                = p_command^1
+    p_constant               = p_constant^1
+    p_helper                 = p_helper^1
+    p_primitive              = p_primitive^1
+    p_ifprimitive            = p_ifprimitive^1
+
+elseif option == 2 then
+
+    local included           = space^0
+
+    p_comment                = (p_comment     * included)^1
+    p_grouping               = (p_grouping    * included)^1
+    p_special                = (p_special     * included)^1
+    p_extra                  = (p_extra       * included)^1
+    p_text                   = (p_text        * included)^1
+
+    p_command                = (p_command     * included)^1
+    p_constant               = (p_constant    * included)^1
+    p_helper                 = (p_helper      * included)^1
+    p_primitive              = (p_primitive   * included)^1
+    p_ifprimitive            = (p_ifprimitive * included)^1
+
+end
+
+local spacing                = token(whitespace,  p_spacing    )
+
+local rest                   = token('default',   p_rest       )
+local preamble               = token('preamble',  p_preamble   )
+local comment                = token('comment',   p_comment    )
+local command                = token('command',   p_command    )
+local constant               = token('data',      p_constant   )
+local helper                 = token('plain',     p_helper     )
+local primitive              = token('primitive', p_primitive  )
+local ifprimitive            = token('primitive', p_ifprimitive)
+local csname                 = token('user',      p_csname     )
+local grouping               = token('grouping',  p_grouping   )
+local special                = token('special',   p_special    )
+local extra                  = token('extra',     p_extra      )
+local text                   = token('default',   p_text       )
 
 ----- startluacode           = token("grouping", P("\\startluacode"))
 ----- stopluacode            = token("grouping", P("\\stopluacode"))
 
-local luastatus              = nil
+local luastatus = false
+local luatag    = nil
+local lualevel  = 0
+
+local function startdisplaylua(_,i,s)
+    luatag = s
+    luastatus = "display"
+    cldlexer._directives.cld_inline = false
+    return true
+end
+
+local function stopdisplaylua(_,i,s)
+    local ok = luatag == s
+    if ok then
+        luastatus = false
+    end
+    return ok
+end
+
+local function startinlinelua(_,i,s)
+    if luastatus == "display" then
+        return false
+    elseif not luastatus then
+        luastatus = "inline"
+        cldlexer._directives.cld_inline = true
+        lualevel = 1
+        return true
+    else
+        lualevel = lualevel + 1
+        return true
+    end
+end
+
+local function stopinlinelua_b(_,i,s) -- {
+    if luastatus == "display" then
+        return false
+    elseif luastatus == "inline" then
+        lualevel = lualevel + 1
+        return false
+    else
+        return true
+    end
+end
+
+local function stopinlinelua_e(_,i,s) -- }
+    if luastatus == "display" then
+        return false
+    elseif luastatus == "inline" then
+        lualevel = lualevel - 1
+        local ok = lualevel <= 0
+        if ok then
+            luastatus = false
+        end
+        return ok
+    else
+        return true
+    end
+end
+
 local luaenvironment         = P("luacode")
 
-local inlinelua              = P("\\ctx") * ( P("lua") + P("command") )
-                             + P("\\cldcontext")
+local inlinelua              = P("\\") * (
+                                    P("ctx") * ( P("lua") + P("command") )
+                                  + P("cldcontext")
+                               )
 
-local startlua               = P("\\start") * Cmt(luaenvironment,function(_,i,s) luastatus = s return true end)
-                             + inlinelua
-                             * space^0
-                             * Cmt(P("{"),function(_,i,s) luastatus = "}" return true end)
-local stoplua                = P("\\stop") * Cmt(luaenvironment,function(_,i,s) return luastatus == s end)
-                             + Cmt(P("}"),function(_,i,s) return luastatus == "}" end)
+local startlua               = P("\\start") * Cmt(luaenvironment,startdisplaylua)
+                             + inlinelua * space^0 * Cmt(P("{"),startinlinelua)
+
+local stoplua                = P("\\stop") * Cmt(luaenvironment,stopdisplaylua)
+                             + Cmt(P("{"),stopinlinelua_b)
+                             + Cmt(P("}"),stopinlinelua_e)
 
 local startluacode           = token("embedded", startlua)
 local stopluacode            = token("embedded", stoplua)
 
--- local metafunenvironment     = P("useMPgraphic")
---                              + P("reusableMPgraphic")
---                              + P("uniqueMPgraphic")
---                              + P("MPcode")
---                              + P("MPpage")
---                              + P("MPinclusions")
---                              + P("MPextensions")
---                              + P("MPgraphic")
-
 local metafunenvironment     = ( P("use") + P("reusable") + P("unique") ) * ("MPgraphic")
                              + P("MP") * ( P("code")+ P("page") + P("inclusions") + P("extensions") + P("graphic") )
 
--- local metafunstatus          = nil -- this does not work, as the status gets lost in an embedded lexer
--- local startmetafun           = P("\\start") * Cmt(metafunenvironment,function(_,i,s) metafunstatus = s return true end)
--- local stopmetafun            = P("\\stop")  * Cmt(metafunenvironment,function(_,i,s) return metafunstatus == s end)
-
 local startmetafun           = P("\\start") * metafunenvironment
 local stopmetafun            = P("\\stop")  * metafunenvironment
 
@@ -226,9 +338,6 @@ local metafunarguments       = (spacing^0 * openargument * argumentcontent * clo
 local startmetafuncode       = token("embedded", startmetafun) * metafunarguments
 local stopmetafuncode        = token("embedded", stopmetafun)
 
-local cldlexer = lexer.load('scite-context-lexer-cld')
-local mpslexer = lexer.load('scite-context-lexer-mps')
-
 lexer.embed_lexer(contextlexer, cldlexer, startluacode,     stopluacode)
 lexer.embed_lexer(contextlexer, mpslexer, startmetafuncode, stopmetafuncode)
 
@@ -238,24 +347,18 @@ lexer.embed_lexer(contextlexer, mpslexer, startmetafuncode, stopmetafuncode)
 _rules = {
     { "whitespace",  spacing     },
     { "preamble",    preamble    },
-
     { "text",        text        },
-
     { "comment",     comment     },
-
     { "constant",    constant    },
     { "helper",      helper      },
     { "command",     command     },
-    { "ifprimitive", ifprimitive },
     { "primitive",   primitive   },
+    { "ifprimitive", ifprimitive },
     { "csname",      csname      },
-
  -- { "whatever",    specialword }, -- not yet, crashes
-
     { "grouping",    grouping    },
     { "special",     special     },
     { "extra",       extra       },
-
     { "rest",        rest        },
 }