diff options
Diffstat (limited to 'context/data/scite/lexers')
22 files changed, 1244 insertions, 1726 deletions
diff --git a/context/data/scite/lexers/archive/data-pre-303.zip b/context/data/scite/lexers/archive/data-pre-303.zip Binary files differnew file mode 100644 index 000000000..4e8116d8a --- /dev/null +++ b/context/data/scite/lexers/archive/data-pre-303.zip diff --git a/context/data/scite/lexers/archive/data-pre-331.zip b/context/data/scite/lexers/archive/data-pre-331.zip Binary files differnew file mode 100644 index 000000000..f99c35147 --- /dev/null +++ b/context/data/scite/lexers/archive/data-pre-331.zip diff --git a/context/data/scite/lexers/archive/data-pre-341.zip b/context/data/scite/lexers/archive/data-pre-341.zip Binary files differnew file mode 100644 index 000000000..002855186 --- /dev/null +++ b/context/data/scite/lexers/archive/data-pre-341.zip diff --git a/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua b/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua deleted file mode 100644 index 7883177b4..000000000 --- a/context/data/scite/lexers/archive/scite-context-lexer-pre-3-3-1.lua +++ /dev/null @@ -1,1100 +0,0 @@ -local info = { - version = 1.324, - comment = "basics for scintilla lpeg lexer for context/metafun", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files", - comment = "contains copyrighted code from mitchell.att.foicica.com", - -} - --- todo: move all code here --- todo: explore adapted dll ... properties + init - --- The fold and lex functions are copied and patched from original code by Mitchell (see --- lexer.lua). All errors are mine. --- --- Starting with SciTE version 3.20 there is an issue with coloring. As we still lack --- a connection with scite itself (properties as well as printing to the log pane) we --- cannot trace this (on windows). As far as I can see, there are no fundamental --- changes in lexer.lua or LexLPeg.cxx so it must be in scintilla itself. So for the --- moment I stick to 3.10. Indicators are: no lexing of 'next' and 'goto <label>' in the --- Lua lexer and no brace highlighting either. Interesting is that it does work ok in --- the cld lexer (so the Lua code is okay). Also the fact that char-def.lua lexes fast --- is a signal that the lexer quits somewhere halfway. --- --- After checking 3.24 and adapting to the new lexer tables things are okay again. So, --- this version assumes 3.24 or higher. In 3.24 we have a different token result, i.e. no --- longer a { tag, pattern } but just two return values. I didn't check other changes but --- will do that when I run into issues. --- --- I've considered making a whole copy and patch the other functions too as we need --- an extra nesting model. However, I don't want to maintain too much. An unfortunate --- change in 3.03 is that no longer a script can be specified. This means that instead --- of loading the extensions via the properties file, we now need to load them in our --- own lexers, unless of course we replace lexer.lua completely (which adds another --- installation issue). --- --- Another change has been that _LEXERHOME is no longer available. It looks like more and --- more functionality gets dropped so maybe at some point we need to ship our own dll/so --- files. For instance, I'd like to have access to the current filename and other scite --- properties. For instance, we could cache some info with each file, if only we had --- knowledge of what file we're dealing with. --- --- For huge files folding can be pretty slow and I do have some large ones that I keep --- open all the time. Loading is normally no ussue, unless one has remembered the status --- and the cursor is at the last line of a 200K line file. Optimizing the fold function --- brought down loading of char-def.lua from 14 sec => 8 sec. Replacing the word_match --- function and optimizing the lex function gained another 2+ seconds. A 6 second load --- is quite ok for me. The changed lexer table structure (no subtables) brings loading --- down to a few seconds. --- --- When the lexer path is copied to the textadept lexer path, and the theme definition to --- theme path (as lexer.lua), the lexer works there as well. When I have time and motive --- I will make a proper setup file to tune the look and feel a bit and associate suffixes --- with the context lexer. The textadept editor has a nice style tracing option but lacks --- the tabs for selecting files that scite has. It also has no integrated run that pipes --- to the log pane (I wonder if it could borrow code from the console2 project). Interesting --- is that the jit version of textadept crashes on lexing large files (and does not feel --- faster either). --- --- Function load(lexer_name) starts with _M.WHITESPACE = lexer_name..'_whitespace' which --- means that we need to have it frozen at the moment we load another lexer. Because spacing --- is used to revert to a parent lexer we need to make sure that we load children as late --- as possible in order not to get the wrong whitespace trigger. This took me quite a while --- to figure out (not being that familiar with the internals). The lex and fold functions --- have been optimized. It is a pitty that there is no proper print available. Another thing --- needed is a default style in ourown theme style definition, as otherwise we get wrong --- nested lexers, especially if they are larger than a view. This is the hardest part of --- getting things right. --- --- Eventually it might be safer to copy the other methods from lexer.lua here as well so --- that we have no dependencies, apart from the c library (for which at some point the api --- will be stable I hope). --- --- It's a pitty that there is no scintillua library for the OSX version of scite. Even --- better would be to have the scintillua library as integral part of scite as that way I --- could use OSX alongside windows and linux (depending on needs). Also nice would be to --- have a proper interface to scite then because currently the lexer is rather isolated and the --- lua version does not provide all standard libraries. It would also be good to have lpeg --- support in the regular scite lua extension (currently you need to pick it up from someplace --- else). - -local lpeg = require 'lpeg' - -local R, P, S, C, V, Cp, Cs, Ct, Cmt, Cc, Cf, Cg, Carg = lpeg.R, lpeg.P, lpeg.S, lpeg.C, lpeg.V, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Carg -local lpegmatch = lpeg.match -local find, gmatch, match, lower, upper, gsub = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub -local concat = table.concat -local global = _G -local type, next, setmetatable, rawset = type, next, setmetatable, rawset - -if lexer then - -- in recent c++ code the lexername and loading is hard coded -elseif _LEXERHOME then - dofile(_LEXERHOME .. '/lexer.lua') -- pre 3.03 situation -else - dofile('lexer.lua') -- whatever -end - -lexer.context = lexer.context or { } -local context = lexer.context - -context.patterns = context.patterns or { } -local patterns = context.patterns - -lexer._CONTEXTEXTENSIONS = true - -local locations = { - -- lexer.context.path, - "data", -- optional data directory - "..", -- regular scite directory -} - -local function collect(name) --- local definitions = loadfile(name .. ".luc") or loadfile(name .. ".lua") - local okay, definitions = pcall(function () return require(name) end) - if okay then - if type(definitions) == "function" then - definitions = definitions() - end - if type(definitions) == "table" then - return definitions - end - end -end - -function context.loaddefinitions(name) - for i=1,#locations do - local data = collect(locations[i] .. "/" .. name) - if data then - return data - end - end -end - --- maybe more efficient: - -function context.word_match(words,word_chars,case_insensitive) - local chars = '%w_' -- maybe just "" when word_chars - if word_chars then - chars = '^([' .. chars .. gsub(word_chars,'([%^%]%-])', '%%%1') ..']+)' - else - chars = '^([' .. chars ..']+)' - end - if case_insensitive then - local word_list = { } - for i=1,#words do - word_list[lower(words[i])] = true - end - return P(function(input, index) - local s, e, word = find(input,chars,index) - return word and word_list[lower(word)] and e + 1 or nil - end) - else - local word_list = { } - for i=1,#words do - word_list[words[i]] = true - end - return P(function(input, index) - local s, e, word = find(input,chars,index) - return word and word_list[word] and e + 1 or nil - end) - end -end - -local idtoken = R("az","AZ","\127\255","__") -local digit = R("09") -local sign = S("+-") -local period = P(".") -local space = S(" \n\r\t\f\v") - -patterns.idtoken = idtoken - -patterns.digit = digit -patterns.sign = sign -patterns.period = period - -patterns.cardinal = digit^1 -patterns.integer = sign^-1 * digit^1 - -patterns.real = - sign^-1 * ( -- at most one - digit^1 * period * digit^0 -- 10.0 10. - + digit^0 * period * digit^1 -- 0.10 .10 - + digit^1 -- 10 - ) - -patterns.restofline = (1-S("\n\r"))^1 -patterns.space = space -patterns.spacing = space^1 -patterns.nospacing = (1-space)^1 -patterns.anything = P(1) - -local endof = S("\n\r\f") - -patterns.startofline = P(function(input,index) - return (index == 1 or lpegmatch(endof,input,index-1)) and index -end) - -function context.exact_match(words,word_chars,case_insensitive) - local characters = concat(words) - local pattern -- the concat catches _ etc - if word_chars == true or word_chars == false or word_chars == nil then - word_chars = "" - end - if type(word_chars) == "string" then - pattern = S(characters) + idtoken - if case_insensitive then - pattern = pattern + S(upper(characters)) + S(lower(characters)) - end - if word_chars ~= "" then - pattern = pattern + S(word_chars) - end - elseif word_chars then - pattern = word_chars - end - if case_insensitive then - local list = { } - for i=1,#words do - list[lower(words[i])] = true - end - return Cmt(pattern^1, function(_,i,s) - return list[lower(s)] -- and i or nil - end) - else - local list = { } - for i=1,#words do - list[words[i]] = true - end - return Cmt(pattern^1, function(_,i,s) - return list[s] -- and i or nil - end) - end -end - --- spell checking (we can only load lua files) --- --- return { --- min = 3, --- max = 40, --- n = 12345, --- words = { --- ["someword"] = "someword", --- ["anotherword"] = "Anotherword", --- }, --- } - -local lists = { } - -function context.setwordlist(tag,limit) -- returns hash (lowercase keys and original values) - if not tag or tag == "" then - return false, 3 - end - local list = lists[tag] - if not list then - list = context.loaddefinitions("spell-" .. tag) - if not list or type(list) ~= "table" then - list = { words = false, min = 3 } - else - list.words = list.words or false - list.min = list.min or 3 - end - lists[tag] = list - end - return list.words, list.min -end - -patterns.wordtoken = R("az","AZ","\127\255") -patterns.wordpattern = patterns.wordtoken^3 -- todo: if limit and #s < limit then - --- -- pre 3.24: --- --- function context.checkedword(validwords,validminimum,s,i) -- ,limit --- if not validwords then -- or #s < validminimum then --- return true, { "text", i } -- { "default", i } --- else --- -- keys are lower --- local word = validwords[s] --- if word == s then --- return true, { "okay", i } -- exact match --- elseif word then --- return true, { "warning", i } -- case issue --- else --- local word = validwords[lower(s)] --- if word == s then --- return true, { "okay", i } -- exact match --- elseif word then --- return true, { "warning", i } -- case issue --- elseif upper(s) == s then --- return true, { "warning", i } -- probably a logo or acronym --- else --- return true, { "error", i } --- end --- end --- end --- end - -function context.checkedword(validwords,validminimum,s,i) -- ,limit - if not validwords then -- or #s < validminimum then - return true, "text", i -- { "default", i } - else - -- keys are lower - local word = validwords[s] - if word == s then - return true, "okay", i -- exact match - elseif word then - return true, "warning", i -- case issue - else - local word = validwords[lower(s)] - if word == s then - return true, "okay", i -- exact match - elseif word then - return true, "warning", i -- case issue - elseif upper(s) == s then - return true, "warning", i -- probably a logo or acronym - else - return true, "error", i - end - end - end -end - -function context.styleofword(validwords,validminimum,s) -- ,limit - if not validwords or #s < validminimum then - return "text" - else - -- keys are lower - local word = validwords[s] - if word == s then - return "okay" -- exact match - elseif word then - return "warning" -- case issue - else - local word = validwords[lower(s)] - if word == s then - return "okay" -- exact match - elseif word then - return "warning" -- case issue - elseif upper(s) == s then - return "warning" -- probably a logo or acronym - else - return "error" - end - end - end -end - --- overloaded functions - -local FOLD_BASE = SC_FOLDLEVELBASE -local FOLD_HEADER = SC_FOLDLEVELHEADERFLAG -local FOLD_BLANK = SC_FOLDLEVELWHITEFLAG - -local get_style_at = GetStyleAt -local get_property = GetProperty -local get_indent_amount = GetIndentAmount - -local h_table, b_table, n_table = { }, { }, { } - -setmetatable(h_table, { __index = function(t,level) local v = { level, FOLD_HEADER } t[level] = v return v end }) -setmetatable(b_table, { __index = function(t,level) local v = { level, FOLD_BLANK } t[level] = v return v end }) -setmetatable(n_table, { __index = function(t,level) local v = { level } t[level] = v return v end }) - --- -- todo: move the local functions outside (see below) .. old variant < 3.24 --- --- local newline = P("\r\n") + S("\r\n") --- local p_yes = Cp() * Cs((1-newline)^1) * newline^-1 --- local p_nop = newline --- --- local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) --- local foldsymbols = lexer._foldsymbols --- if not foldsymbols then --- return { } --- end --- local patterns = foldsymbols._patterns --- if not patterns then --- return { } --- end --- local nofpatterns = #patterns --- if nofpatterns == 0 then --- return { } --- end --- local folds = { } --- local line_num = start_line --- local prev_level = start_level --- local current_level = prev_level --- local validmatches = foldsymbols._validmatches --- if not validmatches then --- validmatches = { } --- for symbol, matches in next, foldsymbols do -- whatever = { start = 1, stop = -1 } --- if not find(symbol,"^_") then -- brrr --- for s, _ in next, matches do --- validmatches[s] = true --- end --- end --- end --- foldsymbols._validmatches = validmatches --- end --- -- of course we could instead build a nice lpeg checker .. something for --- -- a rainy day with a stack of new cd's at hand --- local function action_y(pos,line) --- for i=1,nofpatterns do --- for s, m in gmatch(line,patterns[i]) do --- if validmatches[m] then --- local symbols = foldsymbols[get_style_at(start_pos + pos + s - 1)] --- if symbols then --- local action = symbols[m] --- if action then --- if type(action) == 'number' then -- we could store this in validmatches if there was only one symbol category --- current_level = current_level + action --- else --- current_level = current_level + action(text,pos,line,s,m) --- end --- if current_level < FOLD_BASE then --- current_level = FOLD_BASE --- end --- end --- end --- end --- end --- end --- if current_level > prev_level then --- folds[line_num] = h_table[prev_level] -- { prev_level, FOLD_HEADER } --- else --- folds[line_num] = n_table[prev_level] -- { prev_level } --- end --- prev_level = current_level --- line_num = line_num + 1 --- end --- local function action_n() --- folds[line_num] = b_table[prev_level] -- { prev_level, FOLD_BLANK } --- line_num = line_num + 1 --- end --- if lexer._reset_parser then --- lexer._reset_parser() --- end --- local lpegpattern = (p_yes/action_y + p_nop/action_n)^0 -- not too efficient but indirect function calls are neither but --- lpegmatch(lpegpattern,text) -- keys are not pressed that fast ... large files are slow anyway --- return folds --- end - --- The 3.24 variant; no longer subtable optimization is needed: - -local newline = P("\r\n") + S("\r\n") -local p_yes = Cp() * Cs((1-newline)^1) * newline^-1 -local p_nop = newline - -local folders = { } - -local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) - local folder = folders[lexer] - if not folder then - -- - local pattern, folds, text, start_pos, line_num, prev_level, current_level - -- - local fold_symbols = lexer._foldsymbols - local fold_pattern = lexer._foldpattern -- use lpeg instead (context extension) - -- - if fold_pattern then - -- if no functions are found then we could have a faster one - - -- fold_pattern = Cp() * C(fold_pattern) * Carg(1) / function(s,match,pos) - -- local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)] - -- local l = symbols and symbols[match] - -- if l then - -- local t = type(l) - -- if t == 'number' then - -- current_level = current_level + l - -- elseif t == 'function' then - -- current_level = current_level + l(text, pos, line, s, match) - -- end - -- end - -- end - -- fold_pattern = (fold_pattern + P(1))^0 - -- local action_y = function(pos,line) - -- lpegmatch(fold_pattern,line,1,pos) - -- folds[line_num] = prev_level - -- if current_level > prev_level then - -- folds[line_num] = prev_level + FOLD_HEADER - -- end - -- if current_level < FOLD_BASE then - -- current_level = FOLD_BASE - -- end - -- prev_level = current_level - -- line_num = line_num + 1 - -- end - -- local action_n = function() - -- folds[line_num] = prev_level + FOLD_BLANK - -- line_num = line_num + 1 - -- end - -- pattern = (p_yes/action_y + p_nop/action_n)^0 - - fold_pattern = Cp() * C(fold_pattern) / function(s,match) - local symbols = fold_symbols[get_style_at(start_pos + s)] - if symbols then - local l = symbols[match] - if l then - current_level = current_level + l - end - end - end - local action_y = function() - folds[line_num] = prev_level - if current_level > prev_level then - folds[line_num] = prev_level + FOLD_HEADER - end - if current_level < FOLD_BASE then - current_level = FOLD_BASE - end - prev_level = current_level - line_num = line_num + 1 - end - local action_n = function() - folds[line_num] = prev_level + FOLD_BLANK - line_num = line_num + 1 - end - pattern = ((fold_pattern + (1-newline))^1 * newline / action_y + newline/action_n)^0 - - else - -- the traditional one but a bit optimized - local fold_symbols_patterns = fold_symbols._patterns - local action_y = function(pos,line) - for j = 1, #fold_symbols_patterns do - for s, match in gmatch(line,fold_symbols_patterns[j]) do -- '()('..patterns[i]..')' - local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)] - local l = symbols and symbols[match] - local t = type(l) - if t == 'number' then - current_level = current_level + l - elseif t == 'function' then - current_level = current_level + l(text, pos, line, s, match) - end - end - end - folds[line_num] = prev_level - if current_level > prev_level then - folds[line_num] = prev_level + FOLD_HEADER - end - if current_level < FOLD_BASE then - current_level = FOLD_BASE - end - prev_level = current_level - line_num = line_num + 1 - end - local action_n = function() - folds[line_num] = prev_level + FOLD_BLANK - line_num = line_num + 1 - end - pattern = (p_yes/action_y + p_nop/action_n)^0 - end - -- - local reset_parser = lexer._reset_parser - -- - folder = function(_text_,_start_pos_,_start_line_,_start_level_) - if reset_parser then - reset_parser() - end - folds = { } - text = _text_ - start_pos = _start_pos_ - line_num = _start_line_ - prev_level = _start_level_ - current_level = prev_level - lpegmatch(pattern,text) --- return folds -local t = folds -folds = nil -return t -- so folds can be collected - end - folders[lexer] = folder - end - return folder(text,start_pos,start_line,start_level,lexer) -end - --- local function fold_by_indentation(text,start_pos,start_line,start_level) --- local folds = { } --- local current_line = start_line --- local prev_level = start_level --- for line in gmatch(text,'[\t ]*(.-)\r?\n') do --- if line ~= "" then --- local current_level = FOLD_BASE + get_indent_amount(current_line) --- if current_level > prev_level then -- next level --- local i = current_line - 1 --- while true do --- local f = folds[i] --- if f and f[2] == FOLD_BLANK then --- i = i - 1 --- else --- break --- end --- end --- local f = folds[i] --- if f then --- f[2] = FOLD_HEADER --- end -- low indent --- folds[current_line] = n_table[current_level] -- { current_level } -- high indent --- elseif current_level < prev_level then -- prev level --- local f = folds[current_line - 1] --- if f then --- f[1] = prev_level -- high indent --- end --- folds[current_line] = n_table[current_level] -- { current_level } -- low indent --- else -- same level --- folds[current_line] = n_table[prev_level] -- { prev_level } --- end --- prev_level = current_level --- else --- folds[current_line] = b_table[prev_level] -- { prev_level, FOLD_BLANK } --- end --- current_line = current_line + 1 --- end --- return folds --- end - --- local function fold_by_indentation(text,start_pos,start_line,start_level) --- local folds = { } --- local current_line = start_line --- local prev_level = start_level --- for line in gmatch(text,'[\t ]*(.-)\r?\n') do --- if line ~= '' then --- local current_level = FOLD_BASE + get_indent_amount(current_line) --- if current_level > prev_level then -- next level --- local i = current_line - 1 --- local f --- while true do --- f = folds[i] --- if not f then --- break --- elseif f[2] == FOLD_BLANK then --- i = i - 1 --- else --- f[2] = FOLD_HEADER -- low indent --- break --- end --- end --- folds[current_line] = { current_level } -- high indent --- elseif current_level < prev_level then -- prev level --- local f = folds[current_line - 1] --- if f then --- f[1] = prev_level -- high indent --- end --- folds[current_line] = { current_level } -- low indent --- else -- same level --- folds[current_line] = { prev_level } --- end --- prev_level = current_level --- else --- folds[current_line] = { prev_level, FOLD_BLANK } --- end --- current_line = current_line + 1 --- end --- for line, level in next, folds do --- folds[line] = level[1] + (level[2] or 0) --- end --- return folds --- end - -local folds, current_line, prev_level - -local function action_y() - local current_level = FOLD_BASE + get_indent_amount(current_line) - if current_level > prev_level then -- next level - local i = current_line - 1 - local f - while true do - f = folds[i] - if not f then - break - elseif f[2] == FOLD_BLANK then - i = i - 1 - else - f[2] = FOLD_HEADER -- low indent - break - end - end - folds[current_line] = { current_level } -- high indent - elseif current_level < prev_level then -- prev level - local f = folds[current_line - 1] - if f then - f[1] = prev_level -- high indent - end - folds[current_line] = { current_level } -- low indent - else -- same level - folds[current_line] = { prev_level } - end - prev_level = current_level - current_line = current_line + 1 -end - -local function action_n() - folds[current_line] = { prev_level, FOLD_BLANK } - current_line = current_line + 1 -end - -local pattern = ( S("\t ")^0 * ( (1-S("\n\r"))^1 / action_y + P(true) / action_n) * newline )^0 - -local function fold_by_indentation(text,start_pos,start_line,start_level) - -- initialize - folds = { } - current_line = start_line - prev_level = start_level - -- define - -- -- not here .. pattern binds and local functions are not frozen - -- analyze - lpegmatch(pattern,text) - -- flatten - for line, level in next, folds do - folds[line] = level[1] + (level[2] or 0) - end - -- done --- return folds -local t = folds -folds = nil -return t -- so folds can be collected -end - -local function fold_by_line(text,start_pos,start_line,start_level) - local folds = { } - -- can also be lpeg'd - for _ in gmatch(text,".-\r?\n") do - folds[start_line] = n_table[start_level] -- { start_level } - start_line = start_line + 1 - end - return folds -end - -local threshold_by_lexer = 512 * 1024 -- we don't know the filesize yet -local threshold_by_parsing = 512 * 1024 -- we don't know the filesize yet -local threshold_by_indentation = 512 * 1024 -- we don't know the filesize yet -local threshold_by_line = 512 * 1024 -- we don't know the filesize yet - -function context.fold(text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go - if text == '' then - return { } - end - local lexer = global._LEXER - local fold_by_lexer = lexer._fold - local fold_by_symbols = lexer._foldsymbols - local filesize = 0 -- we don't know that - if fold_by_lexer then - if filesize <= threshold_by_lexer then - return fold_by_lexer(text,start_pos,start_line,start_level,lexer) - end - elseif fold_by_symbols then -- and get_property('fold.by.parsing',1) > 0 then - if filesize <= threshold_by_parsing then - return fold_by_parsing(text,start_pos,start_line,start_level,lexer) - end - elseif get_property('fold.by.indentation',1) > 0 then - if filesize <= threshold_by_indentation then - return fold_by_indentation(text,start_pos,start_line,start_level,lexer) - end - elseif get_property('fold.by.line',1) > 0 then - if filesize <= threshold_by_line then - return fold_by_line(text,start_pos,start_line,start_level,lexer) - end - end - return { } -end - --- The following code is mostly unchanged: - -local function add_rule(lexer, id, rule) - if not lexer._RULES then - lexer._RULES = {} - lexer._RULEORDER = {} - end - lexer._RULES[id] = rule - lexer._RULEORDER[#lexer._RULEORDER + 1] = id -end - -local function add_style(lexer, token_name, style) - local len = lexer._STYLES.len - if len == 32 then - len = len + 8 - end - if len >= 128 then - print('Too many styles defined (128 MAX)') - end - lexer._TOKENS[token_name] = len - lexer._STYLES[len] = style - lexer._STYLES.len = len + 1 -end - -local function join_tokens(lexer) - local patterns, order = lexer._RULES, lexer._RULEORDER - local token_rule = patterns[order[1]] - for i=2,#order do - token_rule = token_rule + patterns[order[i]] - end - lexer._TOKENRULE = token_rule - return lexer._TOKENRULE -end - -local function add_lexer(grammar, lexer, token_rule) - local token_rule = join_tokens(lexer) - local lexer_name = lexer._NAME - local children = lexer._CHILDREN - for i=1,#children do - local child = children[i] - if child._CHILDREN then - add_lexer(grammar, child) - end - local child_name = child._NAME - local rules = child._EMBEDDEDRULES[lexer_name] - local rules_token_rule = grammar['__'..child_name] or rules.token_rule - grammar[child_name] = (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 * V(lexer_name) - local embedded_child = '_' .. child_name - grammar[embedded_child] = rules.start_rule * (-rules.end_rule * rules_token_rule)^0 * rules.end_rule^-1 - token_rule = V(embedded_child) + token_rule - end - grammar['__' .. lexer_name] = token_rule - grammar[lexer_name] = token_rule^0 -end - -local function build_grammar(lexer, initial_rule) - local children = lexer._CHILDREN - if children then - local lexer_name = lexer._NAME - if not initial_rule then - initial_rule = lexer_name - end - local grammar = { initial_rule } - add_lexer(grammar, lexer) - lexer._INITIALRULE = initial_rule - lexer._GRAMMAR = Ct(P(grammar)) - else - lexer._GRAMMAR = Ct(join_tokens(lexer)^0) - end -end - --- so far. We need these local functions in the next one. --- --- Before 3.24 we had tokens[..] = { category, position }, now it's a two values. - -local lineparsers = { } - -function context.lex(text,init_style) - local lexer = global._LEXER - local grammar = lexer._GRAMMAR - if not grammar then - return { } - elseif lexer._LEXBYLINE then -- we could keep token - local tokens = { } - local offset = 0 - local noftokens = 0 - -- -- pre 3.24 - -- - -- for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg - -- local line_tokens = lpegmatch(grammar,line) - -- if line_tokens then - -- for i=1,#line_tokens do - -- local token = line_tokens[i] - -- token[2] = token[2] + offset - -- noftokens = noftokens + 1 - -- tokens[noftokens] = token - -- end - -- end - -- offset = offset + #line - -- if noftokens > 0 and tokens[noftokens][2] ~= offset then - -- noftokens = noftokens + 1 - -- tokens[noftokens] = { 'default', offset + 1 } - -- end - -- end - - -- for line in gmatch(text,'[^\r\n]*\r?\n?') do - -- local line_tokens = lpegmatch(grammar,line) - -- if line_tokens then - -- for i=1,#line_tokens,2 do - -- noftokens = noftokens + 1 - -- tokens[noftokens] = line_tokens[i] - -- noftokens = noftokens + 1 - -- tokens[noftokens] = line_tokens[i + 1] + offset - -- end - -- end - -- offset = offset + #line - -- if noftokens > 0 and tokens[noftokens] ~= offset then - -- noftokens = noftokens + 1 - -- tokens[noftokens] = 'default' - -- noftokens = noftokens + 1 - -- tokens[noftokens] = offset + 1 - -- end - -- end - - local lineparser = lineparsers[lexer] - if not lineparser then -- probably a cmt is more efficient - lineparser = C((1-newline)^0 * newline) / function(line) - local length = #line - local line_tokens = length > 0 and lpegmatch(grammar,line) - if line_tokens then - for i=1,#line_tokens,2 do - noftokens = noftokens + 1 - tokens[noftokens] = line_tokens[i] - noftokens = noftokens + 1 - tokens[noftokens] = line_tokens[i + 1] + offset - end - end - offset = offset + length - if noftokens > 0 and tokens[noftokens] ~= offset then - noftokens = noftokens + 1 - tokens[noftokens] = 'default' - noftokens = noftokens + 1 - tokens[noftokens] = offset + 1 - end - end - lineparser = lineparser^0 - lineparsers[lexer] = lineparser - end - lpegmatch(lineparser,text) - return tokens - - elseif lexer._CHILDREN then - -- as we cannot print, tracing is not possible ... this might change as we can as well - -- generate them all in one go (sharing as much as possible) - local hash = lexer._HASH -- hm, was _hash - if not hash then - hash = { } - lexer._HASH = hash - end - grammar = hash[init_style] - if grammar then - lexer._GRAMMAR = grammar - else - for style, style_num in next, lexer._TOKENS do - if style_num == init_style then - -- the name of the lexers is filtered from the whitespace - -- specification - local lexer_name = match(style,'^(.+)_whitespace') or lexer._NAME - if lexer._INITIALRULE ~= lexer_name then - grammar = hash[lexer_name] - if not grammar then - build_grammar(lexer,lexer_name) - grammar = lexer._GRAMMAR - hash[lexer_name] = grammar - end - end - break - end - end - grammar = grammar or lexer._GRAMMAR - hash[init_style] = grammar - end - return lpegmatch(grammar,text) - else - return lpegmatch(grammar,text) - end -end - --- todo: keywords: one lookup and multiple matches - --- function context.token(name, patt) --- return Ct(patt * Cc(name) * Cp()) --- end --- --- -- hm, changed in 3.24 .. no longer a table - -function context.token(name, patt) - return patt * Cc(name) * Cp() -end - -lexer.fold = context.fold -lexer.lex = context.lex -lexer.token = context.token -lexer.exact_match = context.exact_match - --- helper .. alas ... the lexer's lua instance is rather crippled .. not even --- math is part of it - -local floor = math and math.floor -local char = string.char - -if not floor then - - floor = function(n) - return tonumber(string.format("%d",n)) - end - - math = math or { } - - math.floor = floor - -end - -local function utfchar(n) - if n < 0x80 then - return char(n) - elseif n < 0x800 then - return char( - 0xC0 + floor(n/0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x10000 then - return char( - 0xE0 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x40000 then - return char( - 0xF0 + floor(n/0x40000), - 0x80 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - else - -- return char( - -- 0xF1 + floor(n/0x1000000), - -- 0x80 + floor(n/0x40000), - -- 0x80 + floor(n/0x1000), - -- 0x80 + (floor(n/0x40) % 0x40), - -- 0x80 + (n % 0x40) - -- ) - return "?" - end -end - -context.utfchar = utfchar - --- a helper from l-lpeg: - -local gmatch = string.gmatch - -local function make(t) - local p - for k, v in next, t do - if not p then - if next(v) then - p = P(k) * make(v) - else - p = P(k) - end - else - if next(v) then - p = p + P(k) * make(v) - else - p = p + P(k) - end - end - end - return p -end - -function lpeg.utfchartabletopattern(list) - local tree = { } - for i=1,#list do - local t = tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c] = { } - end - t = t[c] - end - end - return make(tree) -end - --- patterns.invisibles = --- P(utfchar(0x00A0)) -- nbsp --- + P(utfchar(0x2000)) -- enquad --- + P(utfchar(0x2001)) -- emquad --- + P(utfchar(0x2002)) -- enspace --- + P(utfchar(0x2003)) -- emspace --- + P(utfchar(0x2004)) -- threeperemspace --- + P(utfchar(0x2005)) -- fourperemspace --- + P(utfchar(0x2006)) -- sixperemspace --- + P(utfchar(0x2007)) -- figurespace --- + P(utfchar(0x2008)) -- punctuationspace --- + P(utfchar(0x2009)) -- breakablethinspace --- + P(utfchar(0x200A)) -- hairspace --- + P(utfchar(0x200B)) -- zerowidthspace --- + P(utfchar(0x202F)) -- narrownobreakspace --- + P(utfchar(0x205F)) -- math thinspace - -patterns.invisibles = lpeg.utfchartabletopattern { - utfchar(0x00A0), -- nbsp - utfchar(0x2000), -- enquad - utfchar(0x2001), -- emquad - utfchar(0x2002), -- enspace - utfchar(0x2003), -- emspace - utfchar(0x2004), -- threeperemspace - utfchar(0x2005), -- fourperemspace - utfchar(0x2006), -- sixperemspace - utfchar(0x2007), -- figurespace - utfchar(0x2008), -- punctuationspace - utfchar(0x2009), -- breakablethinspace - utfchar(0x200A), -- hairspace - utfchar(0x200B), -- zerowidthspace - utfchar(0x202F), -- narrownobreakspace - utfchar(0x205F), -- math thinspace -} - --- now we can make: - -patterns.iwordtoken = patterns.wordtoken - patterns.invisibles -patterns.iwordpattern = patterns.iwordtoken^3 - --- require("themes/scite-context-theme") - --- In order to deal with some bug in additional styles (I have no cue what is --- wrong, but additional styles get ignored and clash somehow) I just copy the --- original lexer code ... see original for comments. diff --git a/context/data/scite/lexers/data/scite-context-data-context.lua b/context/data/scite/lexers/data/scite-context-data-context.lua index 720eb1d22..6c0293fbd 100644 --- a/context/data/scite/lexers/data/scite-context-data-context.lua +++ b/context/data/scite/lexers/data/scite-context-data-context.lua @@ -1,4 +1,4 @@ return { - ["constants"]={ "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive", "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred", "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "zeropoint", "onepoint", "halfapoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint", "points", "halfpoint", "zeroskip", "zeromuskip", "onemuskip", "pluscxxvii", "pluscxxviii", "pluscclv", "pluscclvi", "normalpagebox", "endoflinetoken", "outputnewlinechar", "emptytoks", "empty", "undefined", "voidbox", "emptybox", "emptyvbox", "emptyhbox", "bigskipamount", "medskipamount", "smallskipamount", "fmtname", "fmtversion", "texengine", "texenginename", "texengineversion", "luatexengine", "pdftexengine", "xetexengine", "unknownengine", "etexversion", "pdftexversion", "xetexversion", "xetexrevision", "activecatcode", "bgroup", "egroup", "endline", "conditionaltrue", "conditionalfalse", "attributeunsetvalue", "uprotationangle", "rightrotationangle", "downrotationangle", "leftrotationangle", "inicatcodes", "ctxcatcodes", "texcatcodes", "notcatcodes", "txtcatcodes", "vrbcatcodes", "prtcatcodes", "nilcatcodes", "luacatcodes", "tpacatcodes", "tpbcatcodes", "xmlcatcodes", "ctdcatcodes", "escapecatcode", "begingroupcatcode", "endgroupcatcode", "mathshiftcatcode", "alignmentcatcode", "endoflinecatcode", "parametercatcode", "superscriptcatcode", "subscriptcatcode", "ignorecatcode", "spacecatcode", "lettercatcode", "othercatcode", "activecatcode", "commentcatcode", "invalidcatcode", "tabasciicode", "newlineasciicode", "formfeedasciicode", "endoflineasciicode", "endoffileasciicode", "spaceasciicode", "hashasciicode", "dollarasciicode", "commentasciicode", "ampersandasciicode", "colonasciicode", "backslashasciicode", "circumflexasciicode", "underscoreasciicode", "leftbraceasciicode", "barasciicode", "rightbraceasciicode", "tildeasciicode", "delasciicode", "lessthanasciicode", "morethanasciicode", "doublecommentsignal", "atsignasciicode", "exclamationmarkasciicode", "questionmarkasciicode", "doublequoteasciicode", "singlequoteasciicode", "forwardslashasciicode", "primeasciicode", "activemathcharcode", "activetabtoken", "activeformfeedtoken", "activeendoflinetoken", "batchmodecode", "nonstopmodecode", "scrollmodecode", "errorstopmodecode", "bottomlevelgroupcode", "simplegroupcode", "hboxgroupcode", "adjustedhboxgroupcode", "vboxgroupcode", "vtopgroupcode", "aligngroupcode", "noaligngroupcode", "outputgroupcode", "mathgroupcode", "discretionarygroupcode", "insertgroupcode", "vcentergroupcode", "mathchoicegroupcode", "semisimplegroupcode", "mathshiftgroupcode", "mathleftgroupcode", "vadjustgroupcode", "charnodecode", "hlistnodecode", "vlistnodecode", "rulenodecode", "insertnodecode", "marknodecode", "adjustnodecode", "ligaturenodecode", "discretionarynodecode", "whatsitnodecode", "mathnodecode", "gluenodecode", "kernnodecode", "penaltynodecode", "unsetnodecode", "mathsnodecode", "charifcode", "catifcode", "numifcode", "dimifcode", "oddifcode", "vmodeifcode", "hmodeifcode", "mmodeifcode", "innerifcode", "voidifcode", "hboxifcode", "vboxifcode", "xifcode", "eofifcode", "trueifcode", "falseifcode", "caseifcode", "definedifcode", "csnameifcode", "fontcharifcode", "fontslantperpoint", "fontinterwordspace", "fontinterwordstretch", "fontinterwordshrink", "fontexheight", "fontemwidth", "fontextraspace", "slantperpoint", "interwordspace", "interwordstretch", "interwordshrink", "exheight", "emwidth", "extraspace", "mathsupdisplay", "mathsupnormal", "mathsupcramped", "mathsubnormal", "mathsubcombined", "mathaxisheight", "startmode", "stopmode", "startnotmode", "stopnotmode", "startmodeset", "stopmodeset", "doifmode", "doifmodeelse", "doifnotmode", "startallmodes", "stopallmodes", "startnotallmodes", "stopnotallmodes", "doifallmodes", "doifallmodeselse", "doifnotallmodes", "startenvironment", "stopenvironment", "environment", "startcomponent", "stopcomponent", "component", "startproduct", "stopproduct", "product", "startproject", "stopproject", "project", "starttext", "stoptext", "startnotext", "stopnotext", "startdocument", "stopdocument", "documentvariable", "setupdocument", "startmodule", "stopmodule", "usemodule", "usetexmodule", "useluamodule", "setupmodule", "currentmoduleparameter", "moduleparameter", "startTEXpage", "stopTEXpage", "enablemode", "disablemode", "preventmode", "globalenablemode", "globaldisablemode", "globalpreventmode", "pushmode", "popmode", "typescriptone", "typescripttwo", "typescriptthree", "mathsizesuffix", "mathordcode", "mathopcode", "mathbincode", "mathrelcode", "mathopencode", "mathclosecode", "mathpunctcode", "mathalphacode", "mathinnercode", "mathnothingcode", "mathlimopcode", "mathnolopcode", "mathboxcode", "mathchoicecode", "mathaccentcode", "mathradicalcode", "constantnumber", "constantnumberargument", "constantdimen", "constantdimenargument", "constantemptyargument", "continueifinputfile", "luastringsep", "!!bs", "!!es", "lefttorightmark", "righttoleftmark", "breakablethinspace", "nobreakspace", "narrownobreakspace", "zerowidthnobreakspace", "ideographicspace", "ideographichalffillspace", "twoperemspace", "threeperemspace", "fourperemspace", "fiveperemspace", "sixperemspace", "figurespace", "punctuationspace", "hairspace", "zerowidthspace", "zerowidthnonjoiner", "zerowidthjoiner", "zwnj", "zwj" }, + ["constants"]={ "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive", "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred", "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "zeropoint", "onepoint", "halfapoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint", "points", "halfpoint", "zeroskip", "zeromuskip", "onemuskip", "pluscxxvii", "pluscxxviii", "pluscclv", "pluscclvi", "normalpagebox", "endoflinetoken", "outputnewlinechar", "emptytoks", "empty", "undefined", "voidbox", "emptybox", "emptyvbox", "emptyhbox", "bigskipamount", "medskipamount", "smallskipamount", "fmtname", "fmtversion", "texengine", "texenginename", "texengineversion", "luatexengine", "pdftexengine", "xetexengine", "unknownengine", "etexversion", "pdftexversion", "xetexversion", "xetexrevision", "activecatcode", "bgroup", "egroup", "endline", "conditionaltrue", "conditionalfalse", "attributeunsetvalue", "uprotationangle", "rightrotationangle", "downrotationangle", "leftrotationangle", "inicatcodes", "ctxcatcodes", "texcatcodes", "notcatcodes", "txtcatcodes", "vrbcatcodes", "prtcatcodes", "nilcatcodes", "luacatcodes", "tpacatcodes", "tpbcatcodes", "xmlcatcodes", "ctdcatcodes", "escapecatcode", "begingroupcatcode", "endgroupcatcode", "mathshiftcatcode", "alignmentcatcode", "endoflinecatcode", "parametercatcode", "superscriptcatcode", "subscriptcatcode", "ignorecatcode", "spacecatcode", "lettercatcode", "othercatcode", "activecatcode", "commentcatcode", "invalidcatcode", "tabasciicode", "newlineasciicode", "formfeedasciicode", "endoflineasciicode", "endoffileasciicode", "spaceasciicode", "hashasciicode", "dollarasciicode", "commentasciicode", "ampersandasciicode", "colonasciicode", "backslashasciicode", "circumflexasciicode", "underscoreasciicode", "leftbraceasciicode", "barasciicode", "rightbraceasciicode", "tildeasciicode", "delasciicode", "lessthanasciicode", "morethanasciicode", "doublecommentsignal", "atsignasciicode", "exclamationmarkasciicode", "questionmarkasciicode", "doublequoteasciicode", "singlequoteasciicode", "forwardslashasciicode", "primeasciicode", "activemathcharcode", "activetabtoken", "activeformfeedtoken", "activeendoflinetoken", "batchmodecode", "nonstopmodecode", "scrollmodecode", "errorstopmodecode", "bottomlevelgroupcode", "simplegroupcode", "hboxgroupcode", "adjustedhboxgroupcode", "vboxgroupcode", "vtopgroupcode", "aligngroupcode", "noaligngroupcode", "outputgroupcode", "mathgroupcode", "discretionarygroupcode", "insertgroupcode", "vcentergroupcode", "mathchoicegroupcode", "semisimplegroupcode", "mathshiftgroupcode", "mathleftgroupcode", "vadjustgroupcode", "charnodecode", "hlistnodecode", "vlistnodecode", "rulenodecode", "insertnodecode", "marknodecode", "adjustnodecode", "ligaturenodecode", "discretionarynodecode", "whatsitnodecode", "mathnodecode", "gluenodecode", "kernnodecode", "penaltynodecode", "unsetnodecode", "mathsnodecode", "charifcode", "catifcode", "numifcode", "dimifcode", "oddifcode", "vmodeifcode", "hmodeifcode", "mmodeifcode", "innerifcode", "voidifcode", "hboxifcode", "vboxifcode", "xifcode", "eofifcode", "trueifcode", "falseifcode", "caseifcode", "definedifcode", "csnameifcode", "fontcharifcode", "fontslantperpoint", "fontinterwordspace", "fontinterwordstretch", "fontinterwordshrink", "fontexheight", "fontemwidth", "fontextraspace", "slantperpoint", "interwordspace", "interwordstretch", "interwordshrink", "exheight", "emwidth", "extraspace", "mathsupdisplay", "mathsupnormal", "mathsupcramped", "mathsubnormal", "mathsubcombined", "mathaxisheight", "startmode", "stopmode", "startnotmode", "stopnotmode", "startmodeset", "stopmodeset", "doifmode", "doifmodeelse", "doifnotmode", "startmodeset", "stopmodeset", "startallmodes", "stopallmodes", "startnotallmodes", "stopnotallmodes", "doifallmodes", "doifallmodeselse", "doifnotallmodes", "startenvironment", "stopenvironment", "environment", "startcomponent", "stopcomponent", "component", "startproduct", "stopproduct", "product", "startproject", "stopproject", "project", "starttext", "stoptext", "startnotext", "stopnotext", "startdocument", "stopdocument", "documentvariable", "setupdocument", "startmodule", "stopmodule", "usemodule", "usetexmodule", "useluamodule", "setupmodule", "currentmoduleparameter", "moduleparameter", "startTEXpage", "stopTEXpage", "enablemode", "disablemode", "preventmode", "globalenablemode", "globaldisablemode", "globalpreventmode", "pushmode", "popmode", "typescriptone", "typescripttwo", "typescriptthree", "mathsizesuffix", "mathordcode", "mathopcode", "mathbincode", "mathrelcode", "mathopencode", "mathclosecode", "mathpunctcode", "mathalphacode", "mathinnercode", "mathnothingcode", "mathlimopcode", "mathnolopcode", "mathboxcode", "mathchoicecode", "mathaccentcode", "mathradicalcode", "constantnumber", "constantnumberargument", "constantdimen", "constantdimenargument", "constantemptyargument", "continueifinputfile", "luastringsep", "!!bs", "!!es", "lefttorightmark", "righttoleftmark", "breakablethinspace", "nobreakspace", "narrownobreakspace", "zerowidthnobreakspace", "ideographicspace", "ideographichalffillspace", "twoperemspace", "threeperemspace", "fourperemspace", "fiveperemspace", "sixperemspace", "figurespace", "punctuationspace", "hairspace", "zerowidthspace", "zerowidthnonjoiner", "zerowidthjoiner", "zwnj", "zwj" }, ["helpers"]={ "startsetups", "stopsetups", "startxmlsetups", "stopxmlsetups", "startluasetups", "stopluasetups", "starttexsetups", "stoptexsetups", "startrawsetups", "stoprawsetups", "startlocalsetups", "stoplocalsetups", "starttexdefinition", "stoptexdefinition", "starttexcode", "stoptexcode", "startcontextcode", "stopcontextcode", "startcontextdefinitioncode", "stopcontextdefinitioncode", "doifsetupselse", "doifsetups", "doifnotsetups", "setup", "setups", "texsetup", "xmlsetup", "luasetup", "directsetup", "doifelsecommandhandler", "doifnotcommandhandler", "doifcommandhandler", "newmode", "setmode", "resetmode", "newsystemmode", "setsystemmode", "resetsystemmode", "pushsystemmode", "popsystemmode", "booleanmodevalue", "newcount", "newdimen", "newskip", "newmuskip", "newbox", "newtoks", "newread", "newwrite", "newmarks", "newinsert", "newattribute", "newif", "newlanguage", "newfamily", "newfam", "newhelp", "then", "begcsname", "strippedcsname", "firstargumentfalse", "firstargumenttrue", "secondargumentfalse", "secondargumenttrue", "thirdargumentfalse", "thirdargumenttrue", "fourthargumentfalse", "fourthargumenttrue", "fifthargumentfalse", "fifthsargumenttrue", "sixthargumentfalse", "sixtsargumenttrue", "doglobal", "dodoglobal", "redoglobal", "resetglobal", "donothing", "dontcomplain", "forgetall", "donetrue", "donefalse", "htdp", "unvoidbox", "hfilll", "vfilll", "mathbox", "mathlimop", "mathnolop", "mathnothing", "mathalpha", "currentcatcodetable", "defaultcatcodetable", "catcodetablename", "newcatcodetable", "startcatcodetable", "stopcatcodetable", "startextendcatcodetable", "stopextendcatcodetable", "pushcatcodetable", "popcatcodetable", "restorecatcodes", "setcatcodetable", "letcatcodecommand", "defcatcodecommand", "uedcatcodecommand", "hglue", "vglue", "hfillneg", "vfillneg", "hfilllneg", "vfilllneg", "ruledhss", "ruledhfil", "ruledhfill", "ruledhfilneg", "ruledhfillneg", "normalhfillneg", "ruledvss", "ruledvfil", "ruledvfill", "ruledvfilneg", "ruledvfillneg", "normalvfillneg", "ruledhbox", "ruledvbox", "ruledvtop", "ruledvcenter", "ruledmbox", "ruledhskip", "ruledvskip", "ruledkern", "ruledmskip", "ruledmkern", "ruledhglue", "ruledvglue", "normalhglue", "normalvglue", "ruledpenalty", "filledhboxb", "filledhboxr", "filledhboxg", "filledhboxc", "filledhboxm", "filledhboxy", "filledhboxk", "scratchcounter", "globalscratchcounter", "scratchdimen", "globalscratchdimen", "scratchskip", "globalscratchskip", "scratchmuskip", "globalscratchmuskip", "scratchtoks", "globalscratchtoks", "scratchbox", "globalscratchbox", "normalbaselineskip", "normallineskip", "normallineskiplimit", "availablehsize", "localhsize", "setlocalhsize", "nextbox", "dowithnextbox", "dowithnextboxcs", "dowithnextboxcontent", "dowithnextboxcontentcs", "scratchwidth", "scratchheight", "scratchdepth", "scratchoffset", "scratchdistance", "scratchhsize", "scratchvsize", "scratchxoffset", "scratchyoffset", "scratchhoffset", "scratchvoffset", "scratchxposition", "scratchyposition", "scratchtopoffset", "scratchbottomoffset", "scratchleftoffset", "scratchrightoffset", "scratchcounterone", "scratchcountertwo", "scratchcounterthree", "scratchdimenone", "scratchdimentwo", "scratchdimenthree", "scratchskipone", "scratchskiptwo", "scratchskipthree", "scratchmuskipone", "scratchmuskiptwo", "scratchmuskipthree", "scratchtoksone", "scratchtokstwo", "scratchtoksthree", "scratchboxone", "scratchboxtwo", "scratchboxthree", "scratchnx", "scratchny", "scratchmx", "scratchmy", "scratchunicode", "scratchleftskip", "scratchrightskip", "scratchtopskip", "scratchbottomskip", "doif", "doifnot", "doifelse", "doifinset", "doifnotinset", "doifinsetelse", "doifnextcharelse", "doifnextoptionalelse", "doifnextoptionalcselse", "doiffastoptionalcheckelse", "doifnextbgroupelse", "doifnextbgroupcselse", "doifnextparenthesiselse", "doifundefinedelse", "doifdefinedelse", "doifundefined", "doifdefined", "doifelsevalue", "doifvalue", "doifnotvalue", "doifnothing", "doifsomething", "doifelsenothing", "doifsomethingelse", "doifvaluenothing", "doifvaluesomething", "doifelsevaluenothing", "doifdimensionelse", "doifnumberelse", "doifnumber", "doifnotnumber", "doifcommonelse", "doifcommon", "doifnotcommon", "doifinstring", "doifnotinstring", "doifinstringelse", "doifassignmentelse", "docheckassignment", "tracingall", "tracingnone", "loggingall", "removetoks", "appendtoks", "prependtoks", "appendtotoks", "prependtotoks", "to", "endgraf", "endpar", "everyendpar", "reseteverypar", "finishpar", "empty", "null", "space", "quad", "enspace", "obeyspaces", "obeylines", "obeyedspace", "obeyedline", "normalspace", "executeifdefined", "singleexpandafter", "doubleexpandafter", "tripleexpandafter", "dontleavehmode", "removelastspace", "removeunwantedspaces", "keepunwantedspaces", "wait", "writestatus", "define", "defineexpandable", "redefine", "setmeasure", "setemeasure", "setgmeasure", "setxmeasure", "definemeasure", "freezemeasure", "measure", "measured", "installcorenamespace", "getvalue", "getuvalue", "setvalue", "setevalue", "setgvalue", "setxvalue", "letvalue", "letgvalue", "resetvalue", "undefinevalue", "ignorevalue", "setuvalue", "setuevalue", "setugvalue", "setuxvalue", "globallet", "glet", "udef", "ugdef", "uedef", "uxdef", "checked", "unique", "getparameters", "geteparameters", "getgparameters", "getxparameters", "forgetparameters", "copyparameters", "getdummyparameters", "dummyparameter", "directdummyparameter", "setdummyparameter", "letdummyparameter", "usedummystyleandcolor", "usedummystyleparameter", "usedummycolorparameter", "processcommalist", "processcommacommand", "quitcommalist", "quitprevcommalist", "processaction", "processallactions", "processfirstactioninset", "processallactionsinset", "unexpanded", "expanded", "startexpanded", "stopexpanded", "protected", "protect", "unprotect", "firstofoneargument", "firstoftwoarguments", "secondoftwoarguments", "firstofthreearguments", "secondofthreearguments", "thirdofthreearguments", "firstoffourarguments", "secondoffourarguments", "thirdoffourarguments", "fourthoffourarguments", "firstoffivearguments", "secondoffivearguments", "thirdoffivearguments", "fourthoffivearguments", "fifthoffivearguments", "firstofsixarguments", "secondofsixarguments", "thirdofsixarguments", "fourthofsixarguments", "fifthofsixarguments", "sixthofsixarguments", "firstofoneunexpanded", "gobbleoneargument", "gobbletwoarguments", "gobblethreearguments", "gobblefourarguments", "gobblefivearguments", "gobblesixarguments", "gobblesevenarguments", "gobbleeightarguments", "gobbleninearguments", "gobbletenarguments", "gobbleoneoptional", "gobbletwooptionals", "gobblethreeoptionals", "gobblefouroptionals", "gobblefiveoptionals", "dorecurse", "doloop", "exitloop", "dostepwiserecurse", "recurselevel", "recursedepth", "dofastloopcs", "dowith", "newconstant", "setnewconstant", "setconstant", "setconstantvalue", "newconditional", "settrue", "setfalse", "settruevalue", "setfalsevalue", "newmacro", "setnewmacro", "newfraction", "newsignal", "dosingleempty", "dodoubleempty", "dotripleempty", "doquadrupleempty", "doquintupleempty", "dosixtupleempty", "doseventupleempty", "dosingleargument", "dodoubleargument", "dotripleargument", "doquadrupleargument", "doquintupleargument", "dosixtupleargument", "doseventupleargument", "dosinglegroupempty", "dodoublegroupempty", "dotriplegroupempty", "doquadruplegroupempty", "doquintuplegroupempty", "permitspacesbetweengroups", "dontpermitspacesbetweengroups", "nopdfcompression", "maximumpdfcompression", "normalpdfcompression", "modulonumber", "dividenumber", "getfirstcharacter", "doiffirstcharelse", "startnointerference", "stopnointerference", "twodigits", "threedigits", "leftorright", "strut", "setstrut", "strutbox", "strutht", "strutdp", "strutwd", "struthtdp", "begstrut", "endstrut", "lineheight", "ordordspacing", "ordopspacing", "ordbinspacing", "ordrelspacing", "ordopenspacing", "ordclosespacing", "ordpunctspacing", "ordinnerspacing", "opordspacing", "opopspacing", "opbinspacing", "oprelspacing", "opopenspacing", "opclosespacing", "oppunctspacing", "opinnerspacing", "binordspacing", "binopspacing", "binbinspacing", "binrelspacing", "binopenspacing", "binclosespacing", "binpunctspacing", "bininnerspacing", "relordspacing", "relopspacing", "relbinspacing", "relrelspacing", "relopenspacing", "relclosespacing", "relpunctspacing", "relinnerspacing", "openordspacing", "openopspacing", "openbinspacing", "openrelspacing", "openopenspacing", "openclosespacing", "openpunctspacing", "openinnerspacing", "closeordspacing", "closeopspacing", "closebinspacing", "closerelspacing", "closeopenspacing", "closeclosespacing", "closepunctspacing", "closeinnerspacing", "punctordspacing", "punctopspacing", "punctbinspacing", "punctrelspacing", "punctopenspacing", "punctclosespacing", "punctpunctspacing", "punctinnerspacing", "innerordspacing", "inneropspacing", "innerbinspacing", "innerrelspacing", "inneropenspacing", "innerclosespacing", "innerpunctspacing", "innerinnerspacing", "normalreqno", "startimath", "stopimath", "normalstartimath", "normalstopimath", "startdmath", "stopdmath", "normalstartdmath", "normalstopdmath", "uncramped", "cramped", "triggermathstyle", "mathstylefont", "mathsmallstylefont", "mathstyleface", "mathsmallstyleface", "mathstylecommand", "mathpalette", "mathstylehbox", "mathstylevbox", "mathstylevcenter", "mathstylevcenteredhbox", "mathstylevcenteredvbox", "mathtext", "setmathsmalltextbox", "setmathtextbox", "triggerdisplaystyle", "triggertextstyle", "triggerscriptstyle", "triggerscriptscriptstyle", "triggeruncrampedstyle", "triggercrampedstyle", "triggersmallstyle", "triggeruncrampedsmallstyle", "triggercrampedsmallstyle", "triggerbigstyle", "triggeruncrampedbigstyle", "triggercrampedbigstyle", "luaexpr", "expdoifelse", "expdoif", "expdoifnot", "expdoifcommonelse", "expdoifinsetelse", "ctxdirectlua", "ctxlatelua", "ctxsprint", "ctxwrite", "ctxcommand", "ctxdirectcommand", "ctxlatecommand", "ctxreport", "ctxlua", "luacode", "lateluacode", "directluacode", "registerctxluafile", "ctxloadluafile", "luaversion", "luamajorversion", "luaminorversion", "ctxluacode", "luaconditional", "luaexpanded", "startluaparameterset", "stopluaparameterset", "luaparameterset", "definenamedlua", "obeylualines", "obeyluatokens", "startluacode", "stopluacode", "startlua", "stoplua", "startctxfunction", "stopctxfunction", "ctxfunction", "startctxfunctiondefinition", "stopctxfunctiondefinition", "carryoverpar", "assumelongusagecs", "Umathbotaccent", "righttolefthbox", "lefttorighthbox", "righttoleftvbox", "lefttorightvbox", "righttoleftvtop", "lefttorightvtop", "rtlhbox", "ltrhbox", "rtlvbox", "ltrvbox", "rtlvtop", "ltrvtop", "autodirhbox", "autodirvbox", "autodirvtop", "lefttoright", "righttoleft", "synchronizelayoutdirection", "synchronizedisplaydirection", "synchronizeinlinedirection", "lesshyphens", "morehyphens", "nohyphens", "dohyphens", "Ucheckedstartdisplaymath", "Ucheckedstopdisplaymath" }, }
\ No newline at end of file diff --git a/context/data/scite/lexers/lexer.lua b/context/data/scite/lexers/lexer.lua new file mode 100644 index 000000000..9582f6a76 --- /dev/null +++ b/context/data/scite/lexers/lexer.lua @@ -0,0 +1,3 @@ +-- this works ok: + +return require("scite-context-lexer") diff --git a/context/data/scite/lexers/scite-context-lexer-cld.lua b/context/data/scite/lexers/scite-context-lexer-cld.lua index 1e30c18a2..9b07b5b80 100644 --- a/context/data/scite/lexers/scite-context-lexer-cld.lua +++ b/context/data/scite/lexers/scite-context-lexer-cld.lua @@ -6,13 +6,14 @@ local info = { license = "see context related readme files", } -local lexer = lexer +local lexer = require("lexer") +local context = lexer.context +local patterns = context.patterns -local cldlexer = { _NAME = "cld", _FILENAME = "scite-context-lexer-cld" } -local whitespace = lexer.WHITESPACE -- maybe we need to fix this -local context = lexer.context +local cldlexer = lexer.new("cld","scite-context-lexer-cld") +local lualexer = lexer.load('scite-context-lexer-lua') -local lualexer = lexer.load('scite-context-lexer-lua') +-- can probably be done nicer now, a bit of a hack cldlexer._rules = lualexer._rules_cld cldlexer._tokenstyles = lualexer._tokenstyles diff --git a/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua b/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua index fdec301be..7e1a3dcc3 100644 --- a/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua +++ b/context/data/scite/lexers/scite-context-lexer-lua-longstring.lua @@ -6,15 +6,18 @@ local info = { license = "see context related readme files", } -local lexer = lexer -local token = lexer.token local P = lpeg.P -local stringlexer = { _NAME = "lua-longstring", _FILENAME = "scite-context-lexer-lua-longstring" } -local whitespace = lexer.WHITESPACE +local lexer = require("lexer") -- require("scite-context-lexer") local context = lexer.context +local patterns = context.patterns -local space = lexer.space +local token = lexer.token + +local stringlexer = lexer.new("lua-longstring","scite-context-lexer-lua-longstring") +local whitespace = stringlexer.whitespace + +local space = patterns.space local nospace = 1 - space local p_spaces = token(whitespace, space ^1) @@ -25,6 +28,6 @@ stringlexer._rules = { { "string", p_string }, } -stringlexer._tokenstyles = lexer.context.styleset +stringlexer._tokenstyles = context.styleset return stringlexer diff --git a/context/data/scite/lexers/scite-context-lexer-lua.lua b/context/data/scite/lexers/scite-context-lexer-lua.lua index 54c480ab3..ebb69c979 100644 --- a/context/data/scite/lexers/scite-context-lexer-lua.lua +++ b/context/data/scite/lexers/scite-context-lexer-lua.lua @@ -6,26 +6,27 @@ local info = { license = "see context related readme files", } --- todo: _G.print (keep _G colored) - -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end +-- beware: all multiline is messy, so even if it's no lexer, it should be an embedded lexer +-- we probably could use a local whitespace variant but this is cleaner -local lexer = lexer -local token, style, colors, exact_match, just_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.just_match, lexer.style_nothing local P, R, S, C, Cg, Cb, Cs, Cmt, Cp = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt, lpeg.Cp local match, find = string.match, string.find local setmetatable = setmetatable --- beware: all multiline is messy, so even if it's no lexer, it should be an embedded lexer --- we probably could use a local whitespace variant but this is cleaner - -local lualexer = { _NAME = "lua", _FILENAME = "scite-context-lexer-lua" } -local whitespace = lexer.WHITESPACE +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns + +local token = lexer.token +local exact_match = lexer.exact_match +local just_match = lexer.just_match + +local lualexer = lexer.new("lua","scite-context-lexer-lua") +local whitespace = lualexer.whitespace local stringlexer = lexer.load("scite-context-lexer-lua-longstring") -local directives = { } -- communication channel +local directives = { } -- communication channel -- this will be extended @@ -124,8 +125,9 @@ local longcomment = Cmt(#('[[' + ('[' * C(equals) * '[')), function(input,index, return stop and stop + 1 or #input + 1 end) -local space = lexer.space -- S(" \n\r\t\f\v") -local any = lexer.any +local space = patterns.space -- S(" \n\r\t\f\v") +local any = patterns.any +local eol = patterns.eol local squote = P("'") local dquote = P('"') @@ -135,7 +137,7 @@ local dashes = P('--') local spacing = token(whitespace, space^1) local rest = token("default", any) -local shortcomment = token("comment", dashes * lexer.nonnewline^0) +local shortcomment = token("comment", dashes * (1-eol)^0) local longcomment = token("comment", dashes * longcomment) -- fails on very long string with \ at end of lines (needs embedded lexer) @@ -160,8 +162,8 @@ local string = shortstring lexer.embed_lexer(lualexer, stringlexer, token("quote",longtwostart), token("string",longtwostring_body) * token("quote",longtwostring_end)) -local integer = P("-")^-1 * (lexer.hex_num + lexer.dec_num) -local number = token("number", lexer.float + integer) +local integer = P("-")^-1 * (patterns.hexadecimal + patterns.decimal) +local number = token("number", patterns.float + integer) -- officially 127-255 are ok but not utf so useless @@ -337,7 +339,7 @@ local texstring = token("quote", longthreestart) * token("string", longthreestring) * token("quote", longthreestop) --- local texcommand = token("user", texcsname) +----- texcommand = token("user", texcsname) local texcommand = token("warning", texcsname) -- local texstring = token("quote", longthreestart) diff --git a/context/data/scite/lexers/scite-context-lexer-mps.lua b/context/data/scite/lexers/scite-context-lexer-mps.lua index f0d88eb3b..dc61786dc 100644 --- a/context/data/scite/lexers/scite-context-lexer-mps.lua +++ b/context/data/scite/lexers/scite-context-lexer-mps.lua @@ -6,17 +6,19 @@ local info = { license = "see context related readme files", } -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local lexer = lexer local global, string, table, lpeg = _G, string, table, lpeg -local token, exact_match = lexer.token, lexer.exact_match local P, R, S, V, C, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt local type = type -local metafunlexer = { _NAME = "mps", _FILENAME = "scite-context-lexer-mps" } -local whitespace = lexer.WHITESPACE +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns + +local token = lexer.token +local exact_match = lexer.exact_match + +local metafunlexer = lexer.new("mps","scite-context-lexer-mps") +local whitespace = metafunlexer.whitespace local metapostprimitives = { } local metapostinternals = { } @@ -65,15 +67,15 @@ do end -local space = lexer.space -- S(" \n\r\t\f\v") -local any = lexer.any +local space = patterns.space -- S(" \n\r\t\f\v") +local any = patterns.any local dquote = P('"') -local cstoken = R("az","AZ") + P("_") -local mptoken = R("az","AZ") +local cstoken = patterns.idtoken +local mptoken = patterns.alpha local leftbrace = P("{") local rightbrace = P("}") -local number = context.patterns.real +local number = patterns.real local cstokentex = R("az","AZ","\127\255") + S("@!?_") @@ -129,15 +131,21 @@ metafunlexer._rules = { metafunlexer._tokenstyles = context.styleset -metafunlexer._foldpattern = R("az")^2 -- separate entry else interference +metafunlexer._foldpattern = patterns.lower^2 -- separate entry else interference metafunlexer._foldsymbols = { _patterns = { '[a-z][a-z]+', }, - ["primitive"] = { + ["plain"] = { ["beginfig"] = 1, ["endfig"] = -1, + ["beginglyph"] = 1, + ["endglyph"] = -1, + -- ["begingraph"] = 1, + -- ["endgraph"] = -1, + }, + ["primitive"] = { ["def"] = 1, ["vardef"] = 1, ["primarydef"] = 1, @@ -152,4 +160,6 @@ metafunlexer._foldsymbols = { } } +-- if inspect then inspect(metafunlexer) end + return metafunlexer diff --git a/context/data/scite/lexers/scite-context-lexer-pdf-object.lua b/context/data/scite/lexers/scite-context-lexer-pdf-object.lua index 6d0b6d8da..dffd492ae 100644 --- a/context/data/scite/lexers/scite-context-lexer-pdf-object.lua +++ b/context/data/scite/lexers/scite-context-lexer-pdf-object.lua @@ -6,19 +6,21 @@ local info = { license = "see context related readme files", } -local lexer = lexer -local token = lexer.token local P, R, S, C, V = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.V -local pdfobjectlexer = { _NAME = "pdf-object", _FILENAME = "scite-context-lexer-pdf-object" } -local whitespace = lexer.WHITESPACE -- triggers states +local lexer = require("lexer") local context = lexer.context local patterns = context.patterns -local space = lexer.space +local token = lexer.token + +local pdfobjectlexer = lexer.new("pdf-object","scite-context-lexer-pdf-object") +local whitespace = pdfobjectlexer.whitespace + +local space = patterns.space local somespace = space^1 -local newline = S("\n\r") +local newline = paterns.eol local real = patterns.real local cardinal = patterns.cardinal diff --git a/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua b/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua index f205e9130..139f06299 100644 --- a/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua +++ b/context/data/scite/lexers/scite-context-lexer-pdf-xref.lua @@ -6,20 +6,22 @@ local info = { license = "see context related readme files", } -local lexer = lexer -local token = lexer.token -local P, R = lpeg.P, lpeg.R - -- xref -- cardinal cardinal [character] -- .. -- %%EOF | startxref | trailer -local pdfxreflexer = { _NAME = "pdf-xref", _FILENAME = "scite-context-lexer-pdf-xref" } -local whitespace = lexer.WHITESPACE -- triggers states +local P, R = lpeg.P, lpeg.R + +local lexer = require("lexer") local context = lexer.context local patterns = context.patterns +local token = lexer.token + +local pdfxreflexer = lexer.new("pdf-xref","scite-context-lexer-pdf-xref") +local whitespace = pdfxreflexer.whitespace + local pdfobjectlexer = lexer.load("scite-context-lexer-pdf-object") local spacing = patterns.spacing @@ -36,10 +38,10 @@ local t_number = token("number", R("09")^1) local t_xref = t_number^1 --- local t_xref = token("default", (1-p_trailer)^1) --- * token("keyword", p_trailer) --- * t_spacing --- * pdfobjectlexer._shared.dictionary +-- t_xref = token("default", (1-p_trailer)^1) +-- * token("keyword", p_trailer) +-- * t_spacing +-- * pdfobjectlexer._shared.dictionary pdfxreflexer._rules = { { 'whitespace', t_spacing }, diff --git a/context/data/scite/lexers/scite-context-lexer-pdf.lua b/context/data/scite/lexers/scite-context-lexer-pdf.lua index 685fdb16e..91abff781 100644 --- a/context/data/scite/lexers/scite-context-lexer-pdf.lua +++ b/context/data/scite/lexers/scite-context-lexer-pdf.lua @@ -6,21 +6,20 @@ local info = { license = "see context related readme files", } -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local lexer = lexer -local token = lexer.token local P, R, S = lpeg.P, lpeg.R, lpeg.S -local pdflexer = { _NAME = "pdf", _FILENAME = "scite-context-lexer-pdf" } -local whitespace = lexer.WHITESPACE -- triggers states +local lexer = require("lexer") +local context = lexer.context +local patterns = context.patterns + +local token = lexer.token + +local pdflexer = lexer.new("pdf","scite-context-lexer-pdf") +local whitespace = pdflexer.whitespace local pdfobjectlexer = lexer.load("scite-context-lexer-pdf-object") local pdfxreflexer = lexer.load("scite-context-lexer-pdf-xref") -local context = lexer.context -local patterns = context.patterns - local space = patterns.space local spacing = patterns.spacing local nospacing = patterns.nospacing diff --git a/context/data/scite/lexers/scite-context-lexer-tex.lua b/context/data/scite/lexers/scite-context-lexer-tex.lua index 55072cde8..bbe0f016c 100644 --- a/context/data/scite/lexers/scite-context-lexer-tex.lua +++ b/context/data/scite/lexers/scite-context-lexer-tex.lua @@ -33,23 +33,24 @@ local info = { ]]-- -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local lexer = lexer local global, string, table, lpeg = _G, string, table, lpeg -local token, exact_match = lexer.token, lexer.exact_match local P, R, S, V, C, Cmt, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt, lpeg.Cp, lpeg.Cc, lpeg.Ct local type, next = type, next local find, match, lower, upper = string.find, string.match, string.lower, string.upper +local lexer = require("lexer") +local context = lexer.context +local patterns = context.patterns + +local token = lexer.token +local exact_match = lexer.exact_match + -- module(...) -local contextlexer = { _NAME = "tex", _FILENAME = "scite-context-lexer-tex" } -local whitespace = lexer.WHITESPACE -local context = lexer.context +local contextlexer = lexer.new("tex","scite-context-lexer-tex") +local whitespace = contextlexer.whitespace local cldlexer = lexer.load('scite-context-lexer-cld') ------ cldlexer = lexer.load('scite-context-lexer-lua') local mpslexer = lexer.load('scite-context-lexer-mps') local commands = { en = { } } @@ -205,8 +206,8 @@ end) local commentline = P('%') * (1-S("\n\r"))^0 local endline = S("\n\r")^1 -local space = lexer.space -- S(" \n\r\t\f\v") -local any = lexer.any +local space = patterns.space -- S(" \n\r\t\f\v") +local any = patterns.any local backslash = P("\\") local hspace = S(" \t") @@ -436,7 +437,7 @@ lexer.embed_lexer(contextlexer, cldlexer, startluacode, stopluacode) lexer.embed_lexer(contextlexer, mpslexer, startmetafuncode, stopmetafuncode) -- Watch the text grabber, after all, we're talking mostly of text (beware, --- no punctuation here as it can be special. We might go for utf here. +-- no punctuation here as it can be special). We might go for utf here. contextlexer._rules = { { "whitespace", spacing }, @@ -462,10 +463,6 @@ contextlexer._rules = { } contextlexer._tokenstyles = context.styleset --- contextlexer._tokenstyles = context.stylesetcopy() -- experiment - --- contextlexer._tokenstyles[#contextlexer._tokenstyles + 1] = { cldlexer._NAME..'_whitespace', lexer.style_whitespace } --- contextlexer._tokenstyles[#contextlexer._tokenstyles + 1] = { mpslexer._NAME..'_whitespace', lexer.style_whitespace } local environment = { ["\\start"] = 1, ["\\stop"] = -1, diff --git a/context/data/scite/lexers/scite-context-lexer-txt.lua b/context/data/scite/lexers/scite-context-lexer-txt.lua index fe062fb94..86570ae64 100644 --- a/context/data/scite/lexers/scite-context-lexer-txt.lua +++ b/context/data/scite/lexers/scite-context-lexer-txt.lua @@ -6,22 +6,23 @@ local info = { license = "see context related readme files", } -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local lexer = lexer -local token = lexer.token local P, S, Cmt, Cp, Ct = lpeg.P, lpeg.S, lpeg.Cmt, lpeg.Cp, lpeg.Ct local find, match = string.find, string.match -local textlexer = { _NAME = "txt", _FILENAME = "scite-context-lexer-txt" } -local whitespace = lexer.WHITESPACE +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns + +local token = lexer.token + +local textlexer = lexer.new("txt","scite-context-lexer-txt") +local whitespace = textlexer.whitespace -local space = lexer.space -local any = lexer.any +local space = patterns.space +local any = patterns.any +local wordtoken = patterns.wordtoken +local wordpattern = patterns.wordpattern -local wordtoken = context.patterns.wordtoken -local wordpattern = context.patterns.wordpattern local checkedword = context.checkedword local styleofword = context.styleofword local setwordlist = context.setwordlist diff --git a/context/data/scite/lexers/scite-context-lexer-web.lua b/context/data/scite/lexers/scite-context-lexer-web.lua index f59a3205d..05c0ce1f3 100644 --- a/context/data/scite/lexers/scite-context-lexer-web.lua +++ b/context/data/scite/lexers/scite-context-lexer-web.lua @@ -8,16 +8,18 @@ local info = { -- this will be extended -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local lexer = lexer -local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing local P, R, S, C, Cg, Cb, Cs, Cmt, lpegmatch = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpeg.Cb, lpeg.Cs, lpeg.Cmt, lpeg.match local setmetatable = setmetatable -local weblexer = { _NAME = "web", _FILENAME = "scite-context-lexer-web" } -local whitespace = lexer.WHITESPACE +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns + +local token = lexer.token +local exact_match = lexer.exact_match + +local weblexer = lexer.new("web","scite-context-lexer-web") +local whitespace = weblexer.whitespace local keywords = { -- copied from cpp.lua -- c @@ -45,9 +47,8 @@ local macros = { -- copied from cpp.lua 'include', 'line', 'pragma', 'undef', 'using', 'warning' } -local space = lexer.space -- S(" \n\r\t\f\v") -local any = lexer.any -local patterns = context.patterns +local space = patterns.space -- S(" \n\r\t\f\v") +local any = patterns.any local restofline = patterns.restofline local startofline = patterns.startofline @@ -59,6 +60,11 @@ local begincomment = P("/*") local endcomment = P("*/") local percent = P("%") +local hexadecimal = patterns.hexadecimal +local decimal = patterns.decimal +local float = patterns.float +local integer = P("-")^-1 * (hexadecimal + decimal) -- also in patterns ? + local spacing = token(whitespace, space^1) local rest = token("default", any) @@ -73,11 +79,9 @@ local shortstring = token("quote", dquote) -- can be shared * token("string", (escaped + (1-squote))^0) * token("quote", squote) -local integer = P("-")^-1 * (lexer.hex_num + lexer.dec_num) -local number = token("number", lexer.float + integer) +local number = token("number", float + integer) local validword = R("AZ","az","__") * R("AZ","az","__","09")^0 - local identifier = token("default",validword) local operator = token("special", S('+-*/%^!=<>;:{}[]().&|?~')) @@ -99,9 +103,9 @@ local endweb = P("@c") local webcomment = token("comment", #beginweb * startofline * beginweb * (1-endweb)^0 * endweb) -local texlexer = lexer.load('scite-context-lexer-tex') +-- local texlexer = lexer.load('scite-context-lexer-tex') -lexer.embed_lexer(weblexer, texlexer, #beginweb * startofline * token("comment",beginweb), token("comment",endweb)) +-- lexer.embed_lexer(weblexer, texlexer, #beginweb * startofline * token("comment",beginweb), token("comment",endweb)) weblexer._rules = { { 'whitespace', spacing }, @@ -148,7 +152,7 @@ weblexer._foldsymbols = { } -- -- by indentation: --- + weblexer._foldpatterns = nil weblexer._foldsymbols = nil diff --git a/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua b/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua index 97253e140..35e07ef18 100644 --- a/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua +++ b/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua @@ -6,15 +6,18 @@ local info = { license = "see context related readme files", } -local lexer = lexer -local token = lexer.token local P = lpeg.P -local xmlcdatalexer = { _NAME = "xml-cdata", _FILENAME = "scite-context-lexer-xml-cdata" } -local whitespace = lexer.WHITESPACE -- triggers states +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns -local space = lexer.space +local token = lexer.token + +local xmlcdatalexer = lexer.new("xml-cdata","scite-context-lexer-xml-cdata") +local whitespace = xmlcdatalexer.whitespace + +local space = patterns.space local nospace = 1 - space - P("]]>") local p_spaces = token(whitespace, space ^1) diff --git a/context/data/scite/lexers/scite-context-lexer-xml-comment.lua b/context/data/scite/lexers/scite-context-lexer-xml-comment.lua index 104310f94..70805b249 100644 --- a/context/data/scite/lexers/scite-context-lexer-xml-comment.lua +++ b/context/data/scite/lexers/scite-context-lexer-xml-comment.lua @@ -6,19 +6,22 @@ local info = { license = "see context related readme files", } -local lexer = lexer -local token = lexer.token local P = lpeg.P -local xmlcommentlexer = { _NAME = "xml-comment", _FILENAME = "scite-context-lexer-xml-comment" } -local whitespace = lexer.WHITESPACE +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns -local space = lexer.space -local nospace = 1 - space - P("-->") +local token = lexer.token -local p_spaces = token(whitespace, space ^1) -local p_comment = token("comment", nospace^1) +local xmlcommentlexer = lexer.new("xml-comment","scite-context-lexer-xml-comment") +local whitespace = xmlcommentlexer.whitespace + +local space = patterns.space +local nospace = 1 - space - P("-->") + +local p_spaces = token(whitespace, space ^1) +local p_comment = token("comment", nospace^1) xmlcommentlexer._rules = { { "whitespace", p_spaces }, diff --git a/context/data/scite/lexers/scite-context-lexer-xml-script.lua b/context/data/scite/lexers/scite-context-lexer-xml-script.lua index fd1aae7f7..13f4cddba 100644 --- a/context/data/scite/lexers/scite-context-lexer-xml-script.lua +++ b/context/data/scite/lexers/scite-context-lexer-xml-script.lua @@ -6,19 +6,22 @@ local info = { license = "see context related readme files", } -local lexer = lexer -local token = lexer.token local P = lpeg.P -local xmlscriptlexer = { _NAME = "xml-script", _FILENAME = "scite-context-lexer-xml-script" } -local whitespace = lexer.WHITESPACE -- triggers states -local context = lexer.context +local lexer = require("lexer") +local context = lexer.context +local patterns = context.patterns -local space = lexer.space -local nospace = 1 - space - (P("</") * P("script") + P("SCRIPT")) * P(">") +local token = lexer.token -local p_spaces = token(whitespace, space ^1) -local p_cdata = token("default", nospace^1) +local xmlscriptlexer = lexer.new("xml-script","scite-context-lexer-xml-script") +local whitespace = xmlscriptlexer.whitespace + +local space = patterns.space +local nospace = 1 - space - (P("</") * P("script") + P("SCRIPT")) * P(">") + +local p_spaces = token(whitespace, space ^1) +local p_cdata = token("default", nospace^1) xmlscriptlexer._rules = { { "whitespace", p_spaces }, diff --git a/context/data/scite/lexers/scite-context-lexer-xml.lua b/context/data/scite/lexers/scite-context-lexer-xml.lua index 241e22591..fa15bbafd 100644 --- a/context/data/scite/lexers/scite-context-lexer-xml.lua +++ b/context/data/scite/lexers/scite-context-lexer-xml.lua @@ -12,26 +12,28 @@ local info = { -- todo: parse entities in attributes -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local lexer = lexer local global, string, table, lpeg = _G, string, table, lpeg -local token, exact_match = lexer.token, lexer.exact_match local P, R, S, V, C, Cmt, Ct, Cp = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt, lpeg.Ct, lpeg.Cp local type = type local match, find = string.match, string.find -local xmllexer = { _NAME = "xml", _FILENAME = "scite-context-lexer-xml" } -local whitespace = lexer.WHITESPACE -- triggers states +local lexer = require("lexer") local context = lexer.context +local patterns = context.patterns + +local token = lexer.token +local exact_match = lexer.exact_match + +local xmllexer = lexer.new("xml","scite-context-lexer-xml") +local whitespace = xmllexer.whitespace local xmlcommentlexer = lexer.load("scite-context-lexer-xml-comment") -- indirect (some issue with the lexer framework) local xmlcdatalexer = lexer.load("scite-context-lexer-xml-cdata") -- indirect (some issue with the lexer framework) local xmlscriptlexer = lexer.load("scite-context-lexer-xml-script") -- indirect (some issue with the lexer framework) local lualexer = lexer.load("scite-context-lexer-lua") -- -local space = lexer.space -- S(" \t\n\r\v\f") -local any = lexer.any -- P(1) +local space = patterns.space +local any = patterns.any local dquote = P('"') local squote = P("'") diff --git a/context/data/scite/lexers/scite-context-lexer.lua b/context/data/scite/lexers/scite-context-lexer.lua index 7c4f7b077..be130077b 100644 --- a/context/data/scite/lexers/scite-context-lexer.lua +++ b/context/data/scite/lexers/scite-context-lexer.lua @@ -1,5 +1,5 @@ local info = { - version = 1.324, + version = 1.400, comment = "basics for scintilla lpeg lexer for context/metafun", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", @@ -8,168 +8,380 @@ local info = { } --- todo: move all code here --- todo: explore adapted dll ... properties + init --- todo: play with hotspot and other properties - --- wish: replace errorlist lexer (per language!) --- wish: access to all scite properties +local trace = true -- false --- The fold and lex functions are copied and patched from original code by Mitchell (see --- lexer.lua). All errors are mine. The ability to use lpeg is a real nice adition and a --- brilliant move. The code is a byproduct of the (mainly Lua based) textadept (still a --- rapidly moving target) that unfortunately misses a realtime output pane. On the other --- hand, SciTE is somewhat crippled by the fact that we cannot pop in our own (language --- dependent) lexer into the output pane (somehow the errorlist lexer is hard coded into --- the editor). Hopefully that will change some day. +-- GET GOING +-- +-- You need to copy this file over lexer.lua. In principle other lexers could +-- work too but not now. Maybe some day. All patterns will move into the patterns +-- name space. I might do the same with styles. If you run an older version of +-- SciTE you can take one of the archives. Pre 3.41 versions can just be copied +-- to the right path, as there we still use part of the normal lexer. +-- +-- REMARK -- --- Starting with SciTE version 3.20 there is an issue with coloring. As we still lack --- a connection with scite itself (properties as well as printing to the log pane) we --- cannot trace this (on windows). As far as I can see, there are no fundamental --- changes in lexer.lua or LexLPeg.cxx so it must be in scintilla itself. So for the --- moment I stick to 3.10. Indicators are: no lexing of 'next' and 'goto <label>' in the --- Lua lexer and no brace highlighting either. Interesting is that it does work ok in --- the cld lexer (so the Lua code is okay). Also the fact that char-def.lua lexes fast --- is a signal that the lexer quits somewhere halfway. +-- We started using lpeg lexing as soon as it came available. Because we had +-- rather demanding files an dalso wanted to use nested lexers, we ended up with +-- our own variant (more robust and faster). As a consequence successive versions +-- had to be adapted to changes in the (still unstable) api. In addition to +-- lexing we also have spell checking and such. +-- +-- STATUS +-- +-- todo: maybe use a special stripped version of the dll (stable api) +-- todo: play with hotspot and other properties +-- wish: access to all scite properties and in fact integrate in scite +-- todo: add proper tracing and so .. not too hard as we can run on mtxrun +-- todo: get rid of these lexers.STYLE_XX and lexers.XX (hide such details) -- --- After checking 3.24 and adapting to the new lexer tables things are okay again. So, --- this version assumes 3.24 or higher. In 3.24 we have a different token result, i.e. no --- longer a { tag, pattern } but just two return values. I didn't check other changes but --- will do that when I run into issues. I had optimized these small tables by hashing which --- was more efficient but this is no longer needed. +-- HISTORY -- --- In 3.3.1 another major change took place: some helper constants (maybe they're no --- longer constants) and functions were moved into the lexer modules namespace but the --- functions are assigned to the Lua module afterward so we cannot alias them beforehand. --- We're probably getting close to a stable interface now. +-- The fold and lex functions are copied and patched from original code by Mitchell +-- (see lexer.lua). All errors are mine. The ability to use lpeg is a real nice +-- adition and a brilliant move. The code is a byproduct of the (mainly Lua based) +-- textadept (still a rapidly moving target) that unfortunately misses a realtime +-- output pane. On the other hand, SciTE is somewhat crippled by the fact that we +-- cannot pop in our own (language dependent) lexer into the output pane (somehow +-- the errorlist lexer is hard coded into the editor). Hopefully that will change +-- some day. -- --- I've considered making a whole copy and patch the other functions too as we need --- an extra nesting model. However, I don't want to maintain too much. An unfortunate --- change in 3.03 is that no longer a script can be specified. This means that instead --- of loading the extensions via the properties file, we now need to load them in our --- own lexers, unless of course we replace lexer.lua completely (which adds another --- installation issue). +-- Starting with SciTE version 3.20 there is an issue with coloring. As we still +-- lack a connection with SciTE itself (properties as well as printing to the log +-- pane) and we cannot trace this (on windows). As far as I can see, there are no +-- fundamental changes in lexer.lua or LexLPeg.cxx so it must be in Scintilla +-- itself. So for the moment I stick to 3.10. Indicators are: no lexing of 'next' +-- and 'goto <label>' in the Lua lexer and no brace highlighting either. Interesting +-- is that it does work ok in the cld lexer (so the Lua code is okay). Also the fact +-- that char-def.lua lexes fast is a signal that the lexer quits somewhere halfway. +-- Maybe there are some hard coded limitations on the amount of styles and/or length +-- if names. -- --- Another change has been that _LEXERHOME is no longer available. It looks like more and --- more functionality gets dropped so maybe at some point we need to ship our own dll/so --- files. For instance, I'd like to have access to the current filename and other scite --- properties. For instance, we could cache some info with each file, if only we had --- knowledge of what file we're dealing with. +-- After checking 3.24 and adapting to the new lexer tables things are okay again. +-- So, this version assumes 3.24 or higher. In 3.24 we have a different token +-- result, i.e. no longer a { tag, pattern } but just two return values. I didn't +-- check other changes but will do that when I run into issues. I had optimized +-- these small tables by hashing which was more efficient but this is no longer +-- needed. For the moment we keep some of that code around as I don't know what +-- happens in future versions. -- --- For huge files folding can be pretty slow and I do have some large ones that I keep --- open all the time. Loading is normally no ussue, unless one has remembered the status --- and the cursor is at the last line of a 200K line file. Optimizing the fold function --- brought down loading of char-def.lua from 14 sec => 8 sec. Replacing the word_match --- function and optimizing the lex function gained another 2+ seconds. A 6 second load --- is quite ok for me. The changed lexer table structure (no subtables) brings loading --- down to a few seconds. +-- In 3.31 another major change took place: some helper constants (maybe they're no +-- longer constants) and functions were moved into the lexer modules namespace but +-- the functions are assigned to the Lua module afterward so we cannot alias them +-- beforehand. We're probably getting close to a stable interface now. I've +-- considered making a whole copy and patch the other functions too as we need an +-- extra nesting model. However, I don't want to maintain too much. An unfortunate +-- change in 3.03 is that no longer a script can be specified. This means that +-- instead of loading the extensions via the properties file, we now need to load +-- them in our own lexers, unless of course we replace lexer.lua completely (which +-- adds another installation issue). -- --- When the lexer path is copied to the textadept lexer path, and the theme definition to --- theme path (as lexer.lua), the lexer works there as well. When I have time and motive --- I will make a proper setup file to tune the look and feel a bit and associate suffixes --- with the context lexer. The textadept editor has a nice style tracing option but lacks --- the tabs for selecting files that scite has. It also has no integrated run that pipes --- to the log pane (I wonder if it could borrow code from the console2 project). Interesting --- is that the jit version of textadept crashes on lexing large files (and does not feel --- faster either). +-- Another change has been that _LEXERHOME is no longer available. It looks like +-- more and more functionality gets dropped so maybe at some point we need to ship +-- our own dll/so files. For instance, I'd like to have access to the current +-- filename and other scite properties. For instance, we could cache some info with +-- each file, if only we had knowledge of what file we're dealing with. -- --- Function load(lexer_name) starts with _M.WHITESPACE = lexer_name..'_whitespace' which --- means that we need to have it frozen at the moment we load another lexer. Because spacing --- is used to revert to a parent lexer we need to make sure that we load children as late --- as possible in order not to get the wrong whitespace trigger. This took me quite a while --- to figure out (not being that familiar with the internals). The lex and fold functions --- have been optimized. It is a pitty that there is no proper print available. Another thing --- needed is a default style in ourown theme style definition, as otherwise we get wrong --- nested lexers, especially if they are larger than a view. This is the hardest part of +-- For huge files folding can be pretty slow and I do have some large ones that I +-- keep open all the time. Loading is normally no ussue, unless one has remembered +-- the status and the cursor is at the last line of a 200K line file. Optimizing the +-- fold function brought down loading of char-def.lua from 14 sec => 8 sec. +-- Replacing the word_match function and optimizing the lex function gained another +-- 2+ seconds. A 6 second load is quite ok for me. The changed lexer table structure +-- (no subtables) brings loading down to a few seconds. +-- +-- When the lexer path is copied to the textadept lexer path, and the theme +-- definition to theme path (as lexer.lua), the lexer works there as well. When I +-- have time and motive I will make a proper setup file to tune the look and feel a +-- bit and associate suffixes with the context lexer. The textadept editor has a +-- nice style tracing option but lacks the tabs for selecting files that scite has. +-- It also has no integrated run that pipes to the log pane. Interesting is that the +-- jit version of textadept crashes on lexing large files (and does not feel faster +-- either; maybe a side effect of known limitations). +-- +-- Function load(lexer_name) starts with _lexers.WHITESPACE = lexer_name .. +-- '_whitespace' which means that we need to have it frozen at the moment we load +-- another lexer. Because spacing is used to revert to a parent lexer we need to +-- make sure that we load children as late as possible in order not to get the wrong +-- whitespace trigger. This took me quite a while to figure out (not being that +-- familiar with the internals). The lex and fold functions have been optimized. It +-- is a pitty that there is no proper print available. Another thing needed is a +-- default style in our own theme style definition, as otherwise we get wrong nested +-- lexers, especially if they are larger than a view. This is the hardest part of -- getting things right. -- --- Eventually it might be safer to copy the other methods from lexer.lua here as well so --- that we have no dependencies, apart from the c library (for which at some point the api --- will be stable I hope). +-- It's a pitty that there is no scintillua library for the OSX version of scite. +-- Even better would be to have the scintillua library as integral part of scite as +-- that way I could use OSX alongside windows and linux (depending on needs). Also +-- nice would be to have a proper interface to scite then because currently the +-- lexer is rather isolated and the lua version does not provide all standard +-- libraries. It would also be good to have lpeg support in the regular scite lua +-- extension (currently you need to pick it up from someplace else). +-- +-- With 3.41 the interface changed again so it gets time to look into the C++ code +-- and consider compiling and patching myself. Loading is more complicated not as +-- the lexer gets loaded automatically so we have little control over extending the +-- code now. After a few days trying all kind of solutions I decided to follow a +-- different approach: drop in a complete replacement. This of course means that I +-- need to keep track of even more changes (which for sure will happen) but at least +-- I get rid of interferences. The api (lexing and configuration) is simply too +-- unstable across versions. Maybe in a few years things have stabelized. (Or maybe +-- it's not really expected that one writes lexers at all.) A side effect is that I +-- now no longer will use shipped lexers but just the built-in ones. Not that it +-- matters much as the context lexers cover what I need (and I can always write +-- more). -- --- It's a pitty that there is no scintillua library for the OSX version of scite. Even --- better would be to have the scintillua library as integral part of scite as that way I --- could use OSX alongside windows and linux (depending on needs). Also nice would be to --- have a proper interface to scite then because currently the lexer is rather isolated and the --- lua version does not provide all standard libraries. It would also be good to have lpeg --- support in the regular scite lua extension (currently you need to pick it up from someplace --- else). +-- In fact, the transition to 3.41 was triggered by an unfateful update of Ubuntu +-- which left me with an incompatible SciTE and lexer library and updating was not +-- possible due to the lack of 64 bit libraries. We'll see what the future brings. +-- +-- Promissing is that the library now can use another Lua instance so maybe some day +-- it will get properly in SciTE and we can use more clever scripting. -local lpeg = require 'lpeg' +-- TRACING +-- +-- The advantage is that we now can check more easily with regular Lua. We can also +-- use wine and print to the console (somehow stdout is intercepted there.) So, I've +-- added a bit of tracing. Interesting is to notice that each document gets its own +-- instance which is pretty inefficient when we are spellchecking (In the past I +-- assumed a shared instance and took some precautions.) + +local lpeg = require("lpeg") -local R, P, S, C, V, Cp, Cs, Ct, Cmt, Cc, Cf, Cg, Carg = lpeg.R, lpeg.P, lpeg.S, lpeg.C, lpeg.V, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Carg -local lpegmatch = lpeg.match -local find, gmatch, match, lower, upper, gsub = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub -local concat = table.concat local global = _G -local type, next, setmetatable, rawset = type, next, setmetatable, rawset - --- less confusing as we also use lexer for the current lexer and local _M = lexer is just ugly - -local lexers = lexer or { } -- + fallback for syntax check - --- ok, let's also move helpers here (todo: all go here) - -local sign = S("+-") -local digit = R("09") -local octdigit = R("07") -local hexdigit = R("09","AF","af") - -lexers.sign = sign -lexers.digit = digit -lexers.octdigit = octdigit -lexers.hexdigit = hexdigit -lexers.xdigit = hexdigit - -lexers.dec_num = digit^1 -lexers.oct_num = P("0") - * octdigit^1 -lexers.hex_num = P("0") * S("xX") - * (hexdigit^0 * '.' * hexdigit^1 + hexdigit^1 * '.' * hexdigit^0 + hexdigit^1) - * (S("pP") * sign^-1 * hexdigit^1)^-1 -lexers.float = sign^-1 - * (digit^0 * '.' * digit^1 + digit^1 * '.' * digit^0 + digit^1) - * S("eE") * sign^-1 * digit^1 - -lexers.dec_int = sign^-1 * lexers.dec_num -lexers.oct_int = sign^-1 * lexers.oct_num -lexers.hex_int = sign^-1 * lexers.hex_num - --- these helpers are set afterwards so we delay their initialization ... there is no need to alias --- each time again and this way we can more easily adapt to updates - -local get_style_at, get_indent_amount, get_property, get_fold_level, FOLD_BASE, FOLD_HEADER, FOLD_BLANK, initialize - -initialize = function() - FOLD_BASE = lexers.FOLD_BASE or SC_FOLDLEVELBASE - FOLD_HEADER = lexers.FOLD_HEADER or SC_FOLDLEVELHEADERFLAG - FOLD_BLANK = lexers.FOLD_BLANK or SC_FOLDLEVELWHITEFLAG - get_style_at = lexers.get_style_at or GetStyleAt - get_indent_amount = lexers.get_indent_amount or GetIndentAmount - get_property = lexers.get_property or GetProperty - get_fold_level = lexers.get_fold_level or GetFoldLevel +local find, gmatch, match, lower, upper, gsub, sub, format = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub, string.sub, string.format +local concat = table.concat +local type, next, setmetatable, rawset, tonumber, tostring = type, next, setmetatable, rawset, tonumber, tostring +local R, P, S, V, C, Cp, Cs, Ct, Cmt, Cc, Cf, Cg, Carg = lpeg.R, lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg, lpeg.Carg +local lpegmatch = lpeg.match + +local nesting = 0 + +local function report(fmt,str,...) + if str then + fmt = format(fmt,str,...) + end + print(format("scite lpeg lexer > %s > %s",nesting == 0 and "-" or nesting,fmt)) +end + +if trace then + report("loading context lexer module (global table: %s)",tostring(global)) +end + +if not package.searchpath then + + -- Unfortunately the io library is only available when we end up + -- in this branch of code. + + if trace then + report("using adapted function 'package.searchpath'") + end + + function package.searchpath(name,path) + local tried = { } + for part in gmatch(path,"[^;]+") do + local filename = gsub(part,"%?",name) + local f = io.open(filename,"r") + if f then + f:close() + return filename + end + tried[#tried + 1] = format("no file '%s'",filename) + end + -- added: local path .. for testing + local f = io.open(filename,"r") + if f then + f:close() + return filename + end + -- + tried[#tried + 1] = format("no file '%s'",filename) + return nil, concat(tried,"\n") + end + +end + +local lexers = { } +local context = { } +lexers.context = context + +local patterns = { } +context.patterns = patterns -- todo: lexers.patterns + +lexers._CONTEXTEXTENSIONS = true +lexers.LEXERPATH = package.path -- no need + +local usedlexers = { } +local parent_lexer = nil + +-- The problem with styles is that there is some nasty interaction with scintilla +-- and each version of lexer dll/so has a different issue. So, from now on we will +-- just add them here. There is also a limit on some 30 styles. Maybe I should +-- hash them in order to reuse. + +local default = { + "nothing", "whitespace", "comment", "string", "number", "keyword", + "identifier", "operator", "error", "preprocessor", "constant", "variable", + "function", "type", "label", "embedded", + -- "regex", "class", + "quote", "special", "extra", "reserved", "okay", "warning", + "command", "internal", "preamble", "grouping", "primitive", "plain", + "user", + -- "invisible", "data", +} + +local predefined = { + 'default', 'linenumber', 'bracelight', 'bracebad', 'controlchar', + 'indentguide', 'calltip' +} + +-- Bah ... ugly ... nicer would be a proper hash .. we now have properties +-- as well as STYLE_* and some connection between them ... why .. ok, we +-- could delay things but who cares. Anyway, at this moment the properties +-- are still unknown. + +local function preparestyles(list) + for i=1,#list do + local k = list[i] + local K = upper(k) + local s = "style." .. k + lexers[K] = k -- is this used + lexers['STYLE_'..K] = '$(' .. k .. ')' + end +end + +preparestyles(default) +preparestyles(predefined) + +-- These helpers are set afterwards so we delay their initialization ... there +-- is no need to alias each time again and this way we can more easily adapt +-- to updates. + +-- These keep changing (values, functions, tables ...) so we nee to check these +-- with each update. Some of them are set in the loader (the require 'lexer' is +-- in fact not a real one as the lexer code is loaded in the dll). It's also not +-- getting more efficient. + +-- FOLD_BASE = lexers.FOLD_BASE or SC_FOLDLEVELBASE +-- FOLD_HEADER = lexers.FOLD_HEADER or SC_FOLDLEVELHEADERFLAG +-- FOLD_BLANK = lexers.FOLD_BLANK or SC_FOLDLEVELWHITEFLAG +-- get_style_at = lexers.get_style_at or GetStyleAt +-- get_indent_amount = lexers.get_indent_amount or GetIndentAmount +-- get_property = lexers.get_property or GetProperty +-- get_fold_level = lexers.get_fold_level or GetFoldLevel + +-- It needs checking: do we have access to all properties now? I'll clean +-- this up anyway as I want a simple clean and stable model. + +local FOLD_BASE = 0 +local FOLD_HEADER = 0 +local FOLD_BLANK = 0 + +local style_at = { } +local indent_amount = { } +local fold_level = { } + +local function initialize() + FOLD_BASE = lexers.FOLD_BASE + FOLD_HEADER = lexers.FOLD_HEADER + FOLD_BLANK = lexers.FOLD_BLANK + -- + style_at = lexers.style_at -- table + indent_amount = lexers.indent_amount -- table + fold_level = lexers.fold_level -- table -- initialize = nil + -- +end + +local function get_property(tag,default) + return lexers.property_int[tag] or lexers.property[tag] or default end --- we create our own extra namespace for extensions and helpers +-- Do we really need this? -lexers.context = lexers.context or { } -local context = lexers.context +lexers.property_expanded = setmetatable({ }, { + __index = function(t, key) + return gsub(lexers.property[key],'[$%%]%b()', function(key) + return t[sub(key,3,-2)] + end) + end, + __newindex = function(t,k,v) + report("properties are read-only, '%s' is not changed",k) + end, +}) -context.patterns = context.patterns or { } -local patterns = context.patterns +-- Style handler. +-- +-- The property table will be set later (after loading) by the library. The +-- styleset is not needed any more as we predefine all styles as defaults +-- anyway (too bug sensitive otherwise). + +local function toproperty(specification) + local serialized = { } + for key, value in next, specification do + if value == true then + serialized[#serialized+1] = key + elseif type(value) == "table" then + serialized[#serialized+1] = key .. ":" .. "#" .. value[1] .. value[2] .. value[3] + else + serialized[#serialized+1] = key .. ":" .. tostring(value) + end + end + return concat(serialized,",") +end -lexers._CONTEXTEXTENSIONS = true +local function tostyles(styles) + local styleset = { } + local property = lexers.property or { } + for k, v in next, styles do + v = toproperty(v) + styleset[k] = v + property["style."..k] = v + end + return styleset +end + +context.toproperty = toproperty +context.tostyles = tostyles + +-- If we had one instance/state of Lua as well as all regular libraries +-- preloaded we could use the context base libraries. So, let's go poor- +-- mans solution now. + +function context.registerstyles(styles) + local styleset = tostyles(styles) + context.styles = styles + context.styleset = styleset + if trace then + local t, n = { }, 0 + for k, v in next, styleset do + t[#t+1] = k + if #k > n then + n = #k + end + end + table.sort(t) + local template = " %-" .. n .. "s : %s" + report("initializing styleset:") + for i=1,#t do + local k = t[i] + report(template,k,styleset[k]) + end + end +end + +-- Some spell checking related stuff. Unfortunately we cannot use a path set +-- by property. local locations = { - -- lexers.context.path, - "data", -- optional data directory - "..", -- regular scite directory + "data", -- optional data directory + "..", -- regular scite directory + "lexers", -- new in 3.41 .. no tracing possible + "lexers/data", -- new in 3.41 .. no tracing possible + "../data", -- new in 3.41 .. no tracing possible } local function collect(name) --- local definitions = loadfile(name .. ".luc") or loadfile(name .. ".lua") local okay, definitions = pcall(function () return require(name) end) if okay then if type(definitions) == "function" then @@ -178,6 +390,7 @@ local function collect(name) if type(definitions) == "table" then return definitions end + else end end @@ -185,9 +398,13 @@ function context.loaddefinitions(name) for i=1,#locations do local data = collect(locations[i] .. "/" .. name) if data then + if trace then + report("definition file '%s' has been loaded",name) + end return data end end + report("unable to load definition file '%s'",name) end function context.word_match(words,word_chars,case_insensitive) @@ -218,39 +435,108 @@ function context.word_match(words,word_chars,case_insensitive) end end -local idtoken = R("az","AZ","\127\255","__") -local digit = R("09") -local sign = S("+-") -local period = P(".") -local space = S(" \n\r\t\f\v") - -patterns.idtoken = idtoken +-- Patterns are grouped in a separate namespace but the regular lexers expect +-- shortcuts to be present in the lexers library. Maybe I'll incorporate some +-- of l-lpeg later. + +do + + local anything = P(1) + local idtoken = R("az","AZ","\127\255","__") + local digit = R("09") + local sign = S("+-") + local period = P(".") + local octdigit = R("07") + local hexdigit = R("09","AF","af") + local lower = R('az') + local upper = R('AZ') + local alpha = upper + lower + local space = S(" \n\r\t\f\v") + local eol = S("\r\n") + local backslash = P("\\") + local decimal = digit^1 + local octal = P("0") + * octdigit^1 + local hexadecimal = P("0") * S("xX") + * (hexdigit^0 * '.' * hexdigit^1 + hexdigit^1 * '.' * hexdigit^0 + hexdigit^1) + * (S("pP") * sign^-1 * hexdigit^1)^-1 -- * + + patterns.idtoken = idtoken + patterns.digit = digit + patterns.sign = sign + patterns.period = period + patterns.octdigit = octdigit + patterns.hexdigit = hexdigit + patterns.ascii = R('\000\127') -- useless + patterns.extend = R('\000\255') -- useless + patterns.control = R('\000\031') + patterns.lower = lower + patterns.upper = upper + patterns.alpha = alpha + patterns.decimal = decimal + patterns.octal = octal + patterns.hexadecimal = hexadecimal + patterns.float = sign^-1 + * (digit^0 * '.' * digit^1 + digit^1 * '.' * digit^0 + digit^1) + * S("eE") * sign^-1 * digit^1 -- * + patterns.cardinal = decimal + + patterns.signeddecimal = sign^-1 * decimal + patterns.signedoctal = sign^-1 * octal + patterns.signedhexadecimal = sign^-1 * hexadecimal + patterns.integer = sign^-1 * (hexadecimal + octal + decimal) + patterns.real = + sign^-1 * ( -- at most one + digit^1 * period * digit^0 -- 10.0 10. + + digit^0 * period * digit^1 -- 0.10 .10 + + digit^1 -- 10 + ) + + patterns.anything = anything + patterns.any = anything + patterns.restofline = (1-eol)^1 + patterns.space = space + patterns.spacing = space^1 + patterns.nospacing = (1-space)^1 + patterns.eol = eol + patterns.newline = P("\r\n") + eol + + local endof = S("\n\r\f") + + patterns.startofline = P(function(input,index) + return (index == 1 or lpegmatch(endof,input,index-1)) and index + end) + + -- These are the expected ones for other lexers. Maybe all in own namespace + -- and provide compatibility layer. + + lexers.any = anything + lexers.ascii = ascii + lexers.extend = extend + lexers.alpha = alpha + lexers.digit = digit + lexers.alnum = alnum + lexers.lower = lower + lexers.upper = upper + lexers.xdigit = hexdigit + lexers.cntrl = control + lexers.graph = R('!~') + lexers.print = R(' ~') + lexers.punct = R('!/', ':@', '[\'', '{~') + lexers.space = space + lexers.newline = S("\r\n\f")^1 + lexers.nonnewline = 1 - lexers.newline + lexers.nonnewline_esc = 1 - (lexers.newline + '\\') + backslash * anything + lexers.dec_num = decimal + lexers.oct_num = octal + lexers.hex_num = hexadecimal + lexers.integer = integer + lexers.float = float + lexers.word = (alpha + '_') * (alpha + digit + '_')^0 -- weird, why digits -patterns.digit = digit -patterns.sign = sign -patterns.period = period - -patterns.cardinal = digit^1 -patterns.integer = sign^-1 * digit^1 - -patterns.real = - sign^-1 * ( -- at most one - digit^1 * period * digit^0 -- 10.0 10. - + digit^0 * period * digit^1 -- 0.10 .10 - + digit^1 -- 10 - ) - -patterns.restofline = (1-S("\n\r"))^1 -patterns.space = space -patterns.spacing = space^1 -patterns.nospacing = (1-space)^1 -patterns.anything = P(1) - -local endof = S("\n\r\f") +end -patterns.startofline = P(function(input,index) - return (index == 1 or lpegmatch(endof,input,index-1)) and index -end) +-- end of patterns function context.exact_match(words,word_chars,case_insensitive) local characters = concat(words) @@ -259,7 +545,7 @@ function context.exact_match(words,word_chars,case_insensitive) word_chars = "" end if type(word_chars) == "string" then - pattern = S(characters) + idtoken + pattern = S(characters) + patterns.idtoken if case_insensitive then pattern = pattern + S(upper(characters)) + S(lower(characters)) end @@ -401,7 +687,7 @@ setmetatable(h_table, { __index = function(t,level) local v = { level, FOLD_HEAD setmetatable(b_table, { __index = function(t,level) local v = { level, FOLD_BLANK } t[level] = v return v end }) setmetatable(n_table, { __index = function(t,level) local v = { level } t[level] = v return v end }) -local newline = P("\r\n") + S("\r\n") +local newline = patterns.newline local p_yes = Cp() * Cs((1-newline)^1) * newline^-1 local p_nop = newline @@ -419,7 +705,7 @@ local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) if fold_pattern then -- if no functions are found then we could have a faster one fold_pattern = Cp() * C(fold_pattern) / function(s,match) - local symbols = fold_symbols[get_style_at(start_pos + s)] + local symbols = fold_symbols[style_at[start_pos + s]] if symbols then local l = symbols[match] if l then @@ -450,7 +736,7 @@ local function fold_by_parsing(text,start_pos,start_line,start_level,lexer) local action_y = function(pos,line) for j = 1, #fold_symbols_patterns do for s, match in gmatch(line,fold_symbols_patterns[j]) do -- '()('..patterns[i]..')' - local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)] + local symbols = fold_symbols[style_at[start_pos + pos + s - 1]] local l = symbols and symbols[match] local t = type(l) if t == 'number' then @@ -503,7 +789,7 @@ end local folds, current_line, prev_level local function action_y() - local current_level = FOLD_BASE + get_indent_amount(current_line) + local current_level = FOLD_BASE + indent_amount[current_line] if current_level > prev_level then -- next level local i = current_line - 1 local f @@ -537,7 +823,7 @@ local function action_n() current_line = current_line + 1 end -local pattern = ( S("\t ")^0 * ( (1-S("\n\r"))^1 / action_y + P(true) / action_n) * newline )^0 +local pattern = ( S("\t ")^0 * ( (1-patterns.eol)^1 / action_y + P(true) / action_n) * newline )^0 local function fold_by_indentation(text,start_pos,start_line,start_level) -- initialize @@ -573,14 +859,13 @@ local threshold_by_parsing = 512 * 1024 -- we don't know the filesize yet local threshold_by_indentation = 512 * 1024 -- we don't know the filesize yet local threshold_by_line = 512 * 1024 -- we don't know the filesize yet -function context.fold(text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go +function context.fold(lexer,text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go if text == '' then return { } end if initialize then initialize() end - local lexer = global._LEXER local fold_by_lexer = lexer._fold local fold_by_symbols = lexer._foldsymbols local filesize = 0 -- we don't know that @@ -604,9 +889,13 @@ function context.fold(text,start_pos,start_line,start_level) -- hm, we had size return { } end +-- function context.fold(lexer,text,start_pos,start_line,start_level) -- hm, we had size thresholds .. where did they go +-- return { } +-- end + -- The following code is mostly unchanged: -local function add_rule(lexer,id,rule) +local function add_rule(lexer,id,rule) -- unchanged if not lexer._RULES then lexer._RULES = { } lexer._RULEORDER = { } @@ -615,31 +904,39 @@ local function add_rule(lexer,id,rule) lexer._RULEORDER[#lexer._RULEORDER + 1] = id end -local function add_style(lexer,token_name,style) - local len = lexer._STYLES.len - if len == 32 then - len = len + 8 +local function add_style(lexer,token_name,style) -- unchanged (well, changed a bit around 3.41) +-- if not lexer._TOKENSTYLES[token_name] then + local num_styles = lexer._numstyles + if num_styles == 32 then + num_styles = num_styles + 8 end - if len >= 128 then - print('Too many styles defined (128 MAX)') + if num_styles >= 255 then + report("there can't be more than %s styles",255) end - lexer._TOKENS[token_name] = len - lexer._STYLES[len] = style - lexer._STYLES.len = len + 1 + lexer._TOKENSTYLES[token_name] = num_styles + lexer._EXTRASTYLES[token_name] = style + lexer._numstyles = num_styles + 1 +-- end end -local function join_tokens(lexer) +-- At some point an 'any' append showed up in the original code ... +-- but I see no need to catch that case ... beter fix the specification. + +local function join_tokens(lexer) -- slightly different from the original (no 'any' append) local patterns = lexer._RULES local order = lexer._RULEORDER - local token_rule = patterns[order[1]] - for i=2,#order do + local token_rule = patterns[order[1]] -- normally whitespace + for i=2, #order do token_rule = token_rule + patterns[order[i]] end + if lexer._TYPE ~= "context" then + token_rule = token_rule + lexers.token(lexers.DEFAULT, patterns.any) + end lexer._TOKENRULE = token_rule return token_rule end -local function add_lexer(grammar, lexer, token_rule) +local function add_lexer(grammar, lexer, token_rule) -- mostly the same as the original local token_rule = join_tokens(lexer) local lexer_name = lexer._NAME local children = lexer._CHILDREN @@ -660,7 +957,7 @@ local function add_lexer(grammar, lexer, token_rule) grammar[lexer_name] = token_rule^0 end -local function build_grammar(lexer, initial_rule) +local function build_grammar(lexer,initial_rule) -- same as the original local children = lexer._CHILDREN if children then local lexer_name = lexer._NAME @@ -676,12 +973,14 @@ local function build_grammar(lexer, initial_rule) end end --- so far. We need these local functions in the next one. +-- So far. We need these local functions in the next one. We have these +-- redefinitions because we memoize the lexers ... it looks like in +-- 3.1.4 something similar now happens with 'lexers'. local lineparsers = { } -function context.lex(text,init_style) - local lexer = global._LEXER +function context.lex(lexer,text,init_style) + -- local lexer = global._LEXER local grammar = lexer._GRAMMAR if initialize then initialize() @@ -731,7 +1030,7 @@ function context.lex(text,init_style) if grammar then lexer._GRAMMAR = grammar else - for style, style_num in next, lexer._TOKENS do + for style, style_num in next, lexer._TOKENSTYLES do if style_num == init_style then -- the name of the lexers is filtered from the whitespace -- specification @@ -756,6 +1055,9 @@ function context.lex(text,init_style) end end + +-- so far + -- todo: keywords: one lookup and multiple matches -- function context.token(name, patt) @@ -768,130 +1070,519 @@ function context.token(name, patt) return patt * Cc(name) * Cp() end +-- The next ones were mostly unchanged (till now), we moved it here when 3.41 +-- became close to impossible to combine with cq. overload and a merge was +-- the only solution. It makes later updates more painful but the update to +-- 3.41 was already a bit of a nightmare anyway. + +-- Loading lexers is rather interwoven with what the dll/so sets and +-- it changes over time. So, we need to keep an eye on changes. One +-- problem that we always faced were the limitations in length of +-- lexer names (as they get app/prepended occasionally to strings with +-- a hard coded limit). So, we always used alternative names and now need +-- to make sure this doesn't clash. As I no longer intend to use shipped +-- lexers I could strip away some of the code in the future, but keeping +-- it as reference makes sense. + +-- I spend quite some time figuring out why 3.41 didn't work or crashed which +-- is hard when no stdout is available and when the io library is absent. In +-- the end of of the problems was in the _NAME setting. We set _NAME +-- to e.g. 'tex' but load from a file with a longer name, which we do +-- as we don't want to clash with existing files, we end up in +-- lexers not being found. + +local function check_properties() + if not lexers.property then + lexers.property = { } + lexers.property_int = setmetatable({ }, { + __index = function(t,k) + return tostring(tonumber(lexers.property[k]) or 0) + end, + __newindex = function(t,k,v) + report("properties are read-only, '%s' is not changed",k) + end, + }) + end +end + +local function check_styles(lexer) + local numstyles = #default + local tokenstyles = { } + for i=1, #default do + tokenstyles[default[i]] = i - 1 + end + for i=1, #predefined do + tokenstyles[predefined[i]] = i + 31 + end + lexer._TOKENSTYLES = tokenstyles + lexer._numstyles = numstyles + lexer._EXTRASTYLES = { } + return lexer +end + +local whitespaces = { } + +local function push_whitespace(name) + table.insert(whitespaces,lexers.WHITESPACE or "whitespace") + lexers.WHITESPACE = name .. "_whitespace" +end + +local function pop_whitespace() + lexers.WHITESPACE = table.remove(whitespaces) or "whitespace" +end + +local function check_whitespace(lexer,name) + if lexer then + lexer.whitespace = (name or lexer.name or lexer._NAME) .. "_whitespace" + end +end + +function context.new(name,filename) + local lexer = { + _TYPE = "context", + -- + _NAME = name, -- used for token building + _FILENAME = filename, -- for diagnostic purposed + -- + name = name, + filename = filename, + whitespace = whitespace, + } + if trace then + report("initializing lexer tagged '%s' from file '%s'",name,filename or name) + end + check_styles(lexer) + check_whitespace(lexer) + return lexer +end + +local function nolexer(name) + local lexer = { + _TYPE = "unset", + _NAME = name, + -- _rules = { }, + } + check_styles(lexer) + check_whitespace(lexer) + return lexer +end + +local function load_lexer(name) + local lexer, okay = nil, false + -- first locate the file (methods have changed over time) + local lexer_file = package.searchpath(name,lexers.LEXERPATH) + if not lexer_file or lexer_file == "" then + report("lexer file '%s' can't be located",name) + else + if trace then + report("loading lexer file '%s'",lexer_file) + end + push_whitespace(name) -- for traditional lexers .. no alt_name yet + okay, lexer = pcall(dofile, lexer_file or '') + pop_whitespace() + if not okay then + report("invalid lexer file '%s'",lexer_file) + elseif trace then + report("lexer file '%s' has been loaded",lexer_file) + end + end + if type(lexer) ~= "table" then + return nolexer(name) + end + if lexer._TYPE ~= "context" then + lexer._TYPE = "native" + check_styles(lexer) + check_whitespace(lexer,name) + end + if not lexer._NAME then + lexer._NAME = name -- so: filename + end + return lexer +end + +-- An optional second argument has been introduced so that one can embed a lexer +-- more than once ... maybe something to look into (as not it's done by remembering +-- the start sequence ... quite okay but maybe suboptimal ... anyway, never change +-- a working solution). + +function context.load(filename) + nesting = nesting + 1 + local lexer = usedlexers[filename] -- we load by filename but the internal name can be short + if lexer then + if trace then + report("reusing lexer '%s'",filename) + end + nesting = nesting - 1 + return lexer + end + if trace then + report("loading lexer '%s'",filename) + end + -- + check_properties() + -- + parent_lexer = nil + -- + lexer = load_lexer(filename) or nolexer(name) + usedlexers[filename] = lexer + -- + if not lexer._rules and not lexer._lexer then + lexer._lexer = parent_lexer + end + -- + if lexer._lexer then + local _l = lexer._lexer + local _r = lexer._rules + local _s = lexer._tokenstyles + if not _l._tokenstyles then + _l._tokenstyles = { } + end + if _r then + local rules = _l._rules + local name = lexer.name + for i=1,#_r do + local rule = _r[i] + rules[#rules + 1] = { + name .. '_' .. rule[1], + rule[2], + } + end + end + if _s then + local tokenstyles = _l._tokenstyles + for token, style in next, _s do + tokenstyles[token] = style + end + end + lexer = l + end + -- + local _r = lexer._rules + if _r then + local _s = lexer._tokenstyles + if _s then + for token, style in next, _s do + add_style(lexer, token, style) + end + end + for i=1,#_r do + local rule = _r[i] + add_rule(lexer, rule[1], rule[2]) + end + build_grammar(lexer) + end + -- + add_style(lexer, lexer.whitespace, lexers.STYLE_WHITESPACE) + -- + local foldsymbols = lexer._foldsymbols + if foldsymbols then + local patterns = foldsymbols._patterns + if patterns then + for i = 1, #patterns do + patterns[i] = '()(' .. patterns[i] .. ')' + end + end + end + -- + lexer.lex = lexers.lex + lexer.fold = lexers.fold + -- + nesting = nesting - 1 + -- + return lexer +end + +function context.embed_lexer(parent, child, start_rule, end_rule) -- mostly the same as the original + local embeddedrules = child._EMBEDDEDRULES + if not embeddedrules then + embeddedrules = { } + child._EMBEDDEDRULES = embeddedrules + end + if not child._RULES then + local rules = child._rules + if not rules then + report("child lexer '%s' has no rules",chile._NAME or "unknown") + rules = { } + child._rules = rules + end + for i=1,#rules do + local rule = rules[i] + add_rule(child, rule[1], rule[2]) + end + end + embeddedrules[parent._NAME] = { + ['start_rule'] = start_rule, + ['token_rule'] = join_tokens(child), + ['end_rule'] = end_rule + } + local children = parent._CHILDREN + if not children then + children = { } + parent._CHILDREN = children + end + children[#children + 1] = child + local tokenstyles = parent._tokenstyles + if not tokenstyles then + tokenstyles = { } + parent._tokenstyles = tokenstyles + end + tokenstyles[child._NAME..'_whitespace'] = lexers.STYLE_WHITESPACE -- check what whitespace + local childstyles = child._tokenstyles + if childstyles then + for token, style in next, childstyles do + tokenstyles[token] = style + end + end + child._lexer = parent + parent_lexer = parent +end + +-- we now move the adapted code to the lexers namespace + +lexers.new = context.new +lexers.load = context.load +lexers.embed_lexer = context.embed_lexer lexers.fold = context.fold lexers.lex = context.lex lexers.token = context.token +lexers.word_match = context.word_match lexers.exact_match = context.exact_match lexers.just_match = context.just_match -- helper .. alas ... the lexer's lua instance is rather crippled .. not even -- math is part of it -local floor = math and math.floor -local char = string.char +do -if not floor then + local floor = math and math.floor + local char = string.char - floor = function(n) - return tonumber(string.format("%d",n)) + if not floor then + + floor = function(n) + return tonumber(format("%d",n)) + end + + math = math or { } + + math.floor = floor + + end + + local function utfchar(n) + if n < 0x80 then + return char(n) + elseif n < 0x800 then + return char( + 0xC0 + floor(n/0x40), + 0x80 + (n % 0x40) + ) + elseif n < 0x10000 then + return char( + 0xE0 + floor(n/0x1000), + 0x80 + (floor(n/0x40) % 0x40), + 0x80 + (n % 0x40) + ) + elseif n < 0x40000 then + return char( + 0xF0 + floor(n/0x40000), + 0x80 + floor(n/0x1000), + 0x80 + (floor(n/0x40) % 0x40), + 0x80 + (n % 0x40) + ) + else + -- return char( + -- 0xF1 + floor(n/0x1000000), + -- 0x80 + floor(n/0x40000), + -- 0x80 + floor(n/0x1000), + -- 0x80 + (floor(n/0x40) % 0x40), + -- 0x80 + (n % 0x40) + -- ) + return "?" + end + end + + context.utfchar = utfchar + + -- a helper from l-lpeg: + + local function make(t) + local p + for k, v in next, t do + if not p then + if next(v) then + p = P(k) * make(v) + else + p = P(k) + end + else + if next(v) then + p = p + P(k) * make(v) + else + p = p + P(k) + end + end + end + return p end - math = math or { } + function lpeg.utfchartabletopattern(list) + local tree = { } + for i=1,#list do + local t = tree + for c in gmatch(list[i],".") do + if not t[c] then + t[c] = { } + end + t = t[c] + end + end + return make(tree) + end - math.floor = floor + patterns.invisibles = lpeg.utfchartabletopattern { + utfchar(0x00A0), -- nbsp + utfchar(0x2000), -- enquad + utfchar(0x2001), -- emquad + utfchar(0x2002), -- enspace + utfchar(0x2003), -- emspace + utfchar(0x2004), -- threeperemspace + utfchar(0x2005), -- fourperemspace + utfchar(0x2006), -- sixperemspace + utfchar(0x2007), -- figurespace + utfchar(0x2008), -- punctuationspace + utfchar(0x2009), -- breakablethinspace + utfchar(0x200A), -- hairspace + utfchar(0x200B), -- zerowidthspace + utfchar(0x202F), -- narrownobreakspace + utfchar(0x205F), -- math thinspace + } + + -- now we can make: + + patterns.iwordtoken = patterns.wordtoken - patterns.invisibles + patterns.iwordpattern = patterns.iwordtoken^3 end -local function utfchar(n) - if n < 0x80 then - return char(n) - elseif n < 0x800 then - return char( - 0xC0 + floor(n/0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x10000 then - return char( - 0xE0 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) - elseif n < 0x40000 then - return char( - 0xF0 + floor(n/0x40000), - 0x80 + floor(n/0x1000), - 0x80 + (floor(n/0x40) % 0x40), - 0x80 + (n % 0x40) - ) +-- The following helpers are not used, partyally replace by other mechanism and +-- when needed I'll first optimize them. I only made them somewhat more readable. + +function lexers.delimited_range(chars, single_line, no_escape, balanced) -- unchanged + local s = sub(chars,1,1) + local e = #chars == 2 and sub(chars,2,2) or s + local range + local b = balanced and s or '' + local n = single_line and '\n' or '' + if no_escape then + local invalid = S(e .. n .. b) + range = patterns.any - invalid + else + local invalid = S(e .. n .. b) + patterns.backslash + range = patterns.any - invalid + patterns.backslash * patterns.any + end + if balanced and s ~= e then + return P { + s * (range + V(1))^0 * e + } else - -- return char( - -- 0xF1 + floor(n/0x1000000), - -- 0x80 + floor(n/0x40000), - -- 0x80 + floor(n/0x1000), - -- 0x80 + (floor(n/0x40) % 0x40), - -- 0x80 + (n % 0x40) - -- ) - return "?" + return s * range^0 * P(e)^-1 end end -context.utfchar = utfchar +function lexers.starts_line(patt) -- unchanged + return P ( function(input, index) + if index == 1 then + return index + end + local char = sub(input,index - 1,index - 1) + if char == '\n' or char == '\r' or char == '\f' then + return index + end + end ) * patt +end --- a helper from l-lpeg: +function lexers.last_char_includes(s) -- unchanged + s = '[' .. gsub(s,'[-%%%[]', '%%%1') .. ']' + return P ( function(input, index) + if index == 1 then + return index + end + local i = index + while match(sub(input,i - 1,i - 1),'[ \t\r\n\f]') do + i = i - 1 + end + if match(sub(input,i - 1,i - 1),s) then + return index + end + end) +end -local gmatch = string.gmatch +function lexers.nested_pair(start_chars, end_chars) -- unchanged + local s = start_chars + local e = P(end_chars)^-1 + return P { + s * (patterns.any - s - end_chars + V(1))^0 * e + } +end -local function make(t) - local p - for k, v in next, t do - if not p then - if next(v) then - p = P(k) * make(v) - else - p = P(k) +local function prev_line_is_comment(prefix, text, pos, line, s) -- unchanged + local start = find(line,'%S') + if start < s and not find(line,prefix,start,true) then + return false + end + local p = pos - 1 + if sub(text,p,p) == '\n' then + p = p - 1 + if sub(text,p,p) == '\r' then + p = p - 1 + end + if sub(text,p,p) ~= '\n' then + while p > 1 and sub(text,p - 1,p - 1) ~= '\n' + do p = p - 1 end - else - if next(v) then - p = p + P(k) * make(v) - else - p = p + P(k) + while find(sub(text,p,p),'^[\t ]$') do + p = p + 1 end + return sub(text,p,p + #prefix - 1) == prefix end end - return p + return false end -function lpeg.utfchartabletopattern(list) - local tree = { } - for i=1,#list do - local t = tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c] = { } - end - t = t[c] - end - end - return make(tree) -end - -patterns.invisibles = lpeg.utfchartabletopattern { - utfchar(0x00A0), -- nbsp - utfchar(0x2000), -- enquad - utfchar(0x2001), -- emquad - utfchar(0x2002), -- enspace - utfchar(0x2003), -- emspace - utfchar(0x2004), -- threeperemspace - utfchar(0x2005), -- fourperemspace - utfchar(0x2006), -- sixperemspace - utfchar(0x2007), -- figurespace - utfchar(0x2008), -- punctuationspace - utfchar(0x2009), -- breakablethinspace - utfchar(0x200A), -- hairspace - utfchar(0x200B), -- zerowidthspace - utfchar(0x202F), -- narrownobreakspace - utfchar(0x205F), -- math thinspace -} - --- now we can make: - -patterns.iwordtoken = patterns.wordtoken - patterns.invisibles -patterns.iwordpattern = patterns.iwordtoken^3 +local function next_line_is_comment(prefix, text, pos, line, s) + local p = find(text,'\n',pos + s) + if p then + p = p + 1 + while find(sub(text,p,p),'^[\t ]$') do + p = p + 1 + end + return sub(text,p,p + #prefix - 1) == prefix + end + return false +end --- require("themes/scite-context-theme") +function lexers.fold_line_comments(prefix) + local property_int = lexers.property_int + return function(text, pos, line, s) + if property_int['fold.line.comments'] == 0 then + return 0 + end + if s > 1 and match(line,'^%s*()') < s then + return 0 + end + local prev_line_comment = prev_line_is_comment(prefix, text, pos, line, s) + local next_line_comment = next_line_is_comment(prefix, text, pos, line, s) + if not prev_line_comment and next_line_comment then + return 1 + end + if prev_line_comment and not next_line_comment then + return -1 + end + return 0 + end +end --- In order to deal with some bug in additional styles (I have no cue what is --- wrong, but additional styles get ignored and clash somehow) I just copy the --- original lexer code ... see original for comments. +-- done return lexers diff --git a/context/data/scite/lexers/themes/scite-context-theme.lua b/context/data/scite/lexers/themes/scite-context-theme.lua index 6e161b22f..9b22b241c 100644 --- a/context/data/scite/lexers/themes/scite-context-theme.lua +++ b/context/data/scite/lexers/themes/scite-context-theme.lua @@ -7,220 +7,112 @@ local info = { } -- context_path = string.split(os.resultof("mtxrun --find-file context.mkiv"))[1] or "" --- global.trace("OEPS") -- how do we get access to the regular lua extensions --- The regular styles set the main lexer styles table but we avoid that in order not --- to end up with updating issues. We just use another table. - -if not lexer._CONTEXTEXTENSIONS then require("scite-context-lexer") end - -local context_path = "t:/sources" -- c:/data/tex-context/tex/texmf-context/tex/base -local font_name = 'Dejavu Sans Mono' -local font_size = 14 - -if not WIN32 then - font_name = '!' .. font_name -end - -local color = lexer.color -local style = lexer.style - -lexer.context = lexer.context or { } -local context = lexer.context +-- What used to be proper Lua definitions are in 3.42 SciTE properties although +-- integration is still somewhat half. Also, the indexed style specification is +-- now a hash (which indeed makes more sense). However, the question is: am I +-- going to rewrite the style bit? It anyway makes more sense to keep this file +-- somewhat neutral as we no longer need to be compatible. However, we cannot be +-- sure of helpers being present yet when this file is loaded, so we are somewhat +-- crippled. On the other hand, I don't see other schemes being used with the +-- context lexers. + +-- The next kludge is no longer needed: +-- +-- if GTK then -- WIN32 GTK OSX CURSES +-- font_name = '!' .. font_name +-- end -context.path = context_path +local font_name = 'Dejavu Sans Mono' +local font_size = '14' local colors = { - red = color('7F', '00', '00'), - green = color('00', '7F', '00'), - blue = color('00', '00', '7F'), - cyan = color('00', '7F', '7F'), - magenta = color('7F', '00', '7F'), - yellow = color('7F', '7F', '00'), - orange = color('B0', '7F', '00'), + red = { '7F', '00', '00' }, + green = { '00', '7F', '00' }, + blue = { '00', '00', '7F' }, + cyan = { '00', '7F', '7F' }, + magenta = { '7F', '00', '7F' }, + yellow = { '7F', '7F', '00' }, + orange = { 'B0', '7F', '00' }, -- - white = color('FF', 'FF', 'FF'), - light = color('CF', 'CF', 'CF'), - grey = color('80', '80', '80'), - dark = color('4F', '4F', '4F'), - black = color('00', '00', '00'), + white = { 'FF', 'FF', 'FF' }, + light = { 'CF', 'CF', 'CF' }, + grey = { '80', '80', '80' }, + dark = { '4F', '4F', '4F' }, + black = { '00', '00', '00' }, -- - selection = color('F7', 'F7', 'F7'), - logpanel = color('E7', 'E7', 'E7'), - textpanel = color('CF', 'CF', 'CF'), - linepanel = color('A7', 'A7', 'A7'), - tippanel = color('44', '44', '44'), + selection = { 'F7', 'F7', 'F7' }, + logpanel = { 'E7', 'E7', 'E7' }, + textpanel = { 'CF', 'CF', 'CF' }, + linepanel = { 'A7', 'A7', 'A7' }, + tippanel = { '44', '44', '44' }, -- - right = color('00', '00', 'FF'), - wrong = color('FF', '00', '00'), + right = { '00', '00', 'FF' }, + wrong = { 'FF', '00', '00' }, } -colors.teal = colors.cyan -colors.purple = colors.magenta - -lexer.colors = colors - --- defaults: - -local style_nothing = style { } ------ style_whitespace = style { } -local style_comment = style { fore = colors.yellow } -local style_string = style { fore = colors.magenta } -local style_number = style { fore = colors.cyan } -local style_keyword = style { fore = colors.blue, bold = true } -local style_identifier = style_nothing -local style_operator = style { fore = colors.blue } -local style_error = style { fore = colors.red } -local style_preproc = style { fore = colors.yellow, bold = true } -local style_constant = style { fore = colors.cyan, bold = true } -local style_variable = style { fore = colors.black } -local style_function = style { fore = colors.black, bold = true } -local style_class = style { fore = colors.black, bold = true } -local style_type = style { fore = colors.blue } -local style_label = style { fore = colors.red, bold = true } -local style_regex = style { fore = colors.magenta } - --- reserved: - -local style_default = style { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel } -local style_text = style { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel } -local style_line_number = style { back = colors.linepanel } -local style_bracelight = style { fore = colors.orange, bold = true } -local style_bracebad = style { fore = colors.orange, bold = true } -local style_indentguide = style { fore = colors.linepanel, back = colors.white } -local style_calltip = style { fore = colors.white, back = colors.tippanel } -local style_controlchar = style_nothing - --- extras: - -local style_quote = style { fore = colors.blue, bold = true } -local style_special = style { fore = colors.blue } -local style_extra = style { fore = colors.yellow } -local style_embedded = style { fore = colors.black, bold = true } ------ style_char = style { fore = colors.magenta } -local style_reserved = style { fore = colors.magenta, bold = true } -local style_definition = style { fore = colors.black, bold = true } -local style_okay = style { fore = colors.dark } -local style_warning = style { fore = colors.orange } -local style_invisible = style { back = colors.orange } -local style_tag = style { fore = colors.cyan } ------ style_standout = style { fore = colors.orange, bold = true } -local style_command = style { fore = colors.green, bold = true } -local style_internal = style { fore = colors.orange, bold = true } - -local style_preamble = style { fore = colors.yellow } -local style_grouping = style { fore = colors.red } -local style_primitive = style { fore = colors.blue, bold = true } -local style_plain = style { fore = colors.dark, bold = true } -local style_user = style { fore = colors.green } -local style_data = style { fore = colors.cyan, bold = true } - - --- used by the generic lexer: - -lexer.style_nothing = style_nothing -- 0 ------.whitespace = style_whitespace -- 1 -lexer.style_comment = style_comment -- 2 -lexer.style_string = style_string -- 3 -lexer.style_number = style_number -- 4 -lexer.style_keyword = style_keyword -- 5 -lexer.style_identifier = style_nothing -- 6 -lexer.style_operator = style_operator -- 7 -lexer.style_error = style_error -- 8 -lexer.style_preproc = style_preproc -- 9 -lexer.style_constant = style_constant -- 10 -lexer.style_variable = style_variable -- 11 -lexer.style_function = style_function -- 12 -lexer.style_class = style_class -- 13 -lexer.style_type = style_type -- 14 -lexer.style_label = style_label -- 15 -lexer.style_regex = style_regexp -- 16 - -lexer.style_default = style_default -- 32 -lexer.style_line_number = style_line_number -- 33 -lexer.style_bracelight = style_bracelight -- 34 -lexer.style_bracebad = style_bracebad -- 35 -lexer.style_indentguide = style_indentguide -- 36 -lexer.style_calltip = style_calltip -- 37 -lexer.style_controlchar = style_controlchar -- 38 - -local styles = { -- as we have globals we could do with less - - -- ["whitespace"] = style_whitespace, -- not to be set! - ["default"] = style_nothing, -- else no good backtracking to start-of-child - -- ["number"] = style_number, - -- ["comment"] = style_comment, - -- ["keyword"] = style_keyword, - -- ["string"] = style_string, - -- ["preproc"] = style_preproc, - -- ["error"] = style_error, - -- ["label"] = style_label, - - ["invisible"] = style_invisible, - ["quote"] = style_quote, - ["special"] = style_special, - ["extra"] = style_extra, - ["embedded"] = style_embedded, - -- ["char"] = style_char, - ["reserved"] = style_reserved, - -- ["definition"] = style_definition, - ["okay"] = style_okay, - ["warning"] = style_warning, - -- ["standout"] = style_standout, - ["command"] = style_command, - ["internal"] = style_internal, - ["preamble"] = style_preamble, - ["grouping"] = style_grouping, - ["primitive"] = style_primitive, - ["plain"] = style_plain, - ["user"] = style_user, - ["data"] = style_data, - - ["text"] = style_text, -- style_default +local styles = { + + ["whitespace"] = { }, + ["default"] = { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel }, + ["number"] = { fore = colors.cyan }, + ["comment"] = { fore = colors.yellow }, + ["keyword"] = { fore = colors.blue, bold = true }, + ["string"] = { fore = colors.magenta }, + -- ["preproc"] = { fore = colors.yellow, bold = true }, + ["error"] = { fore = colors.red }, + ["label"] = { fore = colors.red, bold = true }, + + ["nothing"] = { }, + ["class"] = { fore = colors.black, bold = true }, + ["function"] = { fore = colors.black, bold = true }, + ["constant"] = { fore = colors.cyan, bold = true }, + ["operator"] = { fore = colors.blue }, + ["regex"] = { fore = colors.magenta }, + ["preprocessor"] = { fore = colors.yellow, bold = true }, + ["tag"] = { fore = colors.cyan }, + ["type"] = { fore = colors.blue }, + ["variable"] = { fore = colors.black }, + ["identifier"] = { }, + + ["linenumber"] = { back = colors.linepanel }, + ["bracelight"] = { fore = colors.orange, bold = true }, + ["bracebad"] = { fore = colors.orange, bold = true }, + ["controlchar"] = { }, + ["indentguide"] = { fore = colors.linepanel, back = colors.white }, + ["calltip"] = { fore = colors.white, back = colors.tippanel }, + + ["invisible"] = { back = colors.orange }, + ["quote"] = { fore = colors.blue, bold = true }, + ["special"] = { fore = colors.blue }, + ["extra"] = { fore = colors.yellow }, + ["embedded"] = { fore = colors.black, bold = true }, + ["char"] = { fore = colors.magenta }, + ["reserved"] = { fore = colors.magenta, bold = true }, + ["definition"] = { fore = colors.black, bold = true }, + ["okay"] = { fore = colors.dark }, + ["warning"] = { fore = colors.orange }, + ["standout"] = { fore = colors.orange, bold = true }, + ["command"] = { fore = colors.green, bold = true }, + ["internal"] = { fore = colors.orange, bold = true }, + ["preamble"] = { fore = colors.yellow }, + ["grouping"] = { fore = colors.red }, + ["primitive"] = { fore = colors.blue, bold = true }, + ["plain"] = { fore = colors.dark, bold = true }, + ["user"] = { fore = colors.green }, + ["data"] = { fore = colors.cyan, bold = true }, + + -- equal to default: + + ["text"] = { font = font_name, size = font_size, fore = colors.black, back = colors.textpanel }, } -local styleset = { } - -for k, v in next, styles do - styleset[#styleset+1] = { k, v } -end - -context.styles = styles -context.styleset = styleset - -function context.stylesetcopy() - local t = { } - for i=1,#styleset do - local s = styleset[i] - t[i] = s -t[s[1]] = t[s[2]] -- new style ? - end - t[#t+1] = { "whitespace", style_nothing } -t.whitespace = style_nothing -- new style ? - return t -end - --- We can be sparse if needed: +-- I need to read the C++ code and see how I can load the themes directly +-- in which case we can start thinking of tracing. --- function context.newstyleset(list) --- local t = { } --- if list then --- for i=1,#list do --- t[list[i]] = true --- end --- end --- return t --- end +if lexer and lexer.context and lexer.context.registerstyles then --- function context.usestyle(set,name) --- set[name] = true --- return name --- end + lexer.context.registerstyles(styles) --- function context.usestyleset(set) --- local t = { } --- for k, _ in next, set do --- t[#t+1] = { k, styles[k] or styles.default } --- end --- end +end |