summaryrefslogtreecommitdiff
path: root/context/data/scite/lexers/scite-context-lexer.lua
blob: a2bb35a575c3a918ce8987820cb5573cfb469c13 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
local info = {
    version   = 1.002,
    comment   = "basics for scintilla lpeg lexer for context/metafun",
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files",
}

-- The fold and lex functions are copied and patched from original code by Mitchell (see
-- lexer.lua). All errors are mine.
--
-- I'll probably make a whole copy and patch the other functions too as we need an extra
-- nesting model.
--
-- Also needed: preamble scan once. Can be handled in caller below and _M.preamble.
--
-- For huge files folding can be pretty slow and I do have some large ones that I keep
-- open all the time. Loading is normally no ussue, unless one has remembered the status
-- and the cursor is at the last line of a 200K line file. Optimizing the fold function
-- brought down loading of char-def.lua from 14 sec => 8 sec. Replacing the word_match
-- function and optimizing the lex function gained another 2+ seconds. A 6 second load
-- is quite ok for me.

-- Function load(lexer_name) starts with _M.WHITESPACE = lexer_name..'_whitespace' which
-- means that we need to have it frozen at the moment we load another lexer. Because spacing
-- is used to revert to a parent lexer we need to make sure that we load children as late
-- as possible in order not to get the wrong whitespace trigger. This took me quite a while
-- to figure out (not being that familiar with the internals). BTW, if performance becomes
-- an issue we can rewrite the main lex function (memorize the grammars and speed up the
-- byline variant).

-- Maybe it's safer to copy th eother methods here so that we have no dependencies, apart
-- from the the library.

local R, P, S, C, Cp, Cs, Ct, Cmt, Cc, Cf, Cg = lpeg.R, lpeg.P, lpeg.S, lpeg.C, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg
local lpegmatch = lpeg.match
local find, gmatch, match, lower, upper, gsub = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub
local concat = table.concat
local global = _G
local type, next, setmetatable, rawset = type, next, setmetatable, rawset

dofile(_LEXERHOME .. '/lexer.lua')

lexer.context = lexer.context or { }

local locations = {
 -- lexer.context.path,
    _LEXERHOME .. "/data", -- optional data directory
    _LEXERHOME .. "/..",   -- regular scite directory
}

local function collect(name)
    local definitions = loadfile(name .. ".luc") or loadfile(name .. ".lua")
    if type(definitions) == "function" then
        definitions = definitions()
    end
    if type(definitions) == "table" then
        return definitions
    else
        return nil
    end
end

-- local function exists(name)
--     local f = global.io.open(name)
--     return f and true or false
-- end
--
-- local function collect(name)
--     local f = global.io.open(name .. ".properties")
--     if f then
--         local result = { }
--         local data = gsub(f:read("*all") or "","\\ *[\n\r]+"," ")
--         for name, words in gmatch(data,".-([^%.]-)=(.-)\n") do
--             if name ~= "all" then
--                 local list = { }
--                 for word in gmatch(words,"([^ ]+)") do
--                     list[#list+1] = word
--                 end
--                 result[name] = list
--             end
--         end
--         f:close()
--         if next(result) then
--             return result
--         end
--     end
-- end

function lexer.context.loaddefinitions(name)
    for i=1,#locations do
        local data = collect(locations[i] .. "/" .. name)
        if data then
            return data
        end
    end
end

-- maybe more efficient:

function lexer.context.word_match(words,word_chars,case_insensitive)
    local chars = '%w_' -- maybe just "" when word_chars
    if word_chars then
        chars = '^([' .. chars .. gsub(word_chars,'([%^%]%-])', '%%%1') ..']+)'
    else
        chars = '^([' .. chars ..']+)'
    end
    if case_insensitive then
        local word_list = { }
        for i=1,#words do
            word_list[lower(words[i])] = true
        end
        return P(function(input, index)
            local s, e, word = find(input,chars,index)
            return word and word_list[lower(word)] and e + 1 or nil
        end)
    else
        local word_list = { }
        for i=1,#words do
            word_list[words[i]] = true
        end
        return P(function(input, index)
            local s, e, word = find(input,chars,index)
            return word and word_list[word] and e + 1 or nil
        end)
    end
end

-- nicer (todo: utf):

local defaults = R("az","AZ","\127\255","__")

function lexer.context.exact_match(words,word_chars,case_insensitive)
    local characters = concat(words)
    local pattern -- the concat catches _ etc
    if word_chars == true or word_chars == false or word_chars == nil then
        word_chars = ""
    end
    if type(word_chars) == "string" then
        pattern = S(characters) + defaults
        if case_insensitive then
            pattern = pattern + S(upper(characters)) + S(lower(characters))
        end
        if word_chars ~= "" then
            pattern = pattern + S(word_chars)
        end
    elseif word_chars then
        pattern = word_chars
    end
    if case_insensitive then
        local list = { }
        for i=1,#words do
            list[lower(words[i])] = true
        end
        return Cmt(pattern^1, function(_,i,s)
            return list[lower(s)] -- and i
        end)
    else
        local list = { }
        for i=1,#words do
            list[words[i]] = true
        end
        return Cmt(pattern^1, function(_,i,s)
            return list[s] -- and i
        end)
    end
end


-- spell checking (we can only load lua files)

-- return {
--     words = {
--         ["someword"]    = "someword",
--         ["anotherword"] = "Anotherword",
--     },
-- }

local lists = { }

local splitter = (Cf(Ct("") * (Cg(C((1-S(" \t\n\r"))^1 * Cc(true))) + P(1))^1,rawset) )^0
local splitter = (Cf(Ct("") * (Cg(C(R("az","AZ","\127\255")^1) * Cc(true)) + P(1))^1,rawset) )^0

local function splitwords(words)
    return lpegmatch(splitter,words)
end

function lexer.context.setwordlist(tag,limit) -- returns hash (lowercase keys and original values)
    if not tag or tag == "" then
        return false
    elseif lists[tag] ~= nil then
        return lists[tag]
    else
        local list = lexer.context.loaddefinitions("spell-" .. tag)
        if not list or type(list) ~= "table" then
            lists[tag] = false
            return nil
        elseif type(list.words) == "string" then
            list = splitwords(list.words)
            lists[tag] = list
            return list
        else
            list = list.words or false
            lists[tag] = list
            return list
        end
    end
end

lexer.context.wordpattern = R("az","AZ","\127\255")^3 -- todo: if limit and #s < limit then

function lexer.context.checkedword(validwords,s,i) -- ,limit
    if not validwords then
        return true, { "text", i }
    else
        -- keys are lower
        local word = validwords[s]
        if word == s then
            return true, { "okay", i } -- exact match
        elseif word then
            return true, { "warning", i } -- case issue
        else
            local word = validwords[lower(s)]
            if word == s then
                return true, { "okay", i } -- exact match
            elseif word then
                return true, { "warning", i } -- case issue
            elseif upper(s) == s then
                return true, { "warning", i } -- probably a logo or acronym
            else
                return true, { "error", i }
            end
        end
    end
end

-- overloaded functions

local FOLD_BASE         = SC_FOLDLEVELBASE
local FOLD_HEADER       = SC_FOLDLEVELHEADERFLAG
local FOLD_BLANK        = SC_FOLDLEVELWHITEFLAG

local get_style_at      = GetStyleAt
local get_property      = GetProperty
local get_indent_amount = GetIndentAmount

local h_table, b_table, n_table = { }, { }, { }

setmetatable(h_table, { __index = function(t,level) local v = { level, FOLD_HEADER } t[level] = v return v end })
setmetatable(b_table, { __index = function(t,level) local v = { level, FOLD_BLANK  } t[level] = v return v end })
setmetatable(n_table, { __index = function(t,level) local v = { level              } t[level] = v return v end })

-- local newline    = P("\r\n") + S("\r\n")
-- local splitlines = Ct( ( Ct ( (Cp() * Cs((1-newline)^1) * newline^-1) + (Cp() * Cc("") * newline) ) )^0)
--
-- local lines = lpegmatch(splitlines,text) -- iterating over lines is faster
-- for i=1, #lines do
--     local li = lines[i]
--     local line = li[2]
--     if line ~= "" then
--         local pos = li[1]
--         for i=1,nofpatterns do
--             for s, m in gmatch(line,patterns[i]) do
--                 if hash[m] then
--                     local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)]
--                     if symbols then
--                         local l = symbols[m]
--                         if l then
--                             local t = type(l)
--                             if t == 'number' then
--                                 current_level = current_level + l
--                             elseif t == 'function' then
--                                 current_level = current_level + l(text, pos, line, s, match)
--                             end
--                             if current_level < FOLD_BASE then -- integrate in previous
--                                 current_level = FOLD_BASE
--                             end
--                         end
--                     end
--                 end
--             end
--         end
--         if current_level > prev_level then
--             folds[line_num] = h_table[prev_level] -- { prev_level, FOLD_HEADER }
--         else
--             folds[line_num] = n_table[prev_level] -- { prev_level }
--         end
--         prev_level = current_level
--     else
--         folds[line_num] = b_table[prev_level] -- { prev_level, FOLD_BLANK }
--     end
--     line_num = line_num + 1
-- end
--
-- -- not that much faster but less memory:

local action_y, action_n

local newline    = P("\r\n") + S("\r\n")
local splitlines = ( (
    (Cp() * Cs((1-newline)^1) * newline^-1) / function(p,l) action_y(p,l) end
  + (                           newline   ) / function()    action_n()    end
) )^0

function lexer.context.fold(text, start_pos, start_line, start_level)
    if text == '' then
        return { }
    end
    local lexer = global._LEXER
    if lexer._fold then
        return lexer._fold(text, start_pos, start_line, start_level)
    end
    local folds = { }
    if lexer._foldsymbols then
        local fold_symbols = lexer._foldsymbols
        local line_num = start_line
        local prev_level = start_level
        local current_level = prev_level
        local patterns = fold_symbols._patterns
        local nofpatterns = #patterns
        local hash = fold_symbols._hash
        if not hash then
            hash = { }
            for symbol, matches in next, fold_symbols do
                if not find(symbol,"^_") then
                    for s, _ in next, matches do
                        hash[s] = true
                    end
                end
            end
            fold_symbols._hash = hash
        end
        action_y = function(pos,line) -- we can consider moving this one outside the function
            for i=1,nofpatterns do
                for s, m in gmatch(line,patterns[i]) do
                    if hash[m] then
                        local symbols = fold_symbols[get_style_at(start_pos + pos + s - 1)]
                        if symbols then
                            local l = symbols[m]
                            if l then
                                local t = type(l)
                                if t == 'number' then
                                    current_level = current_level + l
                                    if current_level < FOLD_BASE then -- can this happen?
                                        current_level = FOLD_BASE
                                    end
                                elseif t == 'function' then
                                    current_level = current_level + l(text, pos, line, s, match)
                                    if current_level < FOLD_BASE then
                                        current_level = FOLD_BASE
                                    end
                                end
                            end
                        end
                    end
                end
            end
            if current_level > prev_level then
                folds[line_num] = h_table[prev_level] -- { prev_level, FOLD_HEADER }
            else
                folds[line_num] = n_table[prev_level] -- { prev_level }
            end
            prev_level = current_level
            line_num = line_num + 1
        end
        action_n = function() -- we can consider moving this one outside the function
            folds[line_num] = b_table[prev_level] -- { prev_level, FOLD_BLANK }
            line_num = line_num + 1
        end
        local lines = lpegmatch(splitlines,text)
    elseif get_property('fold.by.indentation',1) == 1 then
        local current_line = start_line
        local prev_level = start_level
        for _, line in gmatch(text,'([\t ]*)(.-)\r?\n') do
            if line ~= "" then
                local current_level = FOLD_BASE + get_indent_amount(current_line)
                if current_level > prev_level then -- next level
                    local i = current_line - 1
                    while true do
                        local f = folds[i]
                        if f and f[2] == FOLD_BLANK then
                            i = i - 1
                        else
                            break
                        end
                    end
                    local f = folds[i]
                    if f then
                        f[2] = FOLD_HEADER
                    end -- low indent
                    folds[current_line] = n_table[current_level] -- { current_level } -- high indent
                elseif current_level < prev_level then -- prev level
                    local f = folds[current_line - 1]
                    if f then
                        f[1] = prev_level -- high indent
                    end
                    folds[current_line] = n_table[current_level] -- { current_level } -- low indent
                else -- same level
                    folds[current_line] = n_table[prev_level] -- { prev_level }
                end
                prev_level = current_level
            else
                folds[current_line] = b_table[prev_level] -- { prev_level, FOLD_BLANK }
            end
            current_line = current_line + 1
        end
    else
        for _ in gmatch(text,".-\r?\n") do
            folds[start_line] = n_table[start_level] -- { start_level }
            start_line = start_line + 1
        end
    end
    return folds
end

function lexer.context.lex(text,init_style)
    local lexer = global._LEXER
    local grammar = lexer._GRAMMAR
    if not grammar then
        return { }
    elseif lexer._LEXBYLINE then -- we could keep token
        local tokens = { }
        local offset = 0
        local noftokens = 0
        if true then
            for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg
                local line_tokens = lpegmatch(grammar,line)
                if line_tokens then
                    for i=1,#line_tokens do
                        local token = line_tokens[i]
                        token[2] = token[2] + offset
                        noftokens = noftokens + 1
                        tokens[noftokens] = token
                    end
                end
                offset = offset + #line
                if noftokens > 0 and tokens[noftokens][2] ~= offset then
                    noftokens = noftokens + 1
                    tokens[noftokens] = { 'default', offset + 1 }
                end
            end
        else -- alternative
            local lasttoken, lastoffset
            for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg
                local line_tokens = lpegmatch(grammar,line)
                if line_tokens then
                    for i=1,#line_tokens do
                        lasttoken = line_tokens[i]
                        lastoffset = lasttoken[2] + offset
                        lasttoken[2] = lastoffset
                        noftokens = noftokens + 1
                        tokens[noftokens] = lasttoken
                    end
                end
                offset = offset + #line
                if lastoffset ~= offset then
                    lastoffset = offset + 1
                    lasttoken = { 'default', lastoffset }
                    noftokens = noftokens + 1
                    tokens[noftokens] = lasttoken
                end
            end
        end
        return tokens
    elseif lexer._CHILDREN then
        -- as we cannot print, tracing is not possible ... this might change as we can as well
        -- generate them all in one go (sharing as much as possible)
        local _hash = lexer._HASH
        if not hash then
            hash = { }
            lexer._HASH = hash
        end
        grammar = hash[init_style]
        if grammar then
            lexer._GRAMMAR = grammar
        else
            for style, style_num in next, lexer._TOKENS do
                if style_num == init_style then
                    local lexer_name = match(style,'^(.+)_whitespace') or lexer._NAME
                    if lexer._INITIALRULE ~= lexer_name then
                        grammar = hash[lexer_name]
                        if not grammar then
                            build_grammar(lexer,lexer_name)
                            grammar = lexer._GRAMMAR
                            hash[lexer_name] = grammar
                        end
                    end
                    break
                end
            end
            grammar = grammar or lexer._GRAMMAR
            hash[init_style] = grammar
        end
        return lpegmatch(grammar,text)
    else
        return lpegmatch(grammar,text)
    end
end

-- todo: keywords: one lookup and multiple matches

function lexer.context.token(name, patt)
    return Ct(patt * Cc(name) * Cp())
end

lexer.fold        = lexer.context.fold
lexer.lex         = lexer.context.lex
lexer.token       = lexer.context.token
lexer.exact_match = lexer.context.exact_match