From 4530daff791f4e5e6b9ded048cad042b9e4ec46f Mon Sep 17 00:00:00 2001 From: Marius Date: Sun, 18 Sep 2011 23:40:29 +0300 Subject: beta 2011.09.18 22:35 --- .../data/scite/lexers/scite-context-lexer-lua.lua | 2 + .../data/scite/lexers/scite-context-lexer-mps.lua | 1 - .../data/scite/lexers/scite-context-lexer-tex.lua | 61 ++++--- .../scite/lexers/scite-context-lexer-xml-cdata.lua | 22 +++ .../lexers/scite-context-lexer-xml-comment.lua | 31 ++++ .../data/scite/lexers/scite-context-lexer-xml.lua | 202 +++++++++++++++++++++ context/data/scite/lexers/scite-context-lexer.lua | 114 ++++++++---- .../scite/lexers/themes/scite-context-theme.lua | 4 + .../data/scite/scite-context-external.properties | 5 +- context/data/scite/scite-context-readme.tex | 2 + context/data/scite/scite-ctx.lua | 2 + tex/context/base/cont-log.mkiv | 3 + tex/context/base/cont-new.mkii | 2 +- tex/context/base/cont-new.mkiv | 2 +- tex/context/base/context-version.pdf | Bin 4094 -> 4088 bytes tex/context/base/context-version.png | Bin 106425 -> 106395 bytes tex/context/base/context.mkii | 2 +- tex/context/base/context.mkiv | 2 +- tex/context/base/l-pdfview.lua | 72 +++++--- tex/context/base/mlib-pps.lua | 23 ++- tex/context/base/s-abr-01.tex | 2 + tex/context/base/s-pre-62.tex | 3 +- tex/context/base/status-files.pdf | Bin 23917 -> 23925 bytes tex/context/base/status-lua.pdf | Bin 162352 -> 162352 bytes tex/context/base/strc-sec.mkiv | 6 +- tex/context/base/syst-aux.mkiv | 12 +- tex/context/interface/cont-cs.xml | 12 +- tex/context/interface/cont-de.xml | 12 +- tex/context/interface/cont-en.xml | 12 +- tex/context/interface/cont-fr.xml | 12 +- tex/context/interface/cont-it.xml | 12 +- tex/context/interface/cont-nl.xml | 12 +- tex/context/interface/cont-pe.xml | 12 +- tex/context/interface/cont-ro.xml | 12 +- tex/generic/context/luatex/luatex-fonts-merged.lua | 2 +- 35 files changed, 524 insertions(+), 149 deletions(-) create mode 100644 context/data/scite/lexers/scite-context-lexer-xml-cdata.lua create mode 100644 context/data/scite/lexers/scite-context-lexer-xml-comment.lua create mode 100644 context/data/scite/lexers/scite-context-lexer-xml.lua diff --git a/context/data/scite/lexers/scite-context-lexer-lua.lua b/context/data/scite/lexers/scite-context-lexer-lua.lua index 49799a978..62577d4a9 100644 --- a/context/data/scite/lexers/scite-context-lexer-lua.lua +++ b/context/data/scite/lexers/scite-context-lexer-lua.lua @@ -12,6 +12,8 @@ local P, R, S, C, Cg, Cb, Cs, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cg, lpe local match, find = string.match, string.find local global = _G +-- beware: all multiline is messy, so even if it's no lexer, it should be an embedded lexer + module(...) local cldlexer = _M diff --git a/context/data/scite/lexers/scite-context-lexer-mps.lua b/context/data/scite/lexers/scite-context-lexer-mps.lua index 2d8cc3a70..afde63bcc 100644 --- a/context/data/scite/lexers/scite-context-lexer-mps.lua +++ b/context/data/scite/lexers/scite-context-lexer-mps.lua @@ -15,7 +15,6 @@ local type, next, pcall, loadfile = type, next, pcall, loadfile module(...) local metafunlexer = _M -local basepath = lexer.context and lexer.context.path or _LEXERHOME local metafunhelpers = { } local metafunconstants = { } diff --git a/context/data/scite/lexers/scite-context-lexer-tex.lua b/context/data/scite/lexers/scite-context-lexer-tex.lua index 340c3f75e..0866b35b4 100644 --- a/context/data/scite/lexers/scite-context-lexer-tex.lua +++ b/context/data/scite/lexers/scite-context-lexer-tex.lua @@ -45,8 +45,6 @@ local contextlexer = _M local cldlexer = lexer.load('scite-context-lexer-cld') local mpslexer = lexer.load('scite-context-lexer-mps') -local basepath = lexer.context and lexer.context.path or _LEXERHOME - local commands = { en = { } } local primitives = { } local helpers = { } @@ -111,9 +109,14 @@ local knowncommand = Cmt(cstoken^1, function(_,i,s) return currentcommands[s] and i end) -local validwords = false +local wordpattern = lexer.context.wordpattern +local checkedword = lexer.context.checkedword +local setwordlist = lexer.context.setwordlist +local validwords = false + +-- % language=uk -local knownpreamble = Cmt(P("% "), function(input,i,_) +local knownpreamble = Cmt(#P("% "), function(input,i,_) -- todo : utfbomb if i < 10 then validwords = false local s, e, word = find(input,'^(.+)[\n\r]',i) -- combine with match @@ -123,7 +126,7 @@ local knownpreamble = Cmt(P("% "), function(input,i,_) currentcommands = commands[interface] or commands.en or { } end local language = match(word,"language=(..)") - validwords = language and lexer.context.setwordlist(language) + validwords = language and setwordlist(language) end end return false @@ -195,26 +198,34 @@ local p_text = cstoken^1 --maybe add punctuation and space -- no looking back = #(1-S("[=")) * cstoken^3 * #(1-S("=]")) -local p_word = Cmt(cstoken^3, function(_,i,s) - if not validwords then - return true, { "text", i } +-- local p_word = Cmt(wordpattern, function(_,i,s) +-- if not validwords then +-- return true, { "text", i } +-- else +-- -- keys are lower +-- local word = validwords[s] +-- if word == s then +-- return true, { "okay", i } -- exact match +-- elseif word then +-- return true, { "warning", i } -- case issue +-- else +-- local word = validwords[lower(s)] +-- if word == s then +-- return true, { "okay", i } -- exact match +-- elseif word then +-- return true, { "warning", i } -- case issue +-- else +-- return true, { "error", i } +-- end +-- end +-- end +-- end) + +local p_word = Cmt(wordpattern, function(_,i,s) + if validwords then + return checkedword(validwords,s,i) else - -- keys are lower - local word = validwords[s] - if word == s then - return true, { "okay", i } -- exact match - elseif word then - return true, { "warning", i } -- case issue - else - local word = validwords[lower(s)] - if word == s then - return true, { "okay", i } -- exact match - elseif word then - return true, { "warning", i } -- case issue - else - return true, { "error", i } - end - end + return true, { "text", i } end end) @@ -274,7 +285,7 @@ local csname = token('user', p_csname ) local grouping = token('grouping', p_grouping ) local special = token('special', p_special ) local extra = token('extra', p_extra ) -local text = token('default', p_text ) +----- text = token('default', p_text ) ----- word = token("okay", p_word ) local word = p_word diff --git a/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua b/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua new file mode 100644 index 000000000..71826099c --- /dev/null +++ b/context/data/scite/lexers/scite-context-lexer-xml-cdata.lua @@ -0,0 +1,22 @@ +local lexer = lexer +local token = lexer.token +local P = lpeg.P + +module(...) + +local commentlexer = _M + +local whitespace = commentlexer.WHITESPACE -- triggers states + +local space = lexer.space +local nospace = 1 - space - P("]]>") + +local p_spaces = token(whitespace, space ^1) +local p_cdata = token("comment", nospace^1) + +_rules = { + { "whitespace", p_spaces }, + { "cdata", p_cdata }, +} + +_tokenstyles = lexer.context.styleset diff --git a/context/data/scite/lexers/scite-context-lexer-xml-comment.lua b/context/data/scite/lexers/scite-context-lexer-xml-comment.lua new file mode 100644 index 000000000..2d9ce66bd --- /dev/null +++ b/context/data/scite/lexers/scite-context-lexer-xml-comment.lua @@ -0,0 +1,31 @@ +local lexer = lexer +local token = lexer.token +local P = lpeg.P + +module(...) + +local commentlexer = _M + +local whitespace = commentlexer.WHITESPACE -- triggers states + +local space = lexer.space +local nospace = 1 - space - P("-->") + +local p_spaces = token(whitespace, space ^1) +local p_comment = token("comment", nospace^1) + +_rules = { + { "whitespace", p_spaces }, + { "comment", p_comment }, +} + +_tokenstyles = lexer.context.styleset + +_foldsymbols = { + _patterns = { + "<%!%-%-", "%-%->", -- comments + }, + ["comment"] = { + ["" ] = -1, + } +} diff --git a/context/data/scite/lexers/scite-context-lexer-xml.lua b/context/data/scite/lexers/scite-context-lexer-xml.lua new file mode 100644 index 000000000..0441585c1 --- /dev/null +++ b/context/data/scite/lexers/scite-context-lexer-xml.lua @@ -0,0 +1,202 @@ +local info = { + version = 1.002, + comment = "scintilla lpeg lexer for metafun", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", +} + +-- adapted from the regular context pretty printer code (after all, lexing +-- boils down to much of the same and there are only so many ways to do +-- things). Simplified a bit as we have a different nesting model. + +-- todo: parse entities in attributes + +local lexer = lexer +local global, string, table, lpeg = _G, string, table, lpeg +local token, style, colors, exact_match, no_style = lexer.token, lexer.style, lexer.colors, lexer.exact_match, lexer.style_nothing +local P, R, S, V, C, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cmt +local type, setmetatable = type, setmetatable +local match, find = string.match, string.find + +module(...) + +local examplelexer = _M + +local whitespace = examplelexer.WHITESPACE -- triggers states + +local space = lexer.space -- S(" \t\n\r\v\f") +local any = lexer.any -- P(1) + +local dquote = P('"') +local squote = P("'") +local colon = P(":") +local semicolon = P(";") +local equal = P("=") +local ampersand = P("&") + +local name = (R("az","AZ","09") + S('_-.'))^1 +local openbegin = P("<") +local openend = P("") + P(">") +local closeend = P(">") +local opencomment = P("") +local openinstruction = P("") +local opencdata = P("") + +local entity = ampersand * (1-semicolon)^1 * semicolon + +local wordpattern = lexer.context.wordpattern +local checkedword = lexer.context.checkedword +local setwordlist = lexer.context.setwordlist +local validwords = false + +-- +-- +-- + +local p_preamble = Cmt(#P("]*%?>%s*<%?context%-xml%-directive%s+editor%s+language%s+(..)%s+%?>") + if not language then + language = match(input,'^<%?xml[^>]*language=[\"\'](..)[\"\'][^>]*%?>',i) + end + if language then + validwords = setwordlist(language) + end + end + return false +end) + + +local p_word = + Cmt(wordpattern, function(_,i,s) + if validwords then + return checkedword(validwords,s,i) + else + return true, { "text", i } + end + end) + +local p_rest = + token("default", any) + +local p_text = + token("default", (1-S("<>&")-space)^1) + +local p_spacing = + token(whitespace, space^1) + +local p_optionalwhitespace = + p_spacing^0 + +local p_localspacing = + token("default", space^1) + +-- Because we want a differently colored open and close we need an embedded lexer (whitespace +-- trigger). What is actually needed is that scintilla applies the current whitespace style. +-- Even using different style keys is not robust as they can be shared. I'll fix the main +-- lexer code. + +local p_sstring = + token("quote",dquote) + * token("string",(1-dquote)^0) -- different from context + * token("quote",dquote) + +local p_dstring = + token("quote",squote) + * token("string",(1-squote)^0) -- different from context + * token("quote",squote) + +-- local p_comment = +-- token("command",opencomment) +-- * token("comment",(1-closecomment)^0) -- different from context +-- * token("command",closecomment) + +-- local p_cdata = +-- token("command",opencdata) +-- * token("comment",(1-closecdata)^0) -- different from context +-- * token("command",closecdata) + +local commentlexer = lexer.load("scite-context-lexer-xml-comment") +local cdatalexer = lexer.load("scite-context-lexer-xml-cdata") + +lexer.embed_lexer(examplelexer, commentlexer, token("command",opencomment), token("command",closecomment)) +lexer.embed_lexer(examplelexer, cdatalexer, token("command",opencdata), token("command",closecdata)) + +-- maybe cdata just text (then we don't need the extra lexer as we only have one comment then) + +local p_name = + token("plain",name) + * ( + token("default",colon) + * token("keyword",name) + )^1 + + token("keyword",name) + +local p_key = p_name + +local p_attributes = ( + p_optionalwhitespace + * p_key + * p_optionalwhitespace + * token("plain",equal) + * p_optionalwhitespace + * (p_dstring + p_sstring) + * p_optionalwhitespace +)^0 + +local p_open = + token("keyword",openbegin) + * p_name + * p_optionalwhitespace + * p_attributes + * token("keyword",closebegin) + +local p_close = + token("keyword",openend) + * p_name + * p_optionalwhitespace + * token("keyword",closeend) + +local p_entity = + token("constant",entity) + +local p_instruction = + token("command",openinstruction * P("xml")) + * p_optionalwhitespace + * p_attributes + * p_optionalwhitespace + * token("command",closeinstruction) + + token("command",openinstruction * name) + * token("default",(1-closeinstruction)^1) + * token("command",closeinstruction) + +_rules = { + { "whitespace", p_spacing }, + { "preamble", p_preamble }, + { "word", p_word }, +-- { "text", p_text }, +-- { "comment", p_comment }, +-- { "cdata", p_cdata }, + { "instruction", p_instruction }, + { "close", p_close }, + { "open", p_open }, + { "entity", p_entity }, + { "rest", p_rest }, +} + +_tokenstyles = lexer.context.styleset + +_foldsymbols = { -- somehow doesn't work yet + _patterns = { + "[<>]", + }, + ["keyword"] = { + ["<"] = 1, [">"] = -1, + }, +} diff --git a/context/data/scite/lexers/scite-context-lexer.lua b/context/data/scite/lexers/scite-context-lexer.lua index 20af5d68f..a2bb35a57 100644 --- a/context/data/scite/lexers/scite-context-lexer.lua +++ b/context/data/scite/lexers/scite-context-lexer.lua @@ -9,6 +9,11 @@ local info = { -- The fold and lex functions are copied and patched from original code by Mitchell (see -- lexer.lua). All errors are mine. -- +-- I'll probably make a whole copy and patch the other functions too as we need an extra +-- nesting model. +-- +-- Also needed: preamble scan once. Can be handled in caller below and _M.preamble. +-- -- For huge files folding can be pretty slow and I do have some large ones that I keep -- open all the time. Loading is normally no ussue, unless one has remembered the status -- and the cursor is at the last line of a 200K line file. Optimizing the fold function @@ -24,6 +29,9 @@ local info = { -- an issue we can rewrite the main lex function (memorize the grammars and speed up the -- byline variant). +-- Maybe it's safer to copy th eother methods here so that we have no dependencies, apart +-- from the the library. + local R, P, S, C, Cp, Cs, Ct, Cmt, Cc, Cf, Cg = lpeg.R, lpeg.P, lpeg.S, lpeg.C, lpeg.Cp, lpeg.Cs, lpeg.Ct, lpeg.Cmt, lpeg.Cc, lpeg.Cf, lpeg.Cg local lpegmatch = lpeg.match local find, gmatch, match, lower, upper, gsub = string.find, string.gmatch, string.match, string.lower, string.upper, string.gsub @@ -158,6 +166,74 @@ function lexer.context.exact_match(words,word_chars,case_insensitive) end end + +-- spell checking (we can only load lua files) + +-- return { +-- words = { +-- ["someword"] = "someword", +-- ["anotherword"] = "Anotherword", +-- }, +-- } + +local lists = { } + +local splitter = (Cf(Ct("") * (Cg(C((1-S(" \t\n\r"))^1 * Cc(true))) + P(1))^1,rawset) )^0 +local splitter = (Cf(Ct("") * (Cg(C(R("az","AZ","\127\255")^1) * Cc(true)) + P(1))^1,rawset) )^0 + +local function splitwords(words) + return lpegmatch(splitter,words) +end + +function lexer.context.setwordlist(tag,limit) -- returns hash (lowercase keys and original values) + if not tag or tag == "" then + return false + elseif lists[tag] ~= nil then + return lists[tag] + else + local list = lexer.context.loaddefinitions("spell-" .. tag) + if not list or type(list) ~= "table" then + lists[tag] = false + return nil + elseif type(list.words) == "string" then + list = splitwords(list.words) + lists[tag] = list + return list + else + list = list.words or false + lists[tag] = list + return list + end + end +end + +lexer.context.wordpattern = R("az","AZ","\127\255")^3 -- todo: if limit and #s < limit then + +function lexer.context.checkedword(validwords,s,i) -- ,limit + if not validwords then + return true, { "text", i } + else + -- keys are lower + local word = validwords[s] + if word == s then + return true, { "okay", i } -- exact match + elseif word then + return true, { "warning", i } -- case issue + else + local word = validwords[lower(s)] + if word == s then + return true, { "okay", i } -- exact match + elseif word then + return true, { "warning", i } -- case issue + elseif upper(s) == s then + return true, { "warning", i } -- probably a logo or acronym + else + return true, { "error", i } + end + end + end +end + -- overloaded functions local FOLD_BASE = SC_FOLDLEVELBASE @@ -348,7 +424,7 @@ function lexer.context.lex(text,init_style) local noftokens = 0 if true then for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg - local line_tokens = lpeg_match(grammar,line) + local line_tokens = lpegmatch(grammar,line) if line_tokens then for i=1,#line_tokens do local token = line_tokens[i] @@ -366,7 +442,7 @@ function lexer.context.lex(text,init_style) else -- alternative local lasttoken, lastoffset for line in gmatch(text,'[^\r\n]*\r?\n?') do -- could be an lpeg - local line_tokens = lpeg_match(grammar,line) + local line_tokens = lpegmatch(grammar,line) if line_tokens then for i=1,#line_tokens do lasttoken = line_tokens[i] @@ -431,37 +507,3 @@ lexer.fold = lexer.context.fold lexer.lex = lexer.context.lex lexer.token = lexer.context.token lexer.exact_match = lexer.context.exact_match - --- spell checking (we can only load lua files) - -local lists = { } - -local splitter = (Cf(Ct("") * (Cg(C((1-S(" \t\n\r"))^1 * Cc(true))) + P(1))^1,rawset) )^0 -local splitter = (Cf(Ct("") * (Cg(C(R("az","AZ","\127\255")^1) * Cc(true)) + P(1))^1,rawset) )^0 - -local function splitwords(words) - return lpegmatch(splitter,words) -end - -function lexer.context.setwordlist(tag) - if not tag or tag == "" then - return false - elseif lists[tag] ~= nil then - return lists[tag] - else - local list = collect("spell-" .. tag) - if not list or type(list) ~= "table" then - lists[tag] = false - return nil - elseif type(list.words) == "string" then - list = splitwords(list.words) - lists[tag] = list - return list - else - list = list.words or false - lists[tag] = list - return list - end - end -end - diff --git a/context/data/scite/lexers/themes/scite-context-theme.lua b/context/data/scite/lexers/themes/scite-context-theme.lua index 94f623cd8..ed1c5086b 100644 --- a/context/data/scite/lexers/themes/scite-context-theme.lua +++ b/context/data/scite/lexers/themes/scite-context-theme.lua @@ -13,6 +13,10 @@ local context_path = "t:/sources" -- c:/data/tex-context/tex/texmf-context/tex/b local font_name = 'Dejavu Sans Mono' local font_size = 14 +if not WIN32 then + font_name = '!' .. font_name +end + local global = _G -- dofile(_LEXERHOME .. '/themes/scite.lua') -- starting point so we miss nothing diff --git a/context/data/scite/scite-context-external.properties b/context/data/scite/scite-context-external.properties index 5cb0e8f98..14bdd222f 100644 --- a/context/data/scite/scite-context-external.properties +++ b/context/data/scite/scite-context-external.properties @@ -23,7 +23,7 @@ lexer.$(file.patterns.metafun)=lpeg_scite-context-lexer-mps lexer.$(file.patterns.context)=lpeg_scite-context-lexer-tex lexer.$(file.patterns.tex)=lpeg_scite-context-lexer-tex lexer.$(file.patterns.lua)=lpeg_scite-context-lexer-lua -lexer.$(file.patterns.xml)=lpeg_xml +lexer.$(file.patterns.example)=lpeg_scite-context-lexer-xml comment.block.lpeg_scite-context-lexer-tex=% comment.block.at.line.start.lpeg_scite-context-lexer-tex=1 @@ -37,6 +37,9 @@ comment.block.at.line.start.lpeg_scite-context-lexer-lua=1 comment.block.lpeg_scite-context-lexer-cld=-- comment.block.at.line.start.lpeg_scite-context-lexer-cld=1 +#~ comment.block.lpeg_scite-context-lexer-xml= - + diff --git a/tex/context/interface/cont-de.xml b/tex/context/interface/cont-de.xml index 61441dd9c..ff375be46 100644 --- a/tex/context/interface/cont-de.xml +++ b/tex/context/interface/cont-de.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/context/interface/cont-en.xml b/tex/context/interface/cont-en.xml index b532f2b21..100148f42 100644 --- a/tex/context/interface/cont-en.xml +++ b/tex/context/interface/cont-en.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/context/interface/cont-fr.xml b/tex/context/interface/cont-fr.xml index 615eda784..d6096aa55 100644 --- a/tex/context/interface/cont-fr.xml +++ b/tex/context/interface/cont-fr.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/context/interface/cont-it.xml b/tex/context/interface/cont-it.xml index 5038f13ac..7675c612c 100644 --- a/tex/context/interface/cont-it.xml +++ b/tex/context/interface/cont-it.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/context/interface/cont-nl.xml b/tex/context/interface/cont-nl.xml index dbe1b0223..a84a88c0e 100644 --- a/tex/context/interface/cont-nl.xml +++ b/tex/context/interface/cont-nl.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/context/interface/cont-pe.xml b/tex/context/interface/cont-pe.xml index 12a54339d..94e042012 100644 --- a/tex/context/interface/cont-pe.xml +++ b/tex/context/interface/cont-pe.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/context/interface/cont-ro.xml b/tex/context/interface/cont-ro.xml index b7fa62d59..8c12b7c35 100644 --- a/tex/context/interface/cont-ro.xml +++ b/tex/context/interface/cont-ro.xml @@ -2,16 +2,18 @@ - + diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 132562502..e4d504969 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 09/17/11 15:56:52 +-- merge date : 09/18/11 22:35:11 do -- begin closure to overcome local limits and interference -- cgit v1.2.3