path: root/tex/context/modules/mkiv/m-scite.mkiv

%D \module
%D   [       file=m-scite,
%D        version=2014.04.28,
%D          title=\CONTEXT\ Extra Modules,
%D       subtitle=\SCITE\ lexers,
%D         author=Hans Hagen,
%D           date=\currentdate,
%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
%C
%C This module is part of the \CONTEXT\ macro||package and is
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.

% We can simplify the scite lexers, as long as we're able to return the
% lexed result table and provide alexer module with the functions that
% the lexer expects (so I need to decipher the cxx file).
%
% lexer._TOKENSTYLES : table
% lexer._CHILDREN    : flag
% lexer._EXTRASTYLES : table
% lexer._GRAMMAR     : flag
%
% lexers.load        : function
% lexers.lex         : function
%
% And some properties that map styles onto scintilla styling. I get the
% impression that we end up with something simpler, a hybrid between the
% scite lexing and the current context way, so we get an intermediate
% step, with some penalty for context, but at least I don't have to
% maintain two sets (three sets as we also have a line based series).

% TODO: as these files are in tds we can locate them and set the lexer root
% to that one. Currently we're on: we're on context/documents.

% TODO: tab

% This is an experiment: eventually we need to hook it into the verbatim code
% and deal with widow lines and so.

\startluacode

-- todo: merge with collapse
-- todo: prehash whitespaces

-- todo: hook into the pretty print code
-- todo: a simple catcode regime with only \ { }

local gsub, sub, find, lower = string.gsub, string.sub, string.find, string.lower
local concat = table.concat
local formatters = string.formatters
local lpegmatch = lpeg.match
local setmetatableindex = table.setmetatableindex

local scite   = require("util-sci")
buffers.scite = scite

-- context output:

local f_def_color = formatters["\\definecolor[slxc%s][h=%02X%02X%02X]%%"]
local f_fore_none = formatters["\\unexpanded\\def\\slx%s#1{{\\slxc%s#1}\\slxbreak}%%"]
local f_fore_bold = formatters["\\unexpanded\\def\\slx%s#1{{\\slxc%s\\bf#1}\\slxbreak}%%"]
local f_none_bold = formatters["\\unexpanded\\def\\slx%s#1{{\\bf#1}\\slxbreak}%%"]
local f_none_none = formatters["\\unexpanded\\def\\slx%s#1{{#1}\\slxbreak}%%"]
local f_texstyled = formatters["\\slx%s{%s}\\slxbreak"]
local f_hanging   = formatters["\\slxb{%r}%s\\slxe"] -- we need to round: lua 5.3

local v_none      = interfaces.variables.none

local f_mapping   = [[
\let\string\slxL\string\letterleftbrace
\let\string\slxR\string\letterrightbrace
\let\string\slxM\string\letterdollar
\let\string\slxV\string\letterbar
\let\string\slxU\string\letterhat
\let\string\slxD\string\letterunderscore
\let\string\slxH\string\letterhash
\let\string\slxB\string\letterbackslash
\let\string\slxP\string\letterpercent
\let\string\slxT\string\lettertilde
\let\string\slxS\string\fixedspace
%]]

local replacer = lpeg.replacer {
    ["{"]  = "\\slxL ",
    ["}"]  = "\\slxR ",
    ["$"]  = "\\slxM ",
    ["^"]  = "\\slxU ",
    ["_"]  = "\\slxD ",
    ["|"]  = "\\slxV ",
    ["#"]  = "\\slxH ",
    ["\\"] = "\\slxB ",
    ["%"]  = "\\slxP ",
    ["~"]  = "\\slxT ",
    [" "]  = "\\slxS ", -- can be made more efficient: \\slxF{n}
}

local colors = nil

local function exportcolors()
    if not colors then
        scite.loadscitelexer()
        local function black(f)
            return (f[1] == f[2]) and (f[2] == f[3]) and (f[3] == 0)
        end
--         local result, r = { f_mapping }, 1
local result, r = { }, 0
        for k, v in table.sortedhash(lexer.context.styles) do
            local fore = v.fore
            if fore and not black(fore) then
                r = r + 1
                result[r] = f_def_color(k,fore[1],fore[2] or fore[1],fore[3] or fore[1])
            end
        end
        r = r + 1
        result[r] = "%"
        for k, v in table.sortedhash(lexer.context.styles) do
            local bold = v.bold
            local fore = v.fore
            r = r + 1
            if fore and not black(fore) then
                if bold then
                    result[r] = f_fore_bold(k,k)
                else
                    result[r] = f_fore_none(k,k)
                end
            else
                if bold then
                    result[r] = f_none_bold(k)
                else
                    result[r] = f_none_none(k)
                end
            end
        end
        colors = concat(result,"\n")
    end
    return colors
end

local function exportwhites()
    return setmetatableindex(function(t,k)
        local v = find(k,"white") and true or false
        t[k] = v
        return v
    end)
end

local function exportstyled(lexer,text)
    local result = lexer.lex(lexer,text,0)
    local start = 1
    local whites = exportwhites()
    local buffer = { }
    for i=1,#result,2 do
        local style = result[i]
        local position = result[i+1]
        local txt = sub(text,start,position-1)
        txt = lpegmatch(replacer,txt)
        if txt == "" then
            -- skip
        elseif whites[style] then
            buffer[#buffer+1] = txt
        else
            buffer[#buffer+1] = f_texstyled(style,txt)
        end
        start = position
    end
    buffer = concat(buffer)
    return buffer
end

function scite.installcommands()
    context(exportcolors())
end

local function slxF(b,e)
    local d = (e - b)/6
    if d > 0 then
        return "\\slxF{" .. d .. "}"
    else
        return " "
    end
end

local p1 = lpeg.tsplitat(lpeg.patterns.newline)
local p2 = lpeg.P("\\slxS ")
local p3 = p2^1
local p4 = lpeg.Cs( ( (lpeg.Cp() * p2 * p2^1 * lpeg.Cp()) / slxF + lpeg.P(1) )^0 )

local function indent(str)
    local l = lpegmatch(p1,str)
    for i=1,#l do
        local s = l[i]
        if #s > 0 then
            local n = lpegmatch(p3,s)
            if n then
                n = (n-1)/6 -- length of "\\slxS "
            else
                n = 0
            end
            if n > 0 then
                s = sub(s,n*6+1)
            end
            s = lpegmatch(p4,s)   -- can be combined
            l[i] = f_hanging(n,s) -- "\\slxb{%s}%s\\slxe "
        end
    end
    return concat(l,"\n")
end

local assignbuffer = buffers.assign
local getcontent   = buffers.getcontent
local loaddata     = io.loaddata
local loadedlexers = scite.loadedlexers

local function lexdata(data,lexname)
    if not data then
        data = ""
    elseif data ~= "" then
        if data and not find(data,"[\r\n]$") then
            -- fix for lexbyline
            data = data .. "\r"
        end
        if lexname == v_none then
--             data = string.formatters["%!tex!"](data)
            data = exportstyled(loadedlexers["dummy"] or loadedlexers.tex,data)
        else
            data = exportstyled(loadedlexers[lexname] or loadedlexers.tex,data)
        end
        data = indent(data)
    end
 -- io.savedata("temp.log",data)
    assignbuffer("lex",data)
end

scite.lexdata = lexdata

function scite.lexbuffer(name,lexname)
    lexdata(getcontent(name) or "",lexname or "tex")
end

function scite.lexfile(filename,lexname)
    lexdata(loaddata(filename) or "",lexname or file.suffix(filename))
end

interfaces.implement {
    name      = "scitelexfile",
    arguments = "string",
    actions   = scite.lexfile,
}

interfaces.implement {
    name      = "scitelexbuffer",
    arguments = { "string", "string" },
    actions   = scite.lexbuffer,
}

interfaces.implement {
    name      = "sciteinstallcommands",
    actions   = scite.installcommands,
}

local startInlineScite  = context.startInlineScite
local stopInlineScite   = context.stopInlineScite
local startDisplayScite = context.startDisplayScite
local stopDisplayScite  = context.stopDisplayScite

local lexdata           = buffers.scite.lexdata

local handler = visualizers.newhandler {
    startinline  = function(settings) startInlineScite(settings.name or "") end,
    stopinline   = function(settings) stopInlineScite() end,
    startdisplay = function(settings) startDisplayScite(settings.name or "") end,
    stopdisplay  = function(settings) stopDisplayScite() end,
    noescape     = true,
}

local knownlexers = scite.knownlexers

local parser = function(content,specification)
    local method = lower(specification.method)
    lexdata(content,knownlexers[method] or method)
end

local visualizer = {
    handler = handler,
    parser  = parser,
}

visualizers.register("cld",   visualizer)
visualizers.register("tex",   visualizer)
visualizers.register("lua",   visualizer)
visualizers.register("mps",   visualizer)
visualizers.register("mp",    visualizer)
visualizers.register("pdf",   visualizer)
visualizers.register("xml",   visualizer)
visualizers.register("bibtex",visualizer)
visualizers.register("btx",   visualizer)
visualizers.register("web",   visualizer)
visualizers.register("cpp",   visualizer)
visualizers.register("txt",   visualizer)
visualizers.register("bnf",   visualizer)
visualizers.register("sql",   visualizer)
visualizers.register("json",  visualizer)

visualizers.register("sas",   visualizer)

function scite.register(name)
    visualizers.register(name,visualizer)
end

moduledata.scite = scite

\stopluacode

\definetyping[TEX] [option=cld]
\definetyping[LUA] [option=lua]
\definetyping[BTX] [option=bibtex]
\definetyping[MPS] [option=mps]
\definetyping[MP]  [option=mp]
\definetyping[PDF] [option=pdf]
\definetyping[CPP] [option=web]
\definetyping[WEB] [option=web]
\definetyping[TXT] [option=txt]
\definetyping[BNF] [option=bnf]    % I might use this in the metafun manual.
\definetyping[SQL] [option=sql]    % To be tested in an upcoming manual.
\definetyping[JSON][option=json]   % To be tested in an upcoming manual.
\definetyping[NONE][option=none]

\definetyping[SAS] [option=sas]

% This is a preliminary interface.

\unprotect

\newdimen\scitespaceskip

\unexpanded\def\buff_scite_slxb#1%
  {% we can have a side float
   \advance\hangindent\numexpr#1+2\relax\scitespaceskip
   \begstrut\hskip#1\scitespaceskip
   \ifcase\hangafter
     \hangafter\plusone\relax
   \fi}

\unexpanded\def\buff_scite_slxe
  {\endstrut\par}

\unexpanded\def\buff_scite_slxs  {\hskip\scitespaceskip\relax}
\unexpanded\def\buff_scite_slxf#1{\hskip#1\scitespaceskip\relax}

\unexpanded\def\installscitecommandsinline
  {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr
   \let\slxb\gobbleoneargument
   \let\slxe\space
   \let\slxbreak\relax
   \let\installscitecommandsinline\relax}

\unexpanded\def\installscitecommandsdisplay
  {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr
   \let\slxb\buff_scite_slxb
   \let\slxe\buff_scite_slxe
   \let\installscitecommandsdisplay\relax}

\clf_sciteinstallcommands

\let\slxb    \gobbleoneargument
\let\slxe    \space
\let\slxbreak\relax

\let\slxL    \letterleftbrace
\let\slxR    \letterrightbrace
\let\slxM    \letterdollar
\let\slxV    \letterbar
\let\slxU    \letterhat
\let\slxD    \letterunderscore
\let\slxH    \letterhash
\let\slxB    \letterbackslash
\let\slxP    \letterpercent
\let\slxT    \lettertilde
\let\slxS    \fixedspace

\let\slxS    \buff_scite_slxs
\let\slxF    \buff_scite_slxf

\def\module_scite_inherit_typing
  {\buff_verbatim_initialize_typing_one
   \buff_verbatim_set_line_margin}

\unexpanded\def\startscite
  {\begingroup
   \dosingleempty\buff_scite_start}

\def\buff_scite_start[#1]%
  {\edef\currentscitelexer{#1}%
   \dostartbuffer[@scite@][startscite][stopscite]}

\unexpanded\def\stopscite
  {\scitebuffer[\ifx\currentscitelexer\empty tex\else\currentscitelexer\fi][@scite@]%
   \endgroup}

\unexpanded\def\scitefile
  {\dosingleargument\module_scite_file}

\unexpanded\def\module_scite_file[#1]%
  {\begingroup
   \setcatcodetable\ctxcatcodes % needed in xml
   \clf_scitelexfile{#1}%
   \tt
   \installscitecommandsdisplay
   \module_scite_inherit_typing
   \dontcomplain
   \raggedright
   \startcontextcode
   \startlines
   \getbuffer[lex]
   \stoplines
   \stopcontextcode
   \endgroup}

\unexpanded\def\scitebuffer
  {\dodoubleempty\module_scite_buffer}

\unexpanded\def\module_scite_buffer[#1][#2]%
  {\begingroup
   \setcatcodetable\ctxcatcodes % needed in xml
   \ifsecondargument
     \clf_scitelexbuffer{#2}{#1}%
   \else
     \clf_scitelexbuffer{#1}{tex}%
   \fi
   \tt
   \installscitecommandsdisplay
   \module_scite_inherit_typing
   \dontcomplain
   \raggedright
   \startlines
   \getbuffer[lex]
   \stoplines
   \endgroup}

\unexpanded\def\sciteinlinebuffer
  {\dodoubleempty\module_scite_buffer_inline}

\unexpanded\def\module_scite_buffer_inline[#1][#2]%
  {\dontleavehmode
   \begingroup
   \setcatcodetable\ctxcatcodes % needed in xml
   \ifsecondargument
     \clf_scitelexbuffer{#2}{#1}%
   \else
     \clf_scitelexbuffer{#1}{tex}%
   \fi
   \tt
   \installscitecommandsinline
   \module_scite_inherit_typing
   \dontcomplain
   \getbuffer[lex]%
   \endgroup}

\unexpanded\def\startInlineScite#1%
  {\dontleavehmode
   \begingroup
   \setcatcodetable\ctxcatcodes % needed in xml
   \tt
   \installscitecommandsinline
   \dontcomplain
   \ignorespaces
   \clf_getbuffercontent{lex}% much faster than getbuffer
   \removeunwantedspaces}

\unexpanded\def\stopInlineScite
  {\endgroup}

\unexpanded\def\startDisplayScite#1%
  {\begingroup
   \setcatcodetable\ctxcatcodes % needed in xml
   \tt
   \installscitecommandsdisplay
   \dontcomplain
\buff_verbatim_initialize_typing_one
\buff_verbatim_set_line_margin
   \raggedright
   \startlines
   \clf_getbuffer{lex}}

\unexpanded\def\stopDisplayScite
  {\stoplines
   \endgroup}

\unexpanded\def\slxbreak
  {\allowbreak}

\protect

\continueifinputfile{m-scite.mkiv}

\setupbodyfont[dejavu,8pt]

\setuplayout
  [width=middle,
   height=middle,
   header=1cm,
   footer=1cm,
   topspace=1cm,
   bottomspace=1cm,
   backspace=1cm]

\startbuffer[demo]
\startsubsubject[title={oeps}]

\startMPcode
    draw fullcircle
        scaled 2cm
        withpen pencircle scaled 1mm
        withcolor .5green;
    draw textext (
        lua (
            "local function f(s) return string.upper(s) end mp.quoted(f('foo'))"
        )
    ) withcolor .5red ;
\stopMPcode

\startluacode
    context("foo")
\stopluacode

\stopsubsubject
\stopbuffer

\starttext

% \scitefile[../lexers/scite-context-lexer.lua]  \page
% \scitefile[t:/manuals/about/about-metafun.tex] \page
% \scitefile[t:/sources/strc-sec.mkiv]           \page
% \scitefile[e:/tmp/mp.w]                        \page
% \scitefile[t:/manuals/hybrid/tugboat.bib]        \page
\scitefile[e:/tmp/test.bib]        \page

% \getbuffer[demo] \scitebuffer[demo]

\startbuffer[foo]
This is text. % line 1
This is text. % line 2
\stopbuffer

\scitebuffer[none][foo]

\sciteinlinebuffer[none][foo]

\stoptext