beta 2010.11.20 12:51

17 files changed, 300 insertions, 32 deletions

diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index d45e7e5fe..7b5348424 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -188,9 +188,11 @@ local utfgmatch        = unicode and unicode.utf8.gmatch
 
 local anything         = P(1)
 local endofstring      = P(-1)
+local alwaysmatched    = P(true)
 
 patterns.anything      = anything
 patterns.endofstring   = endofstring
+patterns.beginofstring = alwaysmatched
 
 local digit, sign      = R('09'), S('+-')
 local cr, lf, crlf     = P("\r"), P("\n"), P("\r\n")
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index d45e7e5fe..7b5348424 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -188,9 +188,11 @@ local utfgmatch        = unicode and unicode.utf8.gmatch
 
 local anything         = P(1)
 local endofstring      = P(-1)
+local alwaysmatched    = P(true)
 
 patterns.anything      = anything
 patterns.endofstring   = endofstring
+patterns.beginofstring = alwaysmatched
 
 local digit, sign      = R('09'), S('+-')
 local cr, lf, crlf     = P("\r"), P("\n"), P("\r\n")
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index d45e7e5fe..7b5348424 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -188,9 +188,11 @@ local utfgmatch        = unicode and unicode.utf8.gmatch
 
 local anything         = P(1)
 local endofstring      = P(-1)
+local alwaysmatched    = P(true)
 
 patterns.anything      = anything
 patterns.endofstring   = endofstring
+patterns.beginofstring = alwaysmatched
 
 local digit, sign      = R('09'), S('+-')
 local cr, lf, crlf     = P("\r"), P("\n"), P("\r\n")
diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua
index 8d86cd761..19e3a142b 100644
--- a/tex/context/base/buff-ini.lua
+++ b/tex/context/base/buff-ini.lua
@@ -1,6 +1,6 @@
 if not modules then modules = { } end modules ['buff-ini'] = {
     version   = 1.001,
-    comment   = "companion to core-buf.mkiv",
+    comment   = "companion to buff-ini.mkiv",
     author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
     copyright = "PRAGMA ADE / ConTeXt Development Team",
     license   = "see context related readme files"
@@ -341,7 +341,7 @@ function buffers.run(name,list,encapsulate)
     end
 end
 
-local printer = (lpeg.patterns.textline/texprint)^0
+local printer = (lpeg.patterns.textline/texprint)^0 -- not the right one, we can use context(b)
 
 function buffers.get(name)
     local b = data[name]
@@ -785,7 +785,7 @@ function buffers.setescapepair(name,pair)
     end
 end
 
--- THIS WILL BECOME A FRAMEWORK: the problem with prety printing is that
+-- THIS WILL BECOME A FRAMEWORK: the problem with pretty printing is that
 -- we deal with snippets and therefore we need tolerant parsing
 
 --~ local type = type
diff --git a/tex/context/base/buff-vis.lua b/tex/context/base/buff-vis.lua
new file mode 100644
index 000000000..1a97f2591
--- /dev/null
+++ b/tex/context/base/buff-vis.lua
@@ -0,0 +1,74 @@
+if not modules then modules = { } end modules ['buff-vis'] = {
+    version   = 1.001,
+    comment   = "companion to buff-vis.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local type = type
+local format = string.format
+local C, P, V, patterns, lpegmatch = lpeg.C, lpeg.P, lpeg.V, lpeg.patterns, lpeg.match
+
+visualizers = visualizers or { }
+
+local patterns = { }  visualizers.patterns = patterns
+
+local fallback = context.verbatim
+
+function visualizers.pattern(visualizer,kind,pattern)
+    if type(visualizer) == "table" and type(kind) == "string" then
+        kind = visualizer[kind] or visualizer.default or fallback
+    else
+        kind = fallback
+    end
+    return C(pattern)/kind
+end
+
+setmetatable(patterns, {
+    __index = function(t,k)
+        local v = require(format("v-%s.lua",k)) or false
+        context.input(format("v-%s.mkiv",k))
+        t[k] = v
+        return v
+    end
+} )
+
+local function visualizestring(method,content)
+    if content and content ~= "" then
+        lpegmatch(patterns[method],content)
+    end
+end
+
+visualizers.visualizestring = visualizestring
+
+function visualizers.visualizefile(method,name)
+    visualizestring(method,resolvers.loadtexfile(name))
+end
+
+function visualizers.visualizebuffer(method,name)
+    lpegmatch(method,buffers.content(name))
+end
+
+local visualizer = {
+    start   = function() context.startSnippet() end,
+    stop    = function() context.stopSnippet() end ,
+    default = context.verbatim,
+}
+
+local patterns = lpeg.patterns
+local pattern = visualizers.pattern
+
+local texvisualizer = P { "process",
+    process =
+        V("start") * V("content") * V("stop"),
+    start =
+        pattern(visualizer,"start",patterns.beginofstring),
+    stop =
+        pattern(visualizer,"stop",patterns.endofstring),
+    content = (
+        pattern(visualizer,"default",patterns.anything)
+    )^1
+}
+
+return texvisualizer
diff --git a/tex/context/base/buff-vis.mkiv b/tex/context/base/buff-vis.mkiv
new file mode 100644
index 000000000..7739dc21f
--- /dev/null
+++ b/tex/context/base/buff-vis.mkiv
@@ -0,0 +1,24 @@
+%D \module
+%D   [       file=buff-vis,
+%D        version=2010.10.19, % replaces old visualizers as we now have lpeg
+%D          title=\CONTEXT\ Buffer Macros,
+%D       subtitle=Visualizers,
+%D         author=Hans Hagen,
+%D           date=\currentdate,
+%D      copyright=PRAGMA ADE]
+%C
+%C This module is part of the \CONTEXT\ macro||package and is
+%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
+%C details.
+
+\writestatus{loading}{ConTeXt Buffer Macros / Visualizers}
+
+\registerctxluafile{buff-vis}{1.001}
+
+\unprotect
+
+\definestartstop
+    [Snippet]
+    [\c!style=\tt]
+
+\protect \endinput
diff --git a/tex/context/base/cldf-ver.lua b/tex/context/base/cldf-ver.lua
index 12b5479b7..8607cb578 100644
--- a/tex/context/base/cldf-ver.lua
+++ b/tex/context/base/cldf-ver.lua
@@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['cldf-ver'] = {
     license   = "see context related readme files"
 }
 
+-- tex.print == newline
+
 local concat, tohandle = table.concat, table.tohandle
 local splitlines = string.splitlines
 local tostring, type = tostring, type
@@ -55,7 +57,7 @@ end
 function context.tobuffer(name,str)
     context.startbuffer { name }
     context.pushcatcodes("verbatim")
-    local lines = type(str) == "string" and splitlines(str) or str
+    local lines = (type(str) == "string" and find(str,"\n") and splitlines(str)) or str
     for i=1,#lines do
         context(lines[i] .. " ")
     end
diff --git a/tex/context/base/cont-new.tex b/tex/context/base/cont-new.tex
index be89194ef..f024ae024 100644
--- a/tex/context/base/cont-new.tex
+++ b/tex/context/base/cont-new.tex
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2010.11.19 22:50}
+\newcontextversion{2010.11.20 12:51}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index e1a7453b2..c888140b0 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -232,6 +232,7 @@
 
 \loadmarkfile{buff-ini}
 \loadmarkfile{buff-ver}
+\loadmarkfile{buff-vis}
 
 \loadmarkfile{strc-blk}
 
diff --git a/tex/context/base/context.tex b/tex/context/base/context.tex
index 65b0be7aa..e88fbee62 100644
--- a/tex/context/base/context.tex
+++ b/tex/context/base/context.tex
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2010.11.19 22:50}
+\edef\contextversion{2010.11.20 12:51}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua
index df59373ae..ff49101eb 100644
--- a/tex/context/base/l-lpeg.lua
+++ b/tex/context/base/l-lpeg.lua
@@ -21,9 +21,11 @@ local utfgmatch        = unicode and unicode.utf8.gmatch
 
 local anything         = P(1)
 local endofstring      = P(-1)
+local alwaysmatched    = P(true)
 
 patterns.anything      = anything
 patterns.endofstring   = endofstring
+patterns.beginofstring = alwaysmatched
 
 local digit, sign      = R('09'), S('+-')
 local cr, lf, crlf     = P("\r"), P("\n"), P("\r\n")
diff --git a/tex/context/base/mult-cld.lua b/tex/context/base/mult-cld.lua
index 6476eaff6..0542c5fcf 100644
--- a/tex/context/base/mult-cld.lua
+++ b/tex/context/base/mult-cld.lua
@@ -22,9 +22,10 @@ if not modules then modules = { } end modules ['mult-cld'] = {
 context       = context or { }
 local context = context
 
-local format, concat = string.format, table.concat
+local format, find, gmatch, splitlines = string.format, string.find, string.gmatch, string.splitlines
 local next, type, tostring, setmetatable = next, type, tostring, setmetatable
-local insert, remove = table.insert, table.remove
+local insert, remove, concat = table.insert, table.remove, table.concat
+local lpegmatch = lpeg.match
 
 local tex = tex
 
@@ -49,6 +50,8 @@ local flush         = texsprint
 local trace_context = logs.new("context") -- here
 local report_cld    = logs.new("cld")
 
+local processlines  = false  experiments.register("context.processlines", function(v) processlines = v end)
+
 local _stack_, _n_ = { }, 0
 
 local function _store_(ti)
@@ -117,6 +120,49 @@ function tex.fprint(...) -- goodie
     texsprint(currentcatcodes,format(...))
 end
 
+-- -- --
+
+local newline    = lpeg.patterns.newline
+local space      = lpeg.patterns.spacer
+local spacing    = newline * space^0
+local content    = lpeg.C((1-spacing)^1)
+local emptyline  = space^0 * newline^2
+local endofline  = space^0 * newline * space^0
+local simpleline = endofline * lpeg.P(-1)
+
+function lpeg.texlinesplitter(f_content,f_endofline,f_emptyline,f_simpleline)
+    local splitlines =
+        simpleline / (f_simpleline or f_endofline)
+      + (
+            emptyline / f_emptyline
+          + endofline / f_endofline
+          + content   / f_content
+        )^0
+    return function(str) return lpegmatch(splitlines,str) end
+end
+
+local function f_content(s)
+    flush(contentcatcodes,s)
+end
+
+local function f_endofline()
+    texsprint(" ")
+end
+
+local function f_emptyline()
+    texprint("")
+end
+
+local function f_simpleline()
+    texprint("")
+end
+
+local flushlines = lpeg.texlinesplitter(f_content,f_endofline,f_emptyline,f_simpleline)
+
+context.flushlines = flushlines -- maybe context.helpers.flushtexlines
+
+-- -- --
+
 local function writer(command,first,...)
     local t = { first, ... }
     flush(currentcatcodes,command) -- todo: ctx|prt|texcatcodes
@@ -135,15 +181,21 @@ local function writer(command,first,...)
             -- nothing
         elseif ti == "" then
             flush(currentcatcodes,"{}")
-        elseif typ == "string" or typ == "number" then
-            if currentcatcodes == contentcatcodes then
+        elseif typ == "string" then
+            if processlines and find(ti,"\n") then -- we can check for ti == "\n"
+                flush(currentcatcodes,"{")
+                flushlines(ti)
+                flush(currentcatcodes,"}")
+            elseif currentcatcodes == contentcatcodes then
                 flush(currentcatcodes,"{",ti,"}")
             else
                 flush(currentcatcodes,"{")
-             -- maybe if string and \n found then auto split
                 flush(contentcatcodes,ti)
                 flush(currentcatcodes,"}")
             end
+        elseif typ == "number" then
+            -- numbers never have funny catcodes
+            flush(currentcatcodes,"{",ti,"}")
         elseif typ == "table" then
             local tn = #ti
             if tn == 0 then
@@ -185,7 +237,8 @@ local function writer(command,first,...)
             flush(currentcatcodes,"{\\mkivflush{",_store_(ti),"}}") -- todo: ctx|prt|texcatcodes
         elseif typ == "boolean" then
             if ti then
-                flush(ctxcatcodes,"^^M")
+             -- flush(currentcatcodes,"^^M")
+                texprint("")
             else
                 direct = true
             end
@@ -219,14 +272,16 @@ end
 
 local function caller(t,f,a,...)
     if not t then
-        -- so we don't need to test in the calling (slower but often no issue)
+        -- so we don't need to test in the calling (slower but often no issue) (will go)
     elseif f ~= nil then
         local typ = type(f)
         if typ == "string" then
             if a then
-                flush(currentcatcodes,format(f,a,...))
+                flush(contentcatcodes,format(f,a,...)) -- was currentcatcodes
+            elseif processlines and find(f,"\n") then
+                flushlines(f)
             else
-                flush(currentcatcodes,f)
+                flush(contentcatcodes,f)
             end
         elseif typ == "number" then
             if a then
@@ -238,11 +293,21 @@ local function caller(t,f,a,...)
             -- ignored: a ...
             flush(currentcatcodes,"{\\mkivflush{",_store_(f),"}}") -- todo: ctx|prt|texcatcodes
         elseif typ == "boolean" then
-            -- ignored: a ...
             if f then
-                flush(currentcatcodes,"^^M")
-            elseif a ~= nil then
-                writer("",a,...)
+                if a ~= nil then
+                    flushlines(f)
+                    -- ignore ... maybe some day
+                else
+                 -- flush(currentcatcodes,"^^M")
+                    texprint("")
+                end
+            else
+                if a ~= nil then
+                    -- no command, same as context(a,...)
+                    writer("",a,...)
+                else
+                    -- ignored
+                end
             end
         elseif typ == "thread" then
             trace_context("coroutines not supported as we cannot yield across boundaries")
@@ -254,6 +319,8 @@ local function caller(t,f,a,...)
     end
 end
 
+local defaultcaller = caller
+
 setmetatable(context, { __index = indexer, __call = caller } )
 
 -- logging
@@ -396,11 +463,7 @@ function context.egroup()
     context("}")
 end
 
---~ function context.verbatim(...)
---~     flush(vrbcatcodes,...)
---~ end
-
--- context.delayed
+-- context.delayed (todo: lines)
 
 local delayed = { } context.delayed = delayed -- maybe also store them
 
@@ -424,7 +487,7 @@ end
 
 setmetatable(delayed, { __index = indexer, __call = caller } )
 
--- context.nested
+-- context.nested (todo: lines)
 
 local nested = { } context.nested = nested
 
@@ -473,7 +536,10 @@ local function indexer(t,k)
 end
 
 local function caller(t,...)
-    flush(vrbcatcodes,...)
+    local savedcatcodes = contentcatcodes
+    contentcatcodes = vrbcatcodes
+    defaultcaller(t,...)
+    contentcatcodes = savedcatcodes
 end
 
 setmetatable(verbatim, { __index = indexer, __call = caller } )
diff --git a/tex/context/base/tabl-tbl.lua b/tex/context/base/tabl-tbl.lua
index dbb3cfd37..4531b2134 100644
--- a/tex/context/base/tabl-tbl.lua
+++ b/tex/context/base/tabl-tbl.lua
@@ -21,6 +21,8 @@ local nested    = lpeg.patterns.nested
 local pattern   = Ct((separator * (C(nested) + Cc("")) * C((1-separator)^0))^0)
 
 function commands.presettabulate(preamble)
+    -- todo: lpeg
+    preamble = string.escapedpattern(preamble)
     preamble = gsub(preamble, "%*(%b{})(%b{})", function(n,p)
         return rep(sub(p,2,-2),tonumber(sub(n,2,-2)) or 1)
     end)
diff --git a/tex/context/base/tabl-tbl.mkiv b/tex/context/base/tabl-tbl.mkiv
index 0e89fa90f..3cfda9ad7 100644
--- a/tex/context/base/tabl-tbl.mkiv
+++ b/tex/context/base/tabl-tbl.mkiv
@@ -177,7 +177,7 @@
 \newconditional  \tabulatefirstflushed
 \newconditional  \tabulateequal
 \newconditional  \tabulatesplit              \settrue\tabulatesplit
-\newconditional  \ifautotabulate
+\newconditional  \tabulateautomode
 \newconditional  \tabulatehandlepbreak       \settrue\tabulatehandlepbreak
 
 \newdimen        \tabulatepwidth
@@ -381,7 +381,7 @@
         \checktabulatehook   % unexpandable
         \ifzeropt\tabulatewidth
             \ifcase\tabulatemodus\else
-               \noexpand\settrue\noexpand\autotabulate
+               \noexpand\settrue\noexpand\tabulateautomode
             \fi
         \else
             \ifnum\tabulatemodus=\zerocount
@@ -1277,7 +1277,7 @@
      \global\setbox\tablebox\tabulatecolumn\vbox
      \bgroup
      \glet\tabulatehook\notabulatehook
-     \ifconditional\autotabulate\hsize\tabulatewidth\fi
+     \ifconditional\tabulateautomode\hsize\tabulatewidth\fi
      % \begstrut % interferes with pre-\pars
      % evt: \appendtoks\begstrut\to\everypar
      \ignorespaces
@@ -1309,7 +1309,7 @@
 
 \def\tabulatebskipthree
   {\vtop\bgroup
-     \ifconditional\autotabulate\hsize\tabulatewidth\fi
+     \ifconditional\tabulateautomode\hsize\tabulatewidth\fi
      % \begstrut % interferes with pre-\pars
      % evt: \appendtoks\begstrut\to\everypar
      \ignorespaces}
@@ -1507,7 +1507,7 @@
      \tabulatepreamble{\aligntab\flushtabulateindent\strut\alignmark\alignmark\aligntab\alignmark\alignmark\tabskip\zeropoint}%
    \fi
    \tabulatewidth\zeropoint
-   \ctxlua{commands.presettabulate("\detokenize{#1}")}%
+   \ctxlua{commands.presettabulate(\!!bs\detokenize{#1}\!!es)}%
    \edef\totaltabulatecolumns{\the\numexpr3*\tabulatecolumns+4}%
    \tabulatewidth\zeropoint
    \initializetableboxes\tabulatecolumns
diff --git a/tex/context/base/v-tex.lua b/tex/context/base/v-tex.lua
new file mode 100644
index 000000000..0a97d28d6
--- /dev/null
+++ b/tex/context/base/v-tex.lua
@@ -0,0 +1,55 @@
+if not modules then modules = { } end modules ['v-tex'] = {
+    version   = 1.001,
+    comment   = "companion to buff-vis.mkiv",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+local verbatim = context.verbatim
+local P, S, V, patterns = lpeg.P, lpeg.S, lpeg.V, lpeg.patterns
+
+local visualizer = {
+    start    = function() context.startTexSnippet() end,
+    stop     = function() context.stopTexSnippet() end ,
+    name     = function(s) verbatim.TexSnippetName(s) end,
+    group    = function(s) verbatim.TexSnippetGroup(s) end,
+    boundary = function(s) verbatim.TexSnippetBoundary(s) end,
+    special  = function(s) verbatim.TexSnippetSpecial(s) end,
+    comment  = function(s) verbatim.TexSnippetComment(s) end,
+    default  = function(s) verbatim(s) end,
+}
+
+-- todo: unicode letters
+
+local comment    = S("%")
+local restofline = (1-patterns.newline)^0
+local anything   = patterns.anything
+local name       = P("\\") * (patterns.letter + S("@!?"))^1
+local escape     = P("\\") * (anything - patterns.newline)^-1 -- else we get \n
+local group      = S("${}")
+local boundary   = S('[]()<>#="')
+local special    = S("/^_-&+'`|")
+
+local pattern = visualizers.pattern
+
+local texvisualizer = P { "process",
+    process =
+        V("start") * V("content") * V("stop"),
+    start =
+        pattern(visualizer,"start",patterns.beginofstring),
+    stop =
+        pattern(visualizer,"stop",patterns.endofstring),
+    content = (
+        pattern(visualizer,"comment",comment)
+      * pattern(visualizer,"default",restofline)
+      + pattern(visualizer,"name",name)
+      + pattern(visualizer,"name",escape)
+      + pattern(visualizer,"group",group)
+      + pattern(visualizer,"boundary",boundary)
+      + pattern(visualizer,"special",special)
+      + pattern(visualizer,"default",anything)
+    )^1
+}
+
+return texvisualizer
diff --git a/tex/context/base/v-tex.mkiv b/tex/context/base/v-tex.mkiv
new file mode 100644
index 000000000..57aab2272
--- /dev/null
+++ b/tex/context/base/v-tex.mkiv
@@ -0,0 +1,34 @@
+% todo: global
+
+\unprotect
+
+\definestartstop
+    [TexSnippet]
+    [\c!style=\tt]
+
+\definestartstop
+    [TexSnippetName]
+    [\c!color=darkgreen,
+     \c!style=bold]
+
+\definestartstop
+    [TexSnippetGroup]
+    [\c!color=darkred,
+     \c!style=bold]
+
+\definestartstop
+    [TexSnippetBoundary]
+    [\c!color=darkblue,
+     \c!style=bold]
+
+\definestartstop
+    [TexSnippetSpecial]
+    [\c!color=darkyellow,
+     \c!style=bold]
+
+\definestartstop
+    [TexSnippetComment]
+    [\c!color=darkyellow,
+     \c!style=bold]
+
+\protect \endinput
diff --git a/tex/generic/context/luatex-fonts-merged.lua b/tex/generic/context/luatex-fonts-merged.lua
index 83a11409e..0209d16bf 100644
--- a/tex/generic/context/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 11/19/10 22:50:14
+-- merge date  : 11/20/10 12:51:46
 
 do -- begin closure to overcome local limits and interference
 
@@ -154,9 +154,11 @@ local utfgmatch        = unicode and unicode.utf8.gmatch
 
 local anything         = P(1)
 local endofstring      = P(-1)
+local alwaysmatched    = P(true)
 
 patterns.anything      = anything
 patterns.endofstring   = endofstring
+patterns.beginofstring = alwaysmatched
 
 local digit, sign      = R('09'), S('+-')
 local cr, lf, crlf     = P("\r"), P("\n"), P("\r\n")