2014-10-20 19:04:00

14 files changed, 217 insertions, 38 deletions

diff --git a/tex/context/base/back-exp.lua b/tex/context/base/back-exp.lua
index bff673f16..5c8a994d6 100644
--- a/tex/context/base/back-exp.lua
+++ b/tex/context/base/back-exp.lua
@@ -1013,6 +1013,8 @@ do
     end
 
     local function collapse_mn(di,i,data,ndata)
+        -- this is tricky ... we need to make sure that we wrap in mrows if we want
+        -- to bypass this one
         local collapsing = di.data
         if data then
             i = i + 1
@@ -1102,6 +1104,17 @@ do
         end
     end
 
+    local no_mrow = {
+        mrow     = true,
+        mfenced  = true,
+        mfrac    = true,
+        mroot    = true,
+        msqrt    = true,
+        mi       = true,
+        mo       = true,
+        mn       = true,
+    }
+
     local function checkmath(root) -- we can provide utf.toentities as an option
         local data = root.data
         if data then
@@ -1142,10 +1155,23 @@ do
             elseif roottg == "mfenced" then
                 local p = properties[root.fulltag]
                 local l, m, r = p.left, p.middle, p.right
+                if l then
+                    l = utfchar(l)
+                end
+                if m then
+                    local t = { }
+                    for i=1,#m do
+                        t[i] = utfchar(m[i])
+                    end
+                    m = concat(t)
+                end
+                if r then
+                    r = utfchar(r)
+                end
                 root.attributes = {
-                    left   = l and utfchar(l),
-                    middle = m and utfchar(m),
-                    right  = r and utfchar(r),
+                    open       = l,
+                    separators = m,
+                    close      = r,
                 }
             end
             if ndata == 0 then
@@ -1160,9 +1186,7 @@ do
                     local tg = d.tg
                     if automathrows and roottg == "mrow" then
                         -- maybe just always ! check spec first
-                        if tg == "mrow" or tg == "mfenced" or tg == "mfrac" or tg == "mroot" or tg == "msqrt"then
-                            root.skip = "comment"
-                        elseif tg == "mo" then
+                        if no_mrow[tg] then
                             root.skip = "comment"
                         end
                     elseif roottg == "mo" then
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 5a72e9074..daf0b11db 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2014.10.20 10:52}
+\newcontextversion{2014.10.20 19:02}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index ec13349e1..bce447fec 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 3482313cf..324fe4ade 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -28,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2014.10.20 10:52}
+\edef\contextversion{2014.10.20 19:02}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/math-tag.lua b/tex/context/base/math-tag.lua
index 9617fa182..6bdebeb42 100644
--- a/tex/context/base/math-tag.lua
+++ b/tex/context/base/math-tag.lua
@@ -98,28 +98,42 @@ local process
 local function processsubsup(start)
     -- At some point we might need to add an attribute signaling the
     -- super- and subscripts because TeX and MathML use a different
-    -- order.
+    -- order. The mrows are needed to keep mn's separated.
     local nucleus = getfield(start,"nucleus")
     local sup     = getfield(start,"sup")
     local sub     = getfield(start,"sub")
     if sub then
         if sup then
             setattr(start,a_tagged,start_tagged("msubsup"))
+         -- start_tagged("mrow")
             process(nucleus)
+         -- stop_tagged()
+            start_tagged("mrow")
             process(sub)
+            stop_tagged()
+            start_tagged("mrow")
             process(sup)
             stop_tagged()
+            stop_tagged()
         else
             setattr(start,a_tagged,start_tagged("msub"))
+         -- start_tagged("mrow")
             process(nucleus)
+         -- stop_tagged()
+            start_tagged("mrow")
             process(sub)
             stop_tagged()
+            stop_tagged()
         end
     elseif sup then
         setattr(start,a_tagged,start_tagged("msup"))
+     -- start_tagged("mrow")
         process(nucleus)
+     -- stop_tagged()
+        start_tagged("mrow")
         process(sup)
         stop_tagged()
+        stop_tagged()
     else
         process(nucleus)
     end
@@ -405,18 +419,33 @@ process = function(start) -- we cannot use the processor as we have no finalizer
                     setattr(start,a_tagged,start_tagged("mfenced",nil,properties)) -- needs checking
                     if delim then
                         start_tagged("ignore")
-                        properties.left = getfield(delim,"small_char")
+                        local chr = getfield(delim,"small_char")
+                        if chr ~= 0 then
+                            properties.left = chr
+                        end
                         process(delim)
                         stop_tagged()
                     end
+                    start_tagged("mrow") -- begin of subsequence
                 elseif subtype == 2 then
                     -- middle
                     if delim then
                         start_tagged("ignore")
-                        fencesstack[#fencesstack].middle = getfield(delim,"small_char")
+                        local top = fencesstack[#fencesstack]
+                        local chr = getfield(delim,"small_char")
+                        if chr ~= 0 then
+                            local mid = top.middle
+                            if mid then
+                                mid[#mid+1] = chr
+                            else
+                                top.middle = { chr }
+                            end
+                        end
                         process(delim)
                         stop_tagged()
                     end
+                    stop_tagged()        -- end of subsequence
+                    start_tagged("mrow") -- begin of subsequence
                 elseif subtype == 3 then
                     local properties = remove(fencesstack)
                     if not properties then
@@ -425,10 +454,14 @@ process = function(start) -- we cannot use the processor as we have no finalizer
                     end
                     if delim then
                         start_tagged("ignore")
-                        properties.right = getfield(delim,"small_char")
+                        local chr = getfield(delim,"small_char")
+                        if chr ~= 0 then
+                            properties.right = chr
+                        end
                         process(delim)
                         stop_tagged()
                     end
+                    stop_tagged() -- end of subsequence
                     stop_tagged()
                 else
                     -- can't happen
diff --git a/tex/context/base/publ-dat.lua b/tex/context/base/publ-dat.lua
index aa46cd52b..ac89918ee 100644
--- a/tex/context/base/publ-dat.lua
+++ b/tex/context/base/publ-dat.lua
@@ -38,7 +38,7 @@ local setmetatableindex = table.setmetatableindex
 
 -- todo: more allocate
 
-local P, R, S, V, C, Cc, Cs, Ct, Carg = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg
+local P, R, S, V, C, Cc, Cs, Ct, Carg, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.C, lpeg.Cc, lpeg.Cs, lpeg.Ct, lpeg.Carg, lpeg.Cmt
 
 local trace             = false  trackers.register("publications", function(v) trace = v end)
 local report            = logs.reporter("publications")
@@ -281,6 +281,13 @@ local function resolve(s,dataset)
     return dataset.shortcuts[s] or defaultshortcuts[s] or s -- can be number
 end
 
+local function showmessage(s)
+    local t = string.splitlines(utilities.strings.striplines(s))
+    for i=1,#t do
+        report("message: %s",t[i])
+    end
+end
+
 local percent    = P("%")
 local start      = P("@")
 local comma      = P(",")
@@ -315,7 +322,7 @@ local keyword    = C((R("az","AZ","09") + S("@_:-"))^1)
 local key        = C((1-space-equal)^1)
 local tag        = C((1-space-comma)^1)
 local reference  = keyword
-local category   = P("@") * C((1-space-left)^1)
+local category   = C((1-space-left)^1)
 local s_quoted   = ((escape*single) + collapsed + (1-single))^0
 local d_quoted   = ((escape*double) + collapsed + (1-double))^0
 
@@ -331,14 +338,24 @@ local value      = Cs((somevalue * ((spacing * hash * spacing)/"" * somevalue)^0
 local forget     = percent^1 * (1-lineending)^0
 local spacing    = spacing * forget^0 * spacing
 local assignment = spacing * key * spacing * equal * spacing * value * spacing
-local shortcut   = P("@") * (P("string") + P("STRING") + P("String")) * spacing * left * ((assignment * Carg(1))/do_shortcut * comma^0)^0  * spacing * right
 local definition = category * spacing * left * spacing * tag * spacing * comma * Ct((assignment * comma^0)^0) * spacing * right * Carg(1) / do_definition
------ comment    = keyword * spacing * left * (1-right)^0 * spacing * right
-local comment    = P("@") * (P("comment") + P("COMMENT") + P("Comment")) * spacing * lpeg.patterns.nestedbraces
 
--- todo \%
+-- local shortcut   = (P("string")  + P("STRING")  + P("String"))  * spacing * left * ((assignment * Carg(1))/do_shortcut * comma^0)^0  * spacing * right
+-- local comment    = (P("comment") + P("COMMENT") + P("Comment")) * spacing * lpeg.patterns.nestedbraces
+-- local message    = (P("message") + P("MESSAGE") + P("Message")) * spacing * lpeg.patterns.argument / showmessage
+
+-- local bibtotable = (space + forget + P("@") * (shortcut + comment + message + definition) + 1)^0
 
-local bibtotable = (space + forget + shortcut + comment + definition + 1)^0
+local crapword   = C((1-space-left)^1)
+local shortcut   = Cmt(crapword,function(_,p,s) return lower(s) == "string"  and p end) * spacing * left * ((assignment * Carg(1))/do_shortcut * comma^0)^0  * spacing * right
+local comment    = Cmt(crapword,function(_,p,s) return lower(s) == "comment" and p end) * spacing * lpeg.patterns.nestedbraces
+local message    = Cmt(crapword,function(_,p,s) return lower(s) == "message" and p end) * spacing * lpeg.patterns.argument / showmessage
+
+local casecrap   = #S("sScCmM") * (shortcut + comment + message)
+
+local bibtotable = (space + forget + P("@") * (casecrap + definition) + 1)^0
+
+-- todo \%
 
 -- loadbibdata  -> dataset.luadata
 -- loadtexdata  -> dataset.luadata
diff --git a/tex/context/base/publ-tra.lua b/tex/context/base/publ-tra.lua
index 9b0f273c2..7f1626782 100644
--- a/tex/context/base/publ-tra.lua
+++ b/tex/context/base/publ-tra.lua
@@ -110,14 +110,17 @@ function tracers.showdatasetcompleteness(settings)
     local categories = fielddata.categories
     local fieldspecs = fielddata.fields
 
+    local lpegmatch  = lpeg.match
+    local texescape  = lpeg.patterns.texescape
+
     local preamble = { "|lBTw(10em)|p|" }
 
     local function required(foundfields,key,value,indirect)
         ctx_NC() ctx_darkgreen(key)
         ctx_NC() if indirect then
-                ctx_darkblue(value)
+                ctx_darkblue(lpegmatch(texescape,value))
              elseif value then
-                context(value)
+                context(lpegmatch(texescape,value))
              else
                 ctx_darkred("\\tttf [missing]")
              end
@@ -128,9 +131,9 @@ function tracers.showdatasetcompleteness(settings)
     local function optional(foundfields,key,value,indirect)
         ctx_NC() context(key)
         ctx_NC() if indirect then
-                ctx_darkblue(value)
+                ctx_darkblue(lpegmatch(texescape,value))
              elseif value then
-                context(value)
+                context(lpegmatch(texescape,value))
              end
         ctx_NC() ctx_NR()
         foundfields[key] = nil
@@ -148,7 +151,7 @@ function tracers.showdatasetcompleteness(settings)
 
     local function extra(key,value)
         ctx_NC() ctx_llap("+") context(key)
-        ctx_NC() context(value)
+        ctx_NC() context(lpegmatch(texescape,value))
         ctx_NC() ctx_NR()
     end
 
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 1e9f5baad..977cbca7c 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index eec816bf2..4734b9353 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/x-math-svg.lua b/tex/context/base/x-math-svg.lua
index b96c2c63e..263827f92 100644
--- a/tex/context/base/x-math-svg.lua
+++ b/tex/context/base/x-math-svg.lua
@@ -70,6 +70,13 @@ function svgmath.process(filename)
         end
     end
     context(function()
+        -- for tracing purposes:
+        for mathdata, pagenumber in next, blobs do
+            local p  = pagedata[pagenumber]
+            p.mathml = mathdata
+            p.number = pagenumber
+        end
+        --
         savetable(statusname, {
             pagedata = pagedata,
             namedata = namedata,
@@ -133,7 +140,7 @@ function svgmath.convert(filename,svgstyle)
             local mode    = info.mode
             local svgname = addsuffix(f_math_tmp(page),"svg")
             local action  = mode == "inline" and f_inline or f_display
-            local x_div  = xmlfirst(xmlconvert(action(-depth)),"/div")
+            local x_div   = xmlfirst(xmlconvert(action(-depth)),"/div")
             local svgdata = io.loaddata(svgname)
             if not svgdata or svgdata == "" then
                 print("error in:",svgname,tostring(mth))
diff --git a/tex/context/base/x-mathml-basics.mkiv b/tex/context/base/x-mathml-basics.mkiv
index 244b8df80..e166995b0 100644
--- a/tex/context/base/x-mathml-basics.mkiv
+++ b/tex/context/base/x-mathml-basics.mkiv
@@ -6,6 +6,39 @@
 
 % this can become a core helper
 
+% bwe could do all of them in lua
+
+\startluacode
+local find = string.find
+local lpegmatch = lpeg.match
+
+local splitter = lpeg.Ct(lpeg.C(lpeg.patterns.nestedbraces + lpeg.patterns.utf8character)^1)
+
+function commands.xmfenced(left,middle,right,content)
+    local l = left   ~= "" and left   or "("
+    local r = right  ~= "" and right  or ")"
+    local m = middle ~= "" and middle and lpegmatch(splitter,middle) or { "," }
+    local c = find(content,"{") and lpegmatch(splitter,content) or { content }
+    local n = #c
+    if n > 1 then
+        context("\\left%s",l)
+        for i=1,n do
+            if i > 1 then
+                context("%s %s",m[i] or m[#m],c[i])
+            else
+                context(c[i])
+            end
+        end
+        context("\\right%s",r)
+    else
+        context("\\left%s %s \\right%s",l,content,r)
+    end
+end
+
+\stopluacode
+
+\unprotect
+
 \unexpanded\def\mexecuteifdefined#1%
   {\ifx#1\empty
      \expandafter\secondoftwoarguments
@@ -83,7 +116,6 @@
   % \mathtriplet {\mathstylehbox{#1}} {#2} {}
     \mathtriplet {\mathematics{#1}} {#2} {}
 \stoptexdefinition
-\stoptexdefinition
 \starttexdefinition unexpanded moveraccentchecker #1#2
     \edef\movertoken{\tochar{#2}}
     \doifelseutfmathabove\movertoken \moveraccent \movertext {#1}{#2}
@@ -135,10 +167,49 @@
 
 % mfenced
 
+% \mfenced{x,y}
+% \mfenced{{x}{y}}
+% \mfenced[separators]{{x}{y}}
+% \mfenced[left][right]{{x}{y}}
+% \mfenced[left][separators][right]{{x}{y}}
+
+\starttexdefinition unexpanded mfenced
+    \dotripleempty\do_mfenced
+\stoptexdefinition
+
+\starttexdefinition unexpanded do_mfenced [#1][#2][#3]#4
+    \mathematics {
+        \ctxcommand{xmfenced(
+            \ifthirdargument  "#1","#2","#3"\else
+            \ifsecondargument  "#1",",","#2"\else
+            \iffirstargument    "(","#1",")"\else
+                                 "(",",",")"\fi\fi\fi
+        ,"#4")}
+    }
+\stoptexdefinition
+
 % mfrac
 
+\starttexdefinition unexpanded mfrac #1#2
+    \mathematics {
+        \frac{#1}{#2}
+    }
+\stoptexdefinition
+
 % mroot msqrt
 
+\starttexdefinition unexpanded mroot #1#2
+    \mathematics {
+        \sqrt[#1]{#2}
+    }
+\stoptexdefinition
+
+\starttexdefinition unexpanded msqrt #1
+    \mathematics {
+        \sqrt{#1}
+    }
+\stoptexdefinition
+
 % menclose
 
 % merror
@@ -161,10 +232,16 @@
 
 % semantics
 
+\protect
+
 \continueifinputfile{x-mathml-basics.mkiv}
 
 \starttext
 
+$\mfenced{1+a}$\par
+$\mfenced[,]{1+a}$\par
+$\mfenced[,]{{1+a}{1+b}}$\par
+
 % $\mover{←}{test}$\par
 % $\mover{\utfchar{"2190}}{test}$\par
 % $\mover{e:leftarrow}{test}$\par
@@ -176,7 +253,7 @@
 % $\mover{test}{x:23DE}$\par
 % $\mover{test}{over}$\par
 
-\mover{test}{⏞}\par
+% \mover{test}{⏞}\par
 % \mover{test}{\utfchar{"23DE}}\par
 % \mover{test}{e:overbrace}\par
 % \mover{test}{x:23DE}\par
diff --git a/tex/context/bib/bibl-apa-it.tex b/tex/context/bib/bibl-apa-it.tex
index 3892dcb74..ebf03313a 100644
--- a/tex/context/bib/bibl-apa-it.tex
+++ b/tex/context/bib/bibl-apa-it.tex
@@ -160,8 +160,7 @@
 \newif\ifeditedbook
 
 \setuppublicationlayout[book]{%
-   \insertauthors{}{ }{\inserteditors{}{, A c. di%
-                         \ifnum\getvalue{editor@num}> 1 s\fi
+   \insertauthors{}{ }{\inserteditors{}{, a c. di%
                              \ \global\editedbooktrue
                      }{\insertthekey{}{. }{}}}%
    \insertpubyear{(}{). }{\unskip.}%
@@ -181,13 +180,13 @@
        \insertcrossref
         {\insertchap{, }{}{}%
          \insertpages{\unskip, pp. }{. }{. }%
-         \insertvolume{Vol.~}{ of~}{}%
+         \insertvolume{Vol.~}{ di~}{}%
          }%
         {}%
         {\insertvolume
                    {, vol.~}%
                    {\insertseries
-                      { of~\bgroup\it}%
+                      { di~\bgroup\it}%
                       {\egroup}
                       {}}
                    {}%
@@ -203,7 +202,7 @@
 }
 
 \setuppublicationlayout[inbook]{%
-   \insertauthors{}{ }{\inserteditors{}{, A c. di%
+   \insertauthors{}{ }{\inserteditors{}{, a c. di%
                              \ \global\editedbooktrue
                      }{\insertthekey{}{. }{}}}%
    \insertpubyear{(}{). }{\unskip.}%
@@ -223,13 +222,13 @@
         \insertcrossref
           {\insertchap{, }{}{}%
            \insertpages{\unskip, pp. }{. }{. }%
-           \insertvolume{Volume~}{ of~}{}%
+           \insertvolume{Volume~}{ di~}{}%
            }%
           {}%
           {\insertvolume
               {, volume~}%
               {\insertseries
-                 { of~\bgroup\it}%
+                 { di~\bgroup\it}%
                  {\egroup}
                  {}}
               {}%
@@ -269,7 +268,7 @@
    \insertarttitle{\bgroup }{\egroup. }{}%
    \inserttitle
     {In \inserteditors{}%
-        {, A c. di, }%
+        {, a c. di, }%
         {}%
     \bgroup\it}%
     {\egroup
@@ -295,7 +294,7 @@
    \insertarttitle{\bgroup }{\egroup. }{}%%
    \inserttitle
     {In \inserteditors{}%
-        {, A c. di, }%
+        {, a c. di, }%
         {}%
     \bgroup\it}%
     {\egroup
@@ -315,7 +314,7 @@
 }
 
 \setuppublicationlayout[proceedings]{%
-   \inserteditors{}{, A c. di%
+   \inserteditors{}{, a c. di%
                              \ \global\editedbooktrue
                      }{\insertthekey{}{ }{}}%
    \insertpubyear{(}{). }{}%
diff --git a/tex/context/sample/darwin.tex b/tex/context/sample/darwin.tex
new file mode 100644
index 000000000..6425bf156
--- /dev/null
+++ b/tex/context/sample/darwin.tex
@@ -0,0 +1,19 @@
+It is interesting to contemplate an entangled bank, clothed with many
+plants of many kinds, with birds singing on the bushes, with various
+insects flitting about, and with worms crawling through the damp earth,
+and to reflect that these elaborately constructed forms, so different
+from each other, and dependent on each other in so complex a manner,
+have all been produced by laws acting around us. These laws, taken in
+the largest sense, being Growth with Reproduction; Inheritance which is
+almost implied by reproduction; Variability from the indirect and
+direct action of the external conditions of life, and from use and
+disuse; a Ratio of Increase so high as to lead to a Struggle for Life,
+and as a consequence to Natural Selection, entailing Divergence of
+Character and the Extinction of less-improved forms. Thus, from the war
+of nature, from famine and death, the most exalted object which we are
+capable of conceiving, namely, the production of the higher animals,
+directly follows. There is grandeur in this view of life, with its
+several powers, having been originally breathed into a few forms or
+into one; and that, whilst this planet has gone cycling on according to
+the fixed law of gravity, from so simple a beginning endless forms most
+beautiful and most wonderful have been, and are being, evolved.
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 68400c088..3304530a0 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 10/20/14 10:52:56
+-- merge date  : 10/20/14 19:02:32
 
 do -- begin closure to overcome local limits and interference