20 files changed, 501 insertions, 201 deletions
diff --git a/tex/context/base/char-def.lua b/tex/context/base/char-def.lua
index f30e82898..84420ae9d 100644
--- a/tex/context/base/char-def.lua
+++ b/tex/context/base/char-def.lua
@@ -310,9 +310,12 @@ characters.data={
   adobename="numbersign",
   category="po",
   cjkwd="na",
+  contextname="texthash",
   description="NUMBER SIGN",
   direction="et",
   linebreak="al",
+  mathclass="binary",
+  mathname="mathhash",
   unicodeslot=0x23,
   variants={
    [0xFE0E]="text style",
@@ -327,25 +330,32 @@ characters.data={
   description="DOLLAR SIGN",
   direction="et",
   linebreak="pr",
+  mathclass="binary",
+  mathname="mathdollar",
   unicodeslot=0x24,
  },
  {
   adobename="percent",
   category="po",
   cjkwd="na",
-  contextname="percent",
+  contextname="textpercent",
   description="PERCENT SIGN",
   direction="et",
   linebreak="po",
+  mathclass="binary",
+  mathname="mathpercent",
   unicodeslot=0x25,
  },
  {
   adobename="ampersand",
   category="po",
   cjkwd="na",
+  contextname="textampersand",
   description="AMPERSAND",
   direction="on",
   linebreak="al",
+  mathclass="binary",
+  mathname="mathampersand",
   unicodeslot=0x26,
  },
  {
@@ -4606,6 +4616,8 @@ characters.data={
   description="LATIN SMALL LETTER LAMBDA WITH STROKE",
   direction="l",
   linebreak="al",
+  mathclass="variable",
+  mathname="lambdabar",
   unicodeslot=0x19B,
  },
  {
diff --git a/tex/context/base/char-ini.lua b/tex/context/base/char-ini.lua
index e5806622c..f7b5f8871 100644
--- a/tex/context/base/char-ini.lua
+++ b/tex/context/base/char-ini.lua
@@ -711,10 +711,10 @@ of the official <l n='api'/>.</p>
 
 -- we could make them virtual: characters.contextnames[n]
 
-function characters.contextname(n) return data[n].contextname or "" end
-function characters.adobename  (n) return data[n].adobename   or "" end
-function characters.description(n) return data[n].description or "" end
--------- characters.category   (n) return data[n].category    or "" end
+function characters.contextname(n) return data[n] and data[n].contextname or "" end
+function characters.adobename  (n) return data[n] and data[n].adobename   or "" end
+function characters.description(n) return data[n] and data[n].description or "" end
+-------- characters.category   (n) return data[n] and data[n].category    or "" end
 
 function characters.category(n,verbose)
     local c = data[n].category
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 592ea2eca..84642185f 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2014.07.30 10:31}
+\newcontextversion{2014.08.19 11:57}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 10780070a..caf2c1abc 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 9564d6135..01bb35232 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -28,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2014.07.30 10:31}
+\edef\contextversion{2014.08.19 11:57}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/enco-ini.mkiv b/tex/context/base/enco-ini.mkiv
index 0fae1aade..07bdaebd9 100644
--- a/tex/context/base/enco-ini.mkiv
+++ b/tex/context/base/enco-ini.mkiv
@@ -208,11 +208,42 @@
 \def\pound     {£} \def\sterling{£}
 \def\promille  {‰} \def\permille{‰}
 
-% tex specific (obsolete)
+% tex specific
+
+\ifdefined\textpercent
+    \let\percent   \textpercent
+    \let\procent   \textpercent
+    \let\ampersand \textampersand
+    \let\dollar    \textdollar
+    \let\hash      \texthash
+\else
+    \def\percent   {\textpercent}
+    \def\procent   {\textpercent}
+    \def\ampersand {\textampersand}
+    \def\dollar    {\textdollar}
+    \def\hash      {\texthash}
+\fi
 
-\def\procent   {\percent}
-\def\dollar    {\textdollar}
-\def\permine   {\fakepermine}
+% from enco-mis:
+
+\unexpanded\def\fakepercent
+  {\mathematics{\normalsuperscript{\scriptscriptstyle0}\kern-.25\emwidth/\kern-.2\emwidth\normalsubscript{\scriptscriptstyle0}}}
+
+\unexpanded\def\fakeperthousand
+  {\mathematics{\normalsuperscript{\scriptscriptstyle0}\kern-.25\emwidth/\kern-.2\emwidth\normalsubscript{\scriptscriptstyle00}}}
+
+\unexpanded\def\fakepermine
+  {\dontleavehmode
+   \bgroup
+   \setbox\scratchbox\hbox
+     {\mathematics{+}}%
+   \hbox to \wd\scratchbox
+     {\hss
+      \mathematics{\normalsuperscript{\scriptscriptstyle-}\kern-.4\emwidth/\kern-.3\emwidth\normalsubscript{\scriptscriptstyle-}}%
+      \hss}%
+   \egroup}
+
+\def\permine{\fakepermine}
 
 % some more
 
@@ -331,25 +362,6 @@
 
 \ctxlua{characters.tex.defineaccents()}
 
-% from enco-mis:
-
-\unexpanded\def\fakepercent
-  {\mathematics{\normalsuperscript{\scriptscriptstyle0}\kern-.25\emwidth/\kern-.2\emwidth\normalsubscript{\scriptscriptstyle0}}}
-
-\unexpanded\def\fakeperthousand
-  {\mathematics{\normalsuperscript{\scriptscriptstyle0}\kern-.25\emwidth/\kern-.2\emwidth\normalsubscript{\scriptscriptstyle00}}}
-
-\unexpanded\def\fakepermine
-  {\dontleavehmode
-   \bgroup
-   \setbox\scratchbox\hbox
-     {\mathematics{+}}%
-   \hbox to \wd\scratchbox
-     {\hss
-      \mathematics{\normalsuperscript{\scriptscriptstyle-}\kern-.4\emwidth/\kern-.3\emwidth\normalsubscript{\scriptscriptstyle-}}%
-      \hss}%
-   \egroup}
-
 %D A smaller and bolder variant, more like the math and monospaced ones.
 
 \unexpanded\def\fakeunderscore
diff --git a/tex/context/base/font-ctx.lua b/tex/context/base/font-ctx.lua
index 2f5ae57e2..e366f746d 100644
--- a/tex/context/base/font-ctx.lua
+++ b/tex/context/base/font-ctx.lua
@@ -78,12 +78,11 @@ local texdefinefont       = tex.definefont
 local texsp               = tex.sp
 
 local fontdata            = hashes.identifiers
-local characters          = hashes.chardata
+local characters          = hashes.characters
 local descriptions        = hashes.descriptions
 local properties          = hashes.properties
 local resources           = hashes.resources
 local csnames             = hashes.csnames
-local marks               = hashes.markdata
 local lastmathids         = hashes.lastmathids
 local exheights           = hashes.exheights
 local emwidths            = hashes.emwidths
@@ -1449,7 +1448,28 @@ local function nametoslot(name)
     end
 end
 
-helpers.nametoslot = nametoslot
+local function indextoslot(index)
+    local r = resources[true]
+    if r then
+        local indices = r.indices
+        if not indices then
+            indices = { }
+            local c = characters[true]
+            for unicode, data in next, c do
+                local di = data.index
+                if di then
+                    indices[di] = unicode
+                end
+            end
+            r.indices = indices
+        end
+        return indices[tonumber(index)]
+    end
+end
+
+
+helpers.nametoslot  = nametoslot
+helpers.indextoslot = indextoslot
 
 -- this will change ...
 
@@ -1604,6 +1624,13 @@ function commands.fontchar(n)
     end
 end
 
+function commands.fontcharbyindex(n)
+    n = indextoslot(n)
+    if n then
+        context_char(n)
+    end
+end
+
 function commands.doifelsecurrentfonthasfeature(name) -- can be made faster with a supportedfeatures hash
     local f = fontdata[currentfont()]
     f = f and f.shared
diff --git a/tex/context/base/font-ini.mkvi b/tex/context/base/font-ini.mkvi
index c1e6d9390..556816f6d 100644
--- a/tex/context/base/font-ini.mkvi
+++ b/tex/context/base/font-ini.mkvi
@@ -2172,6 +2172,9 @@
 \unexpanded\def\fontchar#character%
   {\ctxcommand{fontchar("#character")}}
 
+\unexpanded\def\fontcharbyindex#index% unofficial command, for idris' font building
+  {\ctxcommand{fontcharbyindex(\number#index)}}
+
 %D The next auxilliary macro is an alternative to \type
 %D {\fontname}.
 
diff --git a/tex/context/base/font-nod.lua b/tex/context/base/font-nod.lua
index 8251dc588..2311cebeb 100644
--- a/tex/context/base/font-nod.lua
+++ b/tex/context/base/font-nod.lua
@@ -103,7 +103,7 @@ function char_tracers.collect(head,list,tag,n)
                 ok, fn = false, f
             end
             local c = getchar(head)
-            local i = fontidentifiers[f].indices[c] or 0
+         -- local i = fontidentifiers[f].indices[c] or 0 -- zero anyway as indices is nilled
             if not ok then
                 ok = true
                 n = n + 1
@@ -111,7 +111,8 @@ function char_tracers.collect(head,list,tag,n)
                 list[n][tag] = { }
             end
             local l = list[n][tag]
-            l[#l+1] = { c, f, i }
+         -- l[#l+1] = { c, f, i }
+            l[#l+1] = { c, f }
         elseif id == disc_code then
             -- skip
         else
@@ -127,7 +128,8 @@ function char_tracers.equal(ta, tb)
     else
         for i=1,#ta do
             local a, b = ta[i], tb[i]
-            if a[1] ~= b[1] or a[2] ~= b[2] or a[3] ~= b[3] then
+         -- if a[1] ~= b[1] or a[2] ~= b[2] or a[3] ~= b[3] then
+            if a[1] ~= b[1] or a[2] ~= b[2] then
                 return false
             end
         end
@@ -160,20 +162,20 @@ function char_tracers.unicodes(t,decimal)
     return concat(tt," ")
 end
 
-function char_tracers.indices(t,decimal)
-    local tt = { }
-    for i=1,#t do
-        local n = t[i][3]
-        if n == 0 then
-            tt[i] = "-"
-        elseif decimal then
-            tt[i] = n
-        else
-            tt[i] = f_unicode(n)
-        end
-    end
-    return concat(tt," ")
-end
+-- function char_tracers.indices(t,decimal)
+--     local tt = { }
+--     for i=1,#t do
+--         local n = t[i][3]
+--         if n == 0 then
+--             tt[i] = "-"
+--         elseif decimal then
+--             tt[i] = n
+--         else
+--             tt[i] = f_unicode(n)
+--         end
+--     end
+--     return concat(tt," ")
+-- end
 
 function char_tracers.start()
     local npc = handlers.characters -- should accept nuts too
diff --git a/tex/context/base/math-act.lua b/tex/context/base/math-act.lua
index 879480dce..e767903a2 100644
--- a/tex/context/base/math-act.lua
+++ b/tex/context/base/math-act.lua
@@ -103,15 +103,23 @@ function mathematics.checkprivateparameters(target,original)
     local mathparameters = target.mathparameters
     if mathparameters then
         local parameters = target.parameters
+        local properties = target.properties
         if parameters then
-            if not mathparameters.FractionDelimiterSize then
-                mathparameters.FractionDelimiterSize = 1.01 * parameters.size
-            end
-            if not mathparameters.FractionDelimiterDisplayStyleSize then
-                mathparameters.FractionDelimiterDisplayStyleSize = 2.40 * parameters.size
+            local size = parameters.size
+            if size then
+                if not mathparameters.FractionDelimiterSize then
+                    mathparameters.FractionDelimiterSize = 1.01 * size
+                end
+                if not mathparameters.FractionDelimiterDisplayStyleSize then
+                    mathparameters.FractionDelimiterDisplayStyleSize = 2.40 * size
+                end
+            elseif properties then
+                report_math("invalid parameters in font %a",properties.fullname or "?")
+            else
+                report_math("invalid parameters in font")
             end
-        elseif target.properties then
-            report_math("no parameters in font %a",target.properties.fullname or "?")
+        elseif properties then
+            report_math("no parameters in font %a",properties.fullname or "?")
         else
             report_math("no parameters and properties in font")
         end
diff --git a/tex/context/base/math-ini.mkiv b/tex/context/base/math-ini.mkiv
index caa531c1d..a2f481df2 100644
--- a/tex/context/base/math-ini.mkiv
+++ b/tex/context/base/math-ini.mkiv
@@ -26,6 +26,13 @@
 %D restore a changed mathstyle so best avoid that one. However, there are cases where
 %D we really need to use such grouping.
 
+% Weird, these fail, maybe amp is solved in a later state from char noads (needs a
+% fix in luatex):
+%
+% $\char"26$
+% $\a$
+% $\string&$
+
 % mathop applied to characters centers it vertically
 
 \unprotect
diff --git a/tex/context/base/math-vfu.lua b/tex/context/base/math-vfu.lua
index 6d9a9f903..a683e02cf 100644
--- a/tex/context/base/math-vfu.lua
+++ b/tex/context/base/math-vfu.lua
@@ -41,6 +41,8 @@ local allocate          = utilities.storage.allocate
 local setmetatableindex = table.setmetatableindex
 local formatters        = string.formatters
 
+local chardata          = characters.data
+
 local mathencodings     = allocate()
 fonts.encodings.math    = mathencodings -- better is then: fonts.encodings.vectors
 local vfmath            = allocate()
@@ -432,6 +434,14 @@ local function repeated(main,characters,id,size,unicode,u,n,private,fraction) --
     end
 end
 
+local function cloned(main,characters,id,size,source,target)
+    local data = characters[source]
+    if data then
+        characters[target] = data
+        return true
+    end
+end
+
 -- we use the fact that context defines the smallest sizes first .. a real dirty and ugly hack
 
 local data_of_smaller = nil
@@ -451,10 +461,11 @@ function vfmath.addmissing(main,id,size)
 
     -- here id is the index in fonts (normally 14 or so) and that slot points to self
 
-    local characters = main.characters
-    local shared = main.shared
-    local variables = main.goodies.mathematics and main.goodies.mathematics.variables or { }
+    local characters    = main.characters
+    local shared        = main.shared
+    local variables     = main.goodies.mathematics and main.goodies.mathematics.variables or { }
     local joinrelfactor = variables.joinrelfactor or 3
+
     for i=0x7A,0x7D do
         make(main,characters,id,size,i,1)
     end
@@ -536,23 +547,24 @@ function vfmath.addmissing(main,id,size)
     -- 21CB leftrightharpoon
     -- 21CC rightleftharpoon
 
-    stack    (main,characters,id,size,0x2259,0x0003D,3,0x02227)                       -- \buildrel\wedge\over=
-    jointwo  (main,characters,id,size,0x22C8,0x022B3,joinrelfactor,0x022B2)           -- \mathrel\triangleright\joinrel\mathrel\triangleleft (4 looks better than 3)
-    jointwo  (main,characters,id,size,0x22A7,0x0007C,joinrelfactor,0x0003D)           -- \mathrel|\joinrel=
-    jointwo  (main,characters,id,size,0x2260,0x00338,0,0x0003D)                       -- \not\equal
-    jointwo  (main,characters,id,size,0x2284,0x00338,0,0x02282)                       -- \not\subset
-    jointwo  (main,characters,id,size,0x2285,0x00338,0,0x02283)                       -- \not\supset
-    jointwo  (main,characters,id,size,0x2209,0x00338,0,0x02208)                       -- \not\in
-    jointwo  (main,characters,id,size,0x2254,0x03A,0,0x03D)                           -- := (≔)
+    stack(main,characters,id,size,0x2259,0x0003D,3,0x02227)                       -- \buildrel\wedge\over=
+
+    jointwo(main,characters,id,size,0x22C8,0x022B3,joinrelfactor,0x022B2)           -- \mathrel\triangleright\joinrel\mathrel\triangleleft (4 looks better than 3)
+    jointwo(main,characters,id,size,0x22A7,0x0007C,joinrelfactor,0x0003D)           -- \mathrel|\joinrel=
+    jointwo(main,characters,id,size,0x2260,0x00338,0,0x0003D)                       -- \not\equal
+    jointwo(main,characters,id,size,0x2284,0x00338,0,0x02282)                       -- \not\subset
+    jointwo(main,characters,id,size,0x2285,0x00338,0,0x02283)                       -- \not\supset
+    jointwo(main,characters,id,size,0x2209,0x00338,0,0x02208)                       -- \not\in
+    jointwo(main,characters,id,size,0x2254,0x03A,0,0x03D)                           -- := (≔)
 
     repeated(main,characters,id,size,0x222C,0x222B,2,0xFF800,1/3)
     repeated(main,characters,id,size,0x222D,0x222B,3,0xFF810,1/3)
 
-    characters[0xFE325] = fastcopy(characters[0x2032])
-
-    raise    (main,characters,id,size,0x02032,0xFE325,1,id_of_smaller) -- prime
-    raise    (main,characters,id,size,0x02033,0xFE325,2,id_of_smaller) -- double prime
-    raise    (main,characters,id,size,0x02034,0xFE325,3,id_of_smaller) -- triple prime
+    if cloned(main,characters,id,size,0x2032,0xFE325) then
+        raise(main,characters,id,size,0x2032,0xFE325,1,id_of_smaller) -- prime
+        raise(main,characters,id,size,0x2033,0xFE325,2,id_of_smaller) -- double prime
+        raise(main,characters,id,size,0x2034,0xFE325,3,id_of_smaller) -- triple prime
+    end
 
     -- there are more (needs discussion first):
 
@@ -882,7 +894,7 @@ function vfmath.define(specification,set,goodies)
                                 local ru = rv[unicode]
                                 if not ru then
                                     if trace_virtual then
-                                        report_virtual("unicode slot %U has no index %H in vector %a for font %a",unicode,index,vectorname,fontname)
+                                        report_virtual("unicode slot %U has no index %H in vector %a for font %a (%S)",unicode,index,vectorname,fontname,chardata[unicode].description)
                                     elseif not already_reported then
                                         report_virtual("the mapping is incomplete for %a at %p",name,size)
                                         already_reported = true
diff --git a/tex/context/base/mult-low.lua b/tex/context/base/mult-low.lua
index 3728b0433..21f7c53cc 100644
--- a/tex/context/base/mult-low.lua
+++ b/tex/context/base/mult-low.lua
@@ -13,7 +13,7 @@ return {
         --
         "zerocount", "minusone", "minustwo", "plusone", "plustwo", "plusthree", "plusfour", "plusfive",
         "plussix", "plusseven", "pluseight", "plusnine", "plusten", "plussixteen", "plushundred",
-        "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard",
+        "plusthousand", "plustenthousand", "plustwentythousand", "medcard", "maxcard", "maxcardminusone",
         "zeropoint", "onepoint", "halfapoint", "onebasepoint", "maxdimen", "scaledpoint", "thousandpoint",
         "points", "halfpoint",
         "zeroskip",
diff --git a/tex/context/base/s-fnt-10.mkiv b/tex/context/base/s-fnt-10.mkiv
index 9b6211c2b..2fe82e079 100644
--- a/tex/context/base/s-fnt-10.mkiv
+++ b/tex/context/base/s-fnt-10.mkiv
@@ -100,7 +100,7 @@ function fonts.tracers.show_glyphs()
 end
 \stopluacode
 
-\def\ShowCompleteFont#1#2#3%
+\unexpanded\def\ShowCompleteFont#1#2#3%
   {\bgroup
    \page
    \font\TestFont=#1 at #2
@@ -118,7 +118,7 @@ end
    \page
    \egroup}
 
-\def\ShowAllGlyphs#1#2#3%
+\unexpanded\def\ShowAllGlyphs#1#2#3%
   {\bgroup
    \page
    \def\showglyph##1{\dontleavehmode\strut\char##1\relax\par}
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index faa6c8b27..3969f8073 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 76bbc72b8..7d88c08a4 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/syst-ini.mkiv b/tex/context/base/syst-ini.mkiv
index f1aeb094d..ff74efecc 100644
--- a/tex/context/base/syst-ini.mkiv
+++ b/tex/context/base/syst-ini.mkiv
@@ -417,6 +417,7 @@
 \chardef    \plustwentythousand = 20000
 \chardef    \medcard            = 32768
 \chardef    \maxcard            = 65536 % pdftex has less mathchars
+\chardef    \maxcardminusone    = 65535
 
 %D \macros
 %D   {doubleexpandafter,tripleexpandafter,expanded,startexpanded}
diff --git a/tex/context/base/typo-tal.lua b/tex/context/base/typo-tal.lua
index 5d62d4e47..1e9c815b3 100644
--- a/tex/context/base/typo-tal.lua
+++ b/tex/context/base/typo-tal.lua
@@ -112,8 +112,8 @@ function characteralign.handler(originalhead,where)
     if not a or a == 0 then
         return originalhead, false
     end
-    local column    = div(a,100)
-    local row       = a % 100
+    local column    = div(a,0xFFFF)
+    local row       = a % 0xFFFF
     local dataset   = datasets and datasets[column] or setcharacteralign(column)
     local separator = dataset.separator
     local list      = dataset.list
diff --git a/tex/context/base/typo-tal.mkiv b/tex/context/base/typo-tal.mkiv
index 32b99ff01..91c22332b 100644
--- a/tex/context/base/typo-tal.mkiv
+++ b/tex/context/base/typo-tal.mkiv
@@ -56,7 +56,7 @@
 % D
 % D \typebuffer \blank \getbuffer \blank
 
-\unexpanded\def\signalcharacteralign#1#2{\attribute\characteralignattribute=\numexpr#1*\plushundred+#2\relax}
+\unexpanded\def\signalcharacteralign#1#2{\attribute\characteralignattribute\numexpr#1*\maxcardminusone+#2\relax} % 0xFFFF
 \unexpanded\def\setcharacteralign   #1#2{\ctxcommand{setcharacteralign(\number#1,"#2")}}
 \unexpanded\def\resetcharacteralign     {\ctxcommand{resetcharacteralign()}}
 \unexpanded\def\nocharacteralign        {\attribute\characteralignattribute\attributeunsetvalue}
diff --git a/tex/context/base/x-asciimath.lua b/tex/context/base/x-asciimath.lua
index 00ffd4cc0..60fbb0b5a 100644
--- a/tex/context/base/x-asciimath.lua
+++ b/tex/context/base/x-asciimath.lua
@@ -19,12 +19,14 @@ ugly and unsatisfying code mess down here. Don't take this as an example.</p>
 -- todo: spaces around all elements in cleanup?
 -- todo: filter from files listed in tuc file
 
-local trace_mapping = false  if trackers then trackers.register("modules.asciimath.mapping", function(v) trace_mapping = v end) end
-local trace_detail  = false  if trackers then trackers.register("modules.asciimath.detail",  function(v) trace_detail  = v end) end
+local trace_mapping    = false  if trackers then trackers.register("modules.asciimath.mapping", function(v) trace_mapping = v end) end
+local trace_detail     = false  if trackers then trackers.register("modules.asciimath.detail",  function(v) trace_detail  = v end) end
 
-local asciimath      = { }
-local moduledata     = moduledata or { }
-moduledata.asciimath = asciimath
+local report_asciimath = logs.reporter("mathematics","asciimath")
+
+local asciimath        = { }
+local moduledata       = moduledata or { }
+moduledata.asciimath   = asciimath
 
 if not characters then
     require("char-def")
@@ -32,16 +34,18 @@ if not characters then
     require("char-ent")
 end
 
-local entities = characters.entities or { }
-
-local report_asciimath = logs.reporter("mathematics","asciimath")
-
 local type, rawget = type, rawget
+local concat, insert, remove = table.concat, table.insert, table.remove
+local rep, gmatch, gsub, find = string.rep, string.gmatch, string.gsub, string.find
+
 local lpegmatch, patterns = lpeg.match, lpeg.patterns
 local S, P, R, C, V, Cc, Ct, Cs = lpeg.S, lpeg.P, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Ct, lpeg.Cs
-local concat, remove, sortedhash, sortedkeys, keys = table.concat, table.remove, table.sortedhash, table.sortedkeys, table.keys
-local rep, gmatch, gsub, find = string.rep, string.gmatch, string.gsub, string.find
-local formatters = string.formatters
+
+local sortedhash   = table.sortedhash
+local sortedkeys   = table.sortedkeys
+local formatters   = string.formatters
+
+local entities     = characters.entities or { }
 
 local xmltext      = xml.text
 local xmlinclusion = xml.inclusion
@@ -147,23 +151,32 @@ local reserved = {
 
     -- brackets
 
---     ["("]  = "(,
---     [")"]  = "),
---     ["["]  = "[,
---     ["]"]  = "],
---     ["{"]  = "{,
---     ["}"]  = "},
---     ["(:"] = "〈",
---     [":)"] = "〉",
+ -- ["("]  = "(,
+ -- [")"]  = "),
+ -- ["["]  = "[,
+ -- ["]"]  = "],
+ -- ["{"]  = "{,
+ -- ["}"]  = "},
+ -- ["(:"] = "〈",
+ -- [":)"] = "〉",
 
     -- binary relations
 
     ["="]    = "=",
+    ["eq"]   = "=",
     ["!="]   = "≠",
+    ["ne"]   = "≠",
+    ["neq"]  = "≠",
     ["<"]    = "<",
+    ["lt"]   = "<",
     [">"]    = ">",
+    ["gt"]   = ">",
     ["<="]   = "≤",
+    ["le"]   = "≤",
+    ["leq"]  = "≤",
     [">="]   = "≥",
+    ["ge"]   = "≥",
+    ["geq"]  = "≥",
     ["-<"]   = "≺",
     [">-"]   = "≻",
     ["in"]   = "∈",
@@ -216,8 +229,8 @@ local reserved = {
     ["angle"]   = "∠",
     ["/_"]      = "∠",
     [":."]      = "∴",
-    ["..."]     = "...",               -- ldots
-    ["ldots"]   = "...",               -- ldots
+    ["..."]     = "...", -- ldots
+    ["ldots"]   = "...", -- ldots
     ["cdots"]   = "⋯",
     ["vdots"]   = "⋮",
     ["ddots"]   = "⋱",
@@ -228,6 +241,13 @@ local reserved = {
     ["|~"]      = "⌈",
     ["~|"]      = "⌉",
 
+    -- special
+
+    ["%"] = "\\mathpercent",
+    ["&"] = "\\mathampersand",
+    ["#"] = "\\mathhash",
+    ["$"] = "\\mathdollar",
+
     -- more
     ["_="]      = "≡",
 
@@ -666,6 +686,10 @@ local isunary = {
 
 }
 
+local isfunny = {
+    ["\\sin"]           = true,
+}
+
 local isinfix = {
     ["^"] = true,
     ["_"] = true,
@@ -675,12 +699,14 @@ local isleft = {
     ["\\left\\lparent"]  = true,
     ["\\left\\lbrace"]   = true,
     ["\\left\\lbracket"] = true,
+    ["\\left\\langle"]   = true,
     ["\\left."]          = true,
 }
 local isright = {
     ["\\right\\rparent"]  = true,
     ["\\right\\rbrace"]   = true,
     ["\\right\\rbracket"] = true,
+    ["\\right\\rangle"]   = true,
     ["\\right."]          = true,
 }
 
@@ -803,18 +829,25 @@ local p_text =
 --   + P("〉")  / "\\right\\rangle"
 
 local m_left = {
-  ["(:"] = "\\left\\langle",
-  ["{:"] = "\\left.",
-  ["[:"] = "\\left.",
-  ["("]  = "\\left\\lparent",
-  ["["]  = "\\left\\lbracket",
-  ["{"]  = "\\left\\lbrace",
-  ["<<"] = "\\left\\langle",     -- why not <:
-  ["|_"] = "\\left\\lfloor",
-  ["|~"] = "\\left\\lceil",
-  ["⟨"]  = "\\left\\langle",
-  ["〈"]  = "\\left\\langle",
-  ["〈"]  = "\\left\\langle",
+    ["(:"] = "\\left\\langle",
+    ["{:"] = "\\left.",
+    ["[:"] = "\\left.",
+    ["("]  = "\\left\\lparent",
+    ["["]  = "\\left\\lbracket",
+    ["{"]  = "\\left\\lbrace",
+    ["<<"] = "\\left\\langle",     -- why not <:
+    ["|_"] = "\\left\\lfloor",
+    ["|~"] = "\\left\\lceil",
+    ["⟨"]  = "\\left\\langle",
+    ["〈"]  = "\\left\\langle",
+    ["〈"]  = "\\left\\langle",
+    --
+    ["lparent"]  = "\\left\\lparent",
+    ["lbracket"] = "\\left\\lbracket",
+    ["lbrace"]   = "\\left\\lbrace",
+    ["langle"]   = "\\left\\langle",
+    ["lfloor"]   = "\\left\\lfloor",
+    ["lceil"]    = "\\left\\lceil",
 }
 
 local m_right = {
@@ -830,6 +863,21 @@ local m_right = {
     ["⟩"]  = "\\right\\rangle",
     ["〉"]  = "\\right\\rangle",
     ["〉"]  = "\\right\\rangle",
+    --
+    ["rparent"]  = "\\right\\rparent",
+    ["rbracket"] = "\\right\\rbracket",
+    ["rbrace"]   = "\\right\\rbrace",
+    ["rangle"]   = "\\right\\rangle",
+    ["rfloor"]   = "\\right\\rfloor",
+    ["rceil"]    = "\\right\\rceil",
+}
+
+local islimits = {
+    ["\\sum"]  = true,
+    ["∑"]      = true,
+    ["\\prod"] = true,
+    ["∏"]      = true,
+    ["\\lim"]  = true,
 }
 
 local p_left =
@@ -850,11 +898,8 @@ local p_right =
 -- faster bug also uglier:
 
 local p_special =
---     C("/")
---   +
     P("|")  * Cc("\\|") -- "\\middle\\|" -- maybe always add left / right as in mml ?
-  +
-  P("\\") * (
+  + P("\\") * (
         (
             P(" ") * (
                   Cc("{}") * p_spaces^0 * C(S("^_"))
@@ -867,28 +912,29 @@ local p_special =
 
 -- open | close :: {: | :}
 
-
 local parser = Ct { "tokenizer",
     tokenizer = (
         p_spaces
       + p_number
       + p_text
---       + Ct(p_open * V("tokenizer") * p_close)        -- {: (a+b,=,1),(a+b,=,7) :}
---       + Ct(p_open * V("tokenizer") * p_close_right)  -- {  (a+b,=,1),(a+b,=,7) :}
---       + Ct(p_open_left * V("tokenizer") * p_right)   -- {: (a+b,=,1),(a+b,=,7)  }
+   -- + Ct(p_open * V("tokenizer") * p_close)        -- {: (a+b,=,1),(a+b,=,7) :}
+   -- + Ct(p_open * V("tokenizer") * p_close_right)  -- {  (a+b,=,1),(a+b,=,7) :}
+   -- + Ct(p_open_left * V("tokenizer") * p_right)   -- {: (a+b,=,1),(a+b,=,7)  }
       + Ct(p_left * V("tokenizer") * p_right)        -- {  (a+b,=,1),(a+b,=,7)  }
       + p_special
       + p_reserved
       + p_entity
---       + p_utf - p_close - p_right
+  --  + p_utf - p_close - p_right
       + p_utf - p_right
     )^1,
 }
 
-local function show_state(state,level,t)
-    state = state + 1
-    report_asciimath(table.serialize(t,formatters["stage %s:%s"](level,state)))
-    return state
+local collapse  = nil
+local serialize = table.serialize
+local f_state   = formatters["level %s : %s : intermediate"]
+
+local function show_state(t,level,state)
+    report_asciimath(serialize(t,f_state(level,state)))
 end
 
 local function show_result(str,result)
@@ -896,20 +942,7 @@ local function show_result(str,result)
     report_asciimath("result > %s",result)
 end
 
-local function collapse(t,level)
-    if not t then
-        return ""
-    end
-    local state = 0
-    if trace_detail then
-        if level then
-            level = level + 1
-        else
-            level = 1
-        end
-        state = show_state(state,level,t)
-    end
-    --
+local function collapse_matrices(t)
     local n = #t
     if n > 4 and t[3] == "," then
         local l1 = t[1]
@@ -969,11 +1002,55 @@ local function collapse(t,level)
             end
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
+    return t
+end
+
+local function collapse_bars(t)
+    local n, i, l, m = #t, 1, false, 0
+    while i <= n do
+        local current = t[i]
+        if current == "\\|" then
+            if l then
+                m = m + 1
+                t[l] = "\\left\\|"
+                t[i] = "\\right\\|"
+                t[m] = { unpack(t,l,i) }
+                l = false
+            else
+                l = i
+            end
+        elseif not l then
+            m = m + 1
+            t[m] = current
+        end
+        i = i + 1
     end
-    --
+    if l then
+        local tt = { "\\left ." } -- space fools final checker
+        local tm  = 1
+        for i=1,m do
+            tm = tm + 1
+            tt[tm] = t[i]
+        end
+        tm = tm + 1
+        tt[tm] = "\\middle\\|"
+        for i=l+1,n do
+            tm = tm + 1
+            tt[tm] = t[i]
+        end
+        tm = tm + 1
+        tt[tm] = "\\right ." -- space fools final checker
+        m = tm
+        t = tt
+    elseif m < n then
+        for i=n,m+1,-1 do
+            t[i] = nil
+        end
+    end
+    return t
+end
+
+local function collapse_pairs(t)
     local n, i = #t, 1
     while i < n do
         local current = t[i]
@@ -1000,11 +1077,10 @@ local function collapse(t,level)
             i = i + 1
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_parentheses(t)
     local n, i = #t, 1
     if n > 2 then
         while i < n do
@@ -1012,8 +1088,6 @@ local function collapse(t,level)
             if type(current) == "table" and isleft[t[i-1]] and isright[t[i+1]] then
                 local c = #current
                 if c > 2 and isleft[current[1]] and isright[current[c]] then
---                     current[c] = nil
---                     current[1] = ""
                     remove(current,c)
                     remove(current,1)
                 end
@@ -1023,11 +1097,10 @@ local function collapse(t,level)
             end
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_signs(t)
     local n, m, i = #t, 0, 1
     while i <= n do
         m = m + 1
@@ -1041,8 +1114,6 @@ local function collapse(t,level)
             end
             if type(one) == "table" then
                 if isleft[one[1]] and isright[one[#one]] then
---                     one[1]   = ""
---                     one[#one] = nil
                     remove(one,#one)
                     remove(one,1)
                 end
@@ -1056,6 +1127,30 @@ local function collapse(t,level)
             end
             t[m] = current .. "{" .. one .. "}"
             i = i + 2
+        elseif i + 2 <= n and isfunny[current] then
+            local one = t[i+1]
+            if isinfix[one] then
+                local two = t[i+2]
+                if two == "-" then -- or another sign ? or unary ?
+                    local three = t[i+3]
+                    if three then
+                        if type(three) == "table" then
+                            three = collapse(three,level)
+                        end
+                        t[m] = current .. one .. "{" .. two .. three .. "}"
+                        i = i + 4
+                    else
+                        t[m] = current
+                        i = i + 1
+                    end
+                else
+                    t[m] = current
+                    i = i + 1
+                end
+            else
+                t[m] = current
+                i = i + 1
+            end
         else
             t[m] = current
             i = i + 1
@@ -1070,11 +1165,10 @@ local function collapse(t,level)
             t[i] = nil
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_binaries(t)
     local n, m, i = #t, 0, 1
     while i <= n do
         m = m + 1
@@ -1088,8 +1182,6 @@ local function collapse(t,level)
             end
             if type(one) == "table" then
                 if isleft[one[1]] and isright[one[#one]] then
---                     one[1]   = ""
---                     one[#one] = nil
                     remove(one,#one)
                     remove(one,1)
                 end
@@ -1101,8 +1193,6 @@ local function collapse(t,level)
             end
             if type(two) == "table" then
                 if isleft[two[1]] and isright[two[#two]] then
---                     two[1]   = ""
---                     two[#two] = nil
                     remove(two,#two)
                     remove(two,1)
                 end
@@ -1124,11 +1214,95 @@ local function collapse(t,level)
             t[i] = nil
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
+    return t
+end
+
+local function collapse_infixes_1(t)
+    local n, i = #t, 1
+    while i <= n do
+        local current = t[i]
+        if isinfix[current] then
+            local what = t[i+1]
+            if what then
+                if type(what) == "table" then
+                    local f, l = what[1], what[#what]
+                    if isleft[f] and isright[l] then
+                        remove(what,#what)
+                        remove(what,1)
+                    end
+                    t[i+1] = collapse(what,level) -- collapse ?
+                end
+                i = i + 2
+            else
+                break
+            end
+        else
+            i = i + 1
+        end
     end
-    --
+    return t
+end
+
+function collapse_limits(t)
+    local n, m, i = #t, 0, 1
+    while i <= n do
+        m = m + 1
+        local current = t[i]
+        if islimits[current] then
+            local one, two, first, second = nil, nil, t[i+1], t[i+3]
+            if first and isinfix[first] then
+                one = t[i+2]
+                if one then
+                 -- if type(one) == "table" then
+                 --     if isleft[one[1]] and isright[one[#one]] then
+                 --         remove(one,#one)
+                 --         remove(one,1)
+                 --     end
+                 --     one = collapse(one,level)
+                 -- end
+                    if second and isinfix[second] then
+                        two = t[i+4]
+                     -- if type(two) == "table" then
+                     --     if isleft[two[1]] and isright[two[#two]] then
+                     --         remove(two,#two)
+                     --         remove(two,1)
+                     --     end
+                     --     two = collapse(two,level)
+                     -- end
+                    end
+                    if two then
+                        t[m] = current .. "\\limits" .. first .. "{" .. one .. "}" .. second .. "{" .. two .. "}"
+                        i = i + 5
+                    else
+                        t[m] = current .. "\\limits" .. first .. "{" .. one .. "}"
+                        i = i + 3
+                    end
+                else
+                    t[m] = current
+                    i = i + 1
+                end
+            else
+                t[m] = current
+                i = i + 1
+            end
+        else
+            t[m] = current
+            i = i + 1
+        end
+    end
+    if i == n then -- yes?
+        m = m + 1
+        t[m] = t[n]
+    end
+    if m < n then
+        for i=n,m+1,-1 do
+            t[i] = nil
+        end
+    end
+    return t
+end
+
+local function collapse_tables(t)
     local n, m, i = #t, 0, 1
     while i <= n do
         m = m + 1
@@ -1154,19 +1328,25 @@ local function collapse(t,level)
             t[i] = nil
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_infixes_2(t)
     local n, m, i = #t, 0, 1
     while i < n do
         local current = t[i]
         if isinfix[current] and i > 1 then
             local tl = t[i-1]
             local tr = t[i+1]
-            t[m] = tl .. current .. "{" .. tr .. "}"
-            i = i + 2
+            local ti = t[i+2]
+            local tn = t[i+3]
+            if ti and tn and isinfix[ti] then
+                t[m] = tl .. current .. "{" .. tr .. "}" .. ti .. "{" .. tn .. "}"
+                i = i + 4
+            else
+                t[m] = tl .. current .. "{" .. tr .. "}"
+                i = i + 2
+            end
         else
             m = m + 1
             t[m] = current
@@ -1182,11 +1362,10 @@ local function collapse(t,level)
             t[i] = nil
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_fractions_1(t)
     local n, m, i = #t, 0, 1
     while i < n do
         local current = t[i]
@@ -1222,16 +1401,14 @@ local function collapse(t,level)
             t[i] = nil
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_fractions_2(t)
     local n, m, i = #t, 0, 1
     while i < n do
         local current = t[i]
         if current == "\\slash" and i > 1 then
---             t[m] = "{\\left(" .. t[i-1] .. "\\middle/" .. t[i+1] .. "\\right)}"
             t[m] = "{\\left." .. t[i-1] .. "\\middle/" .. t[i+1] .. "\\right.}"
             i = i + 2
         else
@@ -1249,17 +1426,47 @@ local function collapse(t,level)
             t[i] = nil
         end
     end
-    --
-    if trace_detail then
-        state = show_state(state,level,t)
-    end
-    --
+    return t
+end
+
+local function collapse_result(t)
     local n = #t
-    if t[1] == "\\left." and t[n] == "\\right." then
-        return  concat(t," ",2,n-1)
+    if t[1] == "\\left." and t[n] == "\\right." then -- see bar .. space needed there
+        return concat(t," ",2,n-1)
     else
-        return  concat(t," ")
+        return concat(t," ")
+    end
+end
+
+collapse = function(t,level)
+    -- check
+    if not t then
+        return ""
+    end
+    -- tracing
+    if trace_detail then
+        if level then
+            level = level + 1
+        else
+            level = 1
+        end
+        show_state(t,level,"parsed")
     end
+    -- steps
+    t = collapse_matrices   (t) if trace_detail then show_state(t,level,"matrices")      end
+    t = collapse_bars       (t) if trace_detail then show_state(t,level,"bars")          end
+    t = collapse_pairs      (t) if trace_detail then show_state(t,level,"pairs")         end
+    t = collapse_parentheses(t) if trace_detail then show_state(t,level,"parentheses")   end
+    t = collapse_signs      (t) if trace_detail then show_state(t,level,"signs")         end
+    t = collapse_binaries   (t) if trace_detail then show_state(t,level,"binaries")      end
+    t = collapse_infixes_1  (t) if trace_detail then show_state(t,level,"infixes (1)")   end
+    t = collapse_limits     (t) if trace_detail then show_state(t,level,"limits")        end
+    t = collapse_tables     (t) if trace_detail then show_state(t,level,"tables")        end
+    t = collapse_infixes_2  (t) if trace_detail then show_state(t,level,"infixes (2)")   end
+    t = collapse_fractions_1(t) if trace_detail then show_state(t,level,"fractions (1)") end
+    t = collapse_fractions_2(t) if trace_detail then show_state(t,level,"fractions (2)") end
+    -- done
+    return collapse_result(t)
 end
 
 -- todo: cache simple ones, say #str < 10, maybe weak
@@ -1456,6 +1663,15 @@ if not context then
 --     report_asciimath(cleanedup([[a "α" b]]))
 --     report_asciimath(cleanedup([[//4]]))
 
+--     convert([[sum x]])
+--     convert([[sum^(1)_(2) x]])
+--     convert([[lim_(1)^(2) x]])
+--     convert([[lim_(1) x]])
+--     convert([[lim^(2) x]])
+
+--     convert([[{: rangle]])
+--     convert([[\langle\larr]])
+--     convert([[langlelarr]])
 --     convert([[D_f=[0 ,→〉]])
 --     convert([[ac+sinx+xsqrtx]])
 --     convert([[ac+\alpha x+xsqrtx-cc b*pi**psi-3alephx / bb X]])
@@ -1466,10 +1682,10 @@ if not context then
 --     convert([[//4]])
 --     convert([[ {(a+b,=,1),(a+b,=,7)) ]])
 
--- convert([[ 2/a // 5/b = (2 b) / ( a b) // ( 5 a ) / ( a b ) = (2 b ) / ( 5 a ) ]])
--- convert([[ (2+x)/a // 5/b  ]])
+--     convert([[ 2/a // 5/b = (2 b) / ( a b) // ( 5 a ) / ( a b ) = (2 b ) / ( 5 a ) ]])
+--     convert([[ (2+x)/a // 5/b  ]])
 
--- convert([[ ( 2/a ) // ( 5/b ) = ( (2 b) / ( a b) ) // ( ( 5 a ) / ( a b ) ) = (2 b ) / ( 5 a ) ]])
+--     convert([[ ( 2/a ) // ( 5/b ) = ( (2 b) / ( a b) ) // ( ( 5 a ) / ( a b ) ) = (2 b ) / ( 5 a ) ]])
 
 --     convert([[ (x/y)^3 = x^3/y^3 ]])