2021-08-30 16:23:00

8 files changed, 1002 insertions, 67 deletions

diff --git a/doc/context/documents/general/manuals/lowlevel-alignments.pdf b/doc/context/documents/general/manuals/lowlevel-alignments.pdf
new file mode 100644
index 000000000..bbd187dfb
--- /dev/null
+++ b/doc/context/documents/general/manuals/lowlevel-alignments.pdf
diff --git a/doc/context/documents/general/manuals/lowlevel-expansion.pdf b/doc/context/documents/general/manuals/lowlevel-expansion.pdf
index 8628f4896..6f50c6d22 100644
--- a/doc/context/documents/general/manuals/lowlevel-expansion.pdf
+++ b/doc/context/documents/general/manuals/lowlevel-expansion.pdf
diff --git a/doc/context/presentations/context/2020/context-2020-concepts.pdf b/doc/context/presentations/context/2020/context-2020-concepts.pdf
index 913db5ef3..ca6014f39 100644
--- a/doc/context/presentations/context/2020/context-2020-concepts.pdf
+++ b/doc/context/presentations/context/2020/context-2020-concepts.pdf
diff --git a/doc/context/presentations/context/2020/context-2020-concepts.tex b/doc/context/presentations/context/2020/context-2020-concepts.tex
index a472a33f4..23bdba432 100644
--- a/doc/context/presentations/context/2020/context-2020-concepts.tex
+++ b/doc/context/presentations/context/2020/context-2020-concepts.tex
@@ -2,6 +2,11 @@
 
 \usemodule[present-boring,abbreviations-logos]
 
+% This was a talk about a preliminary mechanism so it has been adapted to run in
+% the current version. I need to check other 2020 files and examples and will do
+% that when I update (especially the low level) manuals. Let me know when there
+% are issues.
+
 \startdocument
   [title={CONCEPTS},
    banner={experiments turned features},
@@ -110,6 +115,10 @@ $\whatevera a = (-2) \quad \whateverb a = (-2)$ \par
 
 \typebuffer \startpacked \getbuffer \stoppacked
 
+{\em We can now also enable and disable specific features in the engine that
+control traditional or \OPENTYPE\ approaches. This is only there for experimental
+and educational purposes.}
+
 \stoptitle
 
 \starttitle[title=Macros]
@@ -163,6 +172,9 @@ Special behaviour:
 #3<-3
 \stoptyping
 
+{\em There are more specifiers and I admit that they are hard to remember. But
+they are mostly used in low level macros anyway.}
+
 \page
 
 Optional tokens (we also show some \TEX-expansion-horror here):
@@ -184,6 +196,9 @@ Optional tokens (we also show some \TEX-expansion-horror here):
 
 or as tokens (\type{\showluatokens\doifelseinset}) on the next page:
 
+{\em There are some expansion related extensions that are discussed in the low
+level expansion manual.}
+
 \page
 
 \starttyping
@@ -248,7 +263,7 @@ As in:
 
 Hyphenation at work:
 
-\startlinecorrection[line]
+\startlinecorrection[line] \small
 \startcombination[distance=3cm,nx=6]
     {\vtop{\hsize 2mm \strut NEDERLANDS\vskip.5\lineheight}} {\type{NEDERLANDS}}
     {\vtop{\hsize 2mm \strut Nederlands\vskip.5\lineheight}} {\type{Nederlands}}
@@ -360,6 +375,9 @@ nederlands\discretionary options 3 {!}{!}{!}nederlands
 At some point it will become \quote {frozen} functionality and that's when it gets
 documented (first we need to integrate and play a bit more with it in \CONTEXT).
 
+{\em There is now a plugin mechanism that provides more control over language
+specific hyphenation, e.g.\ compound words combined with ligatures.}
+
 \stoptitle
 
 \starttitle[title=Local control]
@@ -449,9 +467,12 @@ And a teaser:
 
 \starttyping
 \protected\def\widthofcontent#1{\beginlocalcontrol
-     \setbox\scratchbox\hbox{#1}\endlocalcontrol   \wd\scratchbox}
+     \setbox\scratchbox\hbox{#1}\endlocalcontrol \wd\scratchbox}
 \stoptyping
 
+{\em These mechanisms can have surprising side effects due to input stacking.
+There is some more info in the low level expansion manual.}
+
 \stoptitle
 
 \starttitle[title=Conditionals]
@@ -544,6 +565,10 @@ v: \setbox0\hbox{uncopy \footnote{v: unhcopy}}\setbox2\vbox{\unhcopy0}\box2\par
 % \setprelistbox0\hbox{BEFORE} \setpostlistbox0\hbox{AFTER}
 % \box0
 
+{\em Everything insert related will always have side effects. It's complicated
+by the fact that the page flow interferes with expectations of where notes
+break cq.\ end up.}
+
 \stoptitle
 
 \starttitle[title=Normalizing lines]
@@ -559,16 +584,16 @@ We can have predictable lines:
 Standard (but already with left skips):
 
 \start
-    \bitwiseflip \normalizelinemode -\indentskipmodecode
-    \bitwiseflip \normalizelinemode -\normalizelinemodecode
+    \bitwiseflip \normalizelinemode -\parindentskipnormalizecode
+    \bitwiseflip \normalizelinemode -\normalizelinenormalizecode
     \showmakeup \relax \getbuffer
 \stop
 
 Normalized (enhanced, no shifts, indent skip):
 
 \start
-    \bitwiseflip \normalizelinemode \indentskipmodecode
-    \bitwiseflip \normalizelinemode \normalizelinemodecode
+    \bitwiseflip \normalizelinemode \parindentskipnormalizecode
+    \bitwiseflip \normalizelinemode \normalizelinenormalizecode
     \showmakeup \relax \getbuffer
 \stop
 
@@ -583,36 +608,51 @@ Normalized (enhanced, no shifts, indent skip):
 Standard:
 
 \start
-    \bitwiseflip \normalizelinemode -\indentskipmodecode
-    \bitwiseflip \normalizelinemode -\normalizelinemodecode
+    \bitwiseflip \normalizelinemode -\parindentskipnormalizecode
+    \bitwiseflip \normalizelinemode -\normalizelinenormalizecode
     \showmakeup \relax \getbuffer
 \stop
 
 Normalized:
 
 \start
-    \bitwiseflip \normalizelinemode \indentskipmodecode
-    \bitwiseflip \normalizelinemode \normalizelinemodecode
+    \bitwiseflip \normalizelinemode \parindentskipnormalizecode
+    \bitwiseflip \normalizelinemode \normalizelinenormalizecode
     \showmakeup \relax \getbuffer
 \stop
 
 % {\showmakeup \getbuffer}
 
+{\em There might be some more normalization in the future in other subsystems of
+the engine. One should be aware of this when manipulating node lists after they
+come out such subsystems.}
+
 \stoptitle
 
 \starttitle[title=Freezing paragraph properties]
 
-\startbuffer
-\forgetparagraphfreezing \placefigure[left]{}{} {\bf Andrew Cuomo:} \input cuomo
+\startbuffer[sample]
+\startplacefigure[location=left,number=no] \externalfigure[halslegacy.jpg][width=30pt] \stopplacefigure
+
+{\bf David Stork:} \samplefile{stork}
 \stopbuffer
 
-\typebuffer[style=\tt\small] \start \switchtobodyfont[8pt] \getbuffer \par \stop
+\startbuffer[demo]
+\forgetparagraphfreezing \getbuffer[sample]
+\stopbuffer
 
-\startbuffer
-\setparagraphfreezing \placefigure[left]{}{} {\bf Andrew Cuomo:}  \input cuomo
+\typebuffer[demo][style=\tt\small] \start \switchtobodyfont[6pt] \getbuffer[demo] \par \stop
+
+\startbuffer[demo]
+\setparagraphfreezing    \getbuffer[sample]
 \stopbuffer
 
-\typebuffer[style=\tt\small] \start \switchtobodyfont[8pt] \getbuffer \par \stop
+\typebuffer[demo][style=\tt\small] \start \switchtobodyfont[6pt] \getbuffer[demo] \par \stop
+
+Sample: \typebuffer[sample][style=\tt\small\small]
+
+{\em This feature will stepwise be applied to mechanism and might have side effects when
+users have their own hacks around \TEX's limitations (and side effects).}
 
 \stoptitle
 
@@ -689,6 +729,9 @@ Example 2:
 
 \startcolumns  \startpacked \getbuffer \stoppacked \stopcolumns
 
+{\em These are just weird examples, but you can expect more interesting features to
+show up. Beware of stacking because order matters.}
+
 \stoptitle
 
 \stopdocument
diff --git a/doc/context/sources/general/manuals/cld/cld-luafunctions.tex b/doc/context/sources/general/manuals/cld/cld-luafunctions.tex
index 0b2bcb3da..c6443f088 100644
--- a/doc/context/sources/general/manuals/cld/cld-luafunctions.tex
+++ b/doc/context/sources/general/manuals/cld/cld-luafunctions.tex
@@ -68,7 +68,7 @@ You can use \type {insert} and \type {remove} for adding or replacing entries in
 an indexed table.
 
 \starttyping
-table.insert(t,value,position)
+table.insert(t,position,value)
 value = table.remove(t,position)
 \stoptyping
 
@@ -310,7 +310,7 @@ local merged = table.imerged (one, two, ...)
 \startsummary[title={copy fastcopy}]
 
 When copying a table we need to make a real and deep copy. The \type {copy}
-function is an adapted version from the \LUA\ wiki. The \type {fastopy} is faster
+function is an adapted version from the \LUA\ wiki. The \type {fastcopy} is faster
 because it does not check for circular references and does not share tables when
 possible. In practice using the fast variant is okay.
 
diff --git a/doc/context/sources/general/manuals/lowlevel/lowlevel-alignments.tex b/doc/context/sources/general/manuals/lowlevel/lowlevel-alignments.tex
new file mode 100644
index 000000000..c641e0d65
--- /dev/null
+++ b/doc/context/sources/general/manuals/lowlevel/lowlevel-alignments.tex
@@ -0,0 +1,697 @@
+% language=us runpath=texruns:manuals/lowlevel
+
+\startcomponent lowlevel-alignments
+
+\environment lowlevel-style
+
+\startdocument
+  [title=alignments,
+   color=middlegreen]
+
+\startsection[title=Introduction]
+
+\TEX\ has a couple of subsystems and alignments is one of them. This mechanism is
+used to construct tables or alike. Because alignments use low level primitives to
+set up and construct a table, and because such a setup can be rather extensive, in
+most cases users will rely on macros that hide this.
+
+\startbuffer
+\halign {
+        \alignmark\hss \aligntab
+    \hss\alignmark\hss \aligntab
+    \hss\alignmark     \cr
+    1.1     \aligntab 2,2     \aligntab 3=3     \cr
+    11.11   \aligntab 22,22   \aligntab 33=33   \cr
+    111.111 \aligntab 222,222 \aligntab 333=333 \cr
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+That one doesn't look too complex and comes out as:
+
+\blank\getbuffer\blank
+
+This is how the previous code comes out when we use one of the \CONTEXT\ table
+mechanism.
+
+\startbuffer
+\starttabulate[|l|c|r|]
+  \NC 1.1     \NC 2,2     \NC 3=3     \NC \NR
+  \NC 11.11   \NC 22,22   \NC 33=33   \NC \NR
+  \NC 111.111 \NC 222,222 \NC 333=333 \NC \NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+\blank\getbuffer\blank
+
+That one looks a bit different with respect to spaces, so let's go back to the
+low level variant:
+
+\startbuffer
+\halign {
+        \alignmark\hss \aligntab
+    \hss\alignmark\hss \aligntab
+    \hss\alignmark     \cr
+  1.1\aligntab     2,2\aligntab     3=3\cr
+  11.11\aligntab   22,22\aligntab   33=33\cr
+  111.111\aligntab 222,222\aligntab 333=333\cr
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+Here we don't have spaces in the content part and therefore also no spaces in the
+result:
+
+\blank\getbuffer\blank
+
+You can automate dealing with unwanted spacing:
+
+\startbuffer
+\halign {
+      \ignorespaces\alignmark\unskip\hss \aligntab
+  \hss\ignorespaces\alignmark\unskip\hss \aligntab
+  \hss\ignorespaces\alignmark\unskip     \cr
+  1.1     \aligntab 2,2     \aligntab 3=3     \cr
+  11.11   \aligntab 22,22   \aligntab 33=33   \cr
+  111.111 \aligntab 222,222 \aligntab 333=333 \cr
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+We get:
+
+\blank\getbuffer\blank
+
+By moving the space skipping and cleanup to the so called preamble we don't need
+to deal with it in the content part. We can also deal with inter|-|column spacing
+there:
+
+\startbuffer
+\halign {
+      \ignorespaces\alignmark\unskip\hss \tabskip 1em \aligntab
+  \hss\ignorespaces\alignmark\unskip\hss \tabskip 1em \aligntab
+  \hss\ignorespaces\alignmark\unskip     \tabskip 0pt \cr
+  1.1     \aligntab 2,2     \aligntab 3=3     \cr
+  11.11   \aligntab 22,22   \aligntab 33=33   \cr
+  111.111 \aligntab 222,222 \aligntab 333=333 \cr
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+\blank\getbuffer\blank
+
+If for the moment we forget about spanning columns (\type {\span}) and locally
+ignoring preamble entries (\type {\omit}) these basic commands are not that
+complex to deal with. Here we use \type {\alignmark} but that is just a primitive
+that we use instead of \type {#} while \type {\aligntab} is the same as \type
+{&}, but using the characters instead also assumes that they have the catcode
+that relates to a parameter and alignment tab (and in \CONTEXT\ that is not the
+case). The \TEX book has plenty alignment examples so if you really want to learn
+about them, consult that must|-|have|-|book.
+
+\stopsection
+
+\startsection[title=Between the lines]
+
+The individual rows of a horizontal alignment are treated as lines. This means that,
+as we see in the previous section, the interline spacing is okay. However, that also
+means that when we mix the lines with rules, the normal \TEX\ habits kick in. Take
+this:
+
+\startbuffer
+\halign {
+      \ignorespaces\alignmark\unskip\hss \tabskip 1em \aligntab
+  \hss\ignorespaces\alignmark\unskip\hss \tabskip 1em \aligntab
+  \hss\ignorespaces\alignmark\unskip     \tabskip 0pt \cr
+  \noalign{\hrule}
+  1.1     \aligntab 2,2     \aligntab 3=3     \cr
+  \noalign{\hrule}
+  11.11   \aligntab 22,22   \aligntab 33=33   \cr
+  \noalign{\hrule}
+  111.111 \aligntab 222,222 \aligntab 333=333 \cr
+  \noalign{\hrule}
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+The result doesn't look pretty and actually, when you see documents produced by
+\TEX\ using alignments you should not be surprised to notice rather ugly spacing.
+The user (or the macropackage) should deal with that explicitly, and this is not
+always the case.
+
+\startlinecorrection
+\getbuffer
+\stoplinecorrection
+
+The solution is often easy:
+
+\startbuffer
+\halign {
+      \ignorespaces\strut\alignmark\unskip\hss \tabskip 1em \aligntab
+  \hss\ignorespaces\strut\alignmark\unskip\hss \tabskip 1em \aligntab
+  \hss\ignorespaces\strut\alignmark\unskip     \tabskip 0pt \cr
+  \noalign{\hrule}
+  1.1     \aligntab 2,2     \aligntab 3=3     \cr
+  \noalign{\hrule}
+  11.11   \aligntab 22,22   \aligntab 33=33   \cr
+  \noalign{\hrule}
+  111.111 \aligntab 222,222 \aligntab 333=333 \cr
+  \noalign{\hrule}
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+\startlinecorrection
+\getbuffer
+\stoplinecorrection
+
+The user will not notice it but alignments put some pressure on the general \TEX\
+scanner. Actually, the scanner is either scanning an alignment or it expects
+regular text (including math). When you look at the previous example you see
+\type {\noalign}. When the preamble is read, \TEX\ will pick up rows till it
+finds the final brace. Each row is added to a temporary list and the \type
+{\noalign} will enter a mode where other stuff gets added to that list. It all
+involves subtle look ahead but with minimal overhead. When the whole alignment is
+collected a final pass over that list will package the cells and rows (lines) in
+the appropriate way using information collected (like the maximum width of a cell
+and width of the current cell. It will also deal with spanning cells then.
+
+So let's summarize what happens:
+
+\startitemize[n,packed]
+\startitem
+    scan the preamble that defines the cells (where the last one is repeated
+    when needed)
+\stopitem
+\startitem
+    check for \type {\cr}, \type {\noalign} or a right brace; when a row is
+    entered scan for cells in parallel the preamble so that cell specifications
+    can be applied (then start again)
+\stopitem
+\startitem
+    package the preamble based on information with regards to the cells in
+    a column
+\stopitem
+\startitem
+    apply the preamble packaging information to the columns and also deal with
+    pending cell spans
+\stopitem
+\startitem
+    flush the result to the current list
+\stopitem
+\stopitemize
+
+The second (repeated) step is complicated by the fact that the scanner has to
+look ahead for a \type {\noalign}, \type {\cr}, \type {\omit} or \type {\span}
+and when doing that it has to expand what comes. This can give side effects and
+often results in obscure error messages. When for instance an \type {\if} is seen
+and expanded, the wrong branch can be entered. And when you use protected macros
+embedded alignment commands are not seen at all. Also, nesting \type {\noalign}
+is not permitted.
+
+All these side effects are to be handled in a macro package when it wraps
+alignments in a high level interface and \CONTEXT\ does that for you. But because
+the code doesn't always look pretty then, in \LUAMETATEX\ the alignment mechanism
+has been extended a bit over time.
+
+The first extension was to permit nested usage of \type {\noalign}. This has
+resulted of a little reorganization of the code. A next extension showed up when
+overload protection was introduced and extra prefixes were added. We can signal
+the scanner that a macro is actually a \type {\noalign} variant: \footnote {A
+better prefix would have been \type {\peekaligned} because in the meantime other
+alignment primitives also can use this property.}
+
+\starttyping[option=TEX]
+\noaligned\protected\def\InBetween{\noalign{...}}
+\stoptyping
+
+This extension resulted in a second bit of reorganization (think of internal
+command codes and such) but still the original processing of alignments was
+there.
+
+A third overhaul of the code actually did lead to some adaptations in the way
+alignments are constructed so let's move on to that.
+
+\stopsection
+
+\startsection[title={Pre-, inter- and post-tab skips}]
+
+The basic structure of a preamble and row is actually not that complex: it is
+a mix of tab skip glue and cells (that are just boxes):
+
+\startbuffer
+\tabskip 10pt
+\halign {
+  \strut\alignmark\tabskip 12pt\aligntab
+  \strut\alignmark\tabskip 14pt\aligntab
+  \strut\alignmark\tabskip 16pt\cr
+  \noalign{\hrule}
+  cell 1.1\aligntab cell 1.2\aligntab cell 1.3\cr
+  \noalign{\hrule}
+  cell 2.1\aligntab cell 2.2\aligntab cell 2.3\cr
+  \noalign{\hrule}
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+The tab skips are set in advance and apply to the next cell (or after the last
+one).
+
+\startbuffer[blownup-1]
+\startlinecorrection
+{\showmakeup[glue]\scale[width=\textwidth]{\vbox{\getbuffer}}}
+\stoplinecorrection
+\stopbuffer
+
+\getbuffer[blownup-1]
+
+% \normalizelinemode \zerocount % \discardzerotabskipsnormalizecode
+
+In the \CONTEXT\ table mechanisms the value of \type {\tabskip} is zero
+in most cases. As in:
+
+\startbuffer
+\tabskip 0pt
+\halign {
+  \strut\alignmark\aligntab
+  \strut\alignmark\aligntab
+  \strut\alignmark\cr
+  \noalign{\hrule}
+  cell 1.1\aligntab cell 1.2\aligntab cell 1.3\cr
+  \noalign{\hrule}
+  cell 2.1\aligntab cell 2.2\aligntab cell 2.3\cr
+  \noalign{\hrule}
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+When these ships are zero, they still show up in the end:
+
+\getbuffer[blownup-1]
+
+Normally, in order to achieve certain effects there will be more align entries in
+the preamble than cells in the table, for instance because you want vertical
+lines between cells. When these are not used, you can get quite a bit of empty
+boxes and zero skips. Now, of course this is seldom a problem, but when you have
+a test document where you want to show font properties in a table and that font
+supports a script with some ten thousand glyphs, you can imagine that it
+accumulates and in \LUATEX\ (and \LUAMETATEX) nodes are larger so it is one of
+these cases where in \CONTEXT\ we get messages on the console that node memory is
+bumped.
+
+After playing a bit with stripping zero tab skips I found that the code would not
+really benefit from such a feature: lots of extra tests made if quite ugly. As a
+result a first alternative was to just strip zero skips before an alignment got
+flushed. At least we're then a bit leaner in the processes that come after it.
+This feature is now available as one of the normalizer bits.
+
+But, as we moved on, a more natural approach was to keep the skips in the
+preamble, because that is where a guaranteed alternating skip|/|box is assumed.
+It also makes that the original documentation is still valid. However, in the
+rows construction we can be lean. This is driven by a keyword to \type {\halign}:
+
+\startbuffer
+\tabskip 0pt
+\halign noskips {
+  \strut\alignmark\aligntab
+  \strut\alignmark\aligntab
+  \strut\alignmark\cr
+  \noalign{\hrule}
+  cell 1.1\aligntab cell 1.2\aligntab cell 1.3\cr
+  \noalign{\hrule}
+  cell 2.1\aligntab cell 2.2\aligntab cell 2.3\cr
+  \noalign{\hrule}
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+No zero tab skips show up here:
+
+\getbuffer[blownup-1]
+
+When playing with all this the \LUAMETATEX\ engine also got a tracing option for
+alignments. We already had one that showed some of the \type{\noalign} side
+effects, but showing the preamble was not yet there. This is what \typ
+{\tracingalignments = 2} results in:
+
+% {\tracingalignments2 \setbox0\vbox{\getbuffer}}
+
+\starttyping[option=TEX]
+<preamble>
+\glue[ignored][...] 0.0pt
+\alignrecord
+..{\strut }
+..<content>
+..{\endtemplate }
+\glue[ignored][...] 0.0pt
+\alignrecord
+..{\strut }
+..<content>
+..{\endtemplate }
+\glue[ignored][...] 0.0pt
+\alignrecord
+..{\strut }
+..<content>
+..{\endtemplate }
+\glue[ignored][...] 0.0pt
+\stoptyping
+
+The \type {ignored} subtype is (currently) only used for these alignment tab
+skips and it triggers a check later on when the rows are constructed. The \type
+{<content>} is what get injected in the cell (represented by \type {\alignmark}).
+The pseudo primitives are internal and not public.
+
+\stopsection
+
+\startsection[title={Cell widths}]
+
+Imagine this:
+
+\startbuffer
+\halign {
+  x\hbox to 3cm{\strut    \alignmark\hss}\aligntab
+  x\hbox to 3cm{\strut\hss\alignmark\hss}\aligntab
+  x\hbox to 3cm{\strut\hss\alignmark    }\cr
+  cell 1.1\aligntab cell 1.2\aligntab cell 1.3\cr
+  cell 2.1\aligntab cell 2.2\aligntab cell 2.3\cr
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+which renders as:
+
+\startbuffer[blownup-2]
+\startlinecorrection
+{\showboxes\scale[width=\textwidth]{\vbox{\getbuffer}}}
+\stoplinecorrection
+\stopbuffer
+
+{\showboxes\getbuffer[blownup-2]}
+
+A reason to have boxes here is that it enforces a cell width but that is done at
+the cost of an extra wrapper. In \LUAMETATEX\ the \type {hlist} nodes are rather
+large because we have more options than in original \TEX, for instance offsets
+and orientation. So, in a table with 10K rows of 4 cells yet get 40K extra \type
+{hlist} nodes allocated. Now, one can argue that we have plenty of memory but
+being lazy is not really a sign of proper programming.
+
+\startbuffer
+\halign {
+  x\tabsize 3cm\strut    \alignmark\hss\aligntab
+  x\tabsize 3cm\strut\hss\alignmark\aligntab
+  x\tabsize 3cm\strut\hss\alignmark\hss\cr
+  cell 1.1\aligntab cell 1.2\aligntab cell 1.3\cr
+  cell 2.1\aligntab cell 2.2\aligntab cell 2.3\cr
+}
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+If you look carefully you will see that this time we don't have the embedded
+boxes:
+
+{\showboxes\getbuffer[blownup-2]}
+
+So, both the sparse skip and new \type {\tabsize} feature help to make these
+extreme tables (spanning hundreds of pages) not consume irrelevant memory and
+also make that later on we don't have to consult useless nodes.
+
+\stopsection
+
+\startsection[title=Plugins]
+
+Yet another \LUAMETATEX\ extension is a callback that kicks in between the
+preamble preroll and finalizing the alignment. Initially as test and
+demonstration a basic character alignment feature was written but that works so
+well that in some places it can replace (or compliment) the already existing
+features in the \CONTEXT\ table mechanisms.
+
+\startbuffer
+\starttabulate[|lG{.}|cG{,}|rG{=}|cG{x}|]
+\NC 1.1     \NC 2,2     \NC 3=3     \NC a 0xFF   \NC \NR
+\NC 11.11   \NC 22,22   \NC 33=33   \NC b 0xFFF  \NC \NR
+\NC 111.111 \NC 222,222 \NC 333=333 \NC c 0xFFFF \NC \NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+The tabulate mechanism in \CONTEXT\ is rather old and stable and it is the
+preferred way to deal with tabular content in the text flow. However, adding the
+\type {G} specifier (as variant of the \type {g} one) could be done without
+interference or drop in performance. This new \type {G} specifier tells the
+tabulate mechanism that in that column the given character is used to vertically
+align the content that has this character.
+
+\blank\getbuffer\blank
+
+Let's make clear that this is {\em not} an engine feature but a \CONTEXT\ one. It
+is however made easy by this callback mechanism. We can of course use this feature
+with the low level alignment primitives, assuming that you tell the machinery that
+the plugin is to be kicked in.
+
+\startbuffer
+\halign noskips \alignmentcharactertrigger \bgroup
+    \tabskip2em
+        \setalignmentcharacter.\ignorespaces\alignmark\unskip\hss \aligntab
+    \hss\setalignmentcharacter,\ignorespaces\alignmark\unskip\hss \aligntab
+    \hss\setalignmentcharacter=\ignorespaces\alignmark\unskip     \aligntab
+    \hss                       \ignorespaces\alignmark\unskip\hss \cr
+      1.1   \aligntab   2,2   \aligntab   3=3   \aligntab \setalignmentcharacter{.}\relax 4.4\cr
+     11.11  \aligntab  22,22  \aligntab  33=33  \aligntab \setalignmentcharacter{,}\relax 44,44\cr
+    111.111 \aligntab 222,222 \aligntab 333=333 \aligntab \setalignmentcharacter{!}\relax 444!444\cr
+        x   \aligntab     x   \aligntab     x   \aligntab \setalignmentcharacter{/}\relax /\cr
+       .1   \aligntab    ,2   \aligntab    =3   \aligntab \setalignmentcharacter{?}\relax ?4\cr
+       .111 \aligntab    ,222 \aligntab    =333 \aligntab \setalignmentcharacter{=}\relax 44=444\cr
+\egroup
+\stopbuffer
+
+{\switchtobodyfont[8pt] \typebuffer[option=TEX]}
+
+This rather verbose setup renders as:
+
+\blank\getbuffer\blank
+
+Using a high level interface makes sense but local control over such alignment too, so
+here follow some more examples. Here we use different alignment characters:
+
+\startbuffer
+\starttabulate[|lG{.}|cG{,}|rG{=}|cG{x}|]
+\NC 1.1     \NC 2,2     \NC 3=3     \NC a 0xFF   \NC \NR
+\NC 11.11   \NC 22,22   \NC 33=33   \NC b 0xFFF  \NC \NR
+\NC 111.111 \NC 222,222 \NC 333=333 \NC c 0xFFFF \NC \NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[option=TEX] \getbuffer
+
+In this example we specify the characters in the cells. We still need to add a
+specifier in the preamble definition because that will trigger the plugin.
+
+\startbuffer
+\starttabulate[|lG{}|lG{}|]
+\NC \showglyphs \setalignmentcharacter{.}1.1     \NC \setalignmentcharacter{.}1.1     \NC\NR
+\NC \showglyphs \setalignmentcharacter{,}11,11   \NC \setalignmentcharacter{,}11,11   \NC\NR
+    \NC \showglyphs \setalignmentcharacter{=}111=111 \NC \setalignmentcharacter{=}111=111 \NC\NR
+\stoptabulate
+\stopbuffer
+
+{\switchtobodyfont[8pt] \typebuffer[option=TEX]} \getbuffer
+
+You can mix these approaches:
+
+\startbuffer
+\starttabulate[|lG{.}|lG{}|]
+\NC 1.1     \NC \setalignmentcharacter{.}1.1     \NC\NR
+\NC 11.11   \NC \setalignmentcharacter{.}11.11   \NC\NR
+\NC 111.111 \NC \setalignmentcharacter{.}111.111 \NC\NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[option=TEX] \getbuffer
+
+Here the already present alignment feature, that at some point in tabulate might
+use this new feature, is meant for numbers, but here we can go wild with words,
+although of course you need to keep in mind that we deal with typeset text, so
+there may be no match.
+
+\startbuffer
+\starttabulate[|lG{.}|rG{.}|]
+\NC foo.bar \NC foo.bar \NC \NR
+\NC  oo.ba  \NC  oo.ba  \NC \NR
+\NC   o.b   \NC   o.b   \NC \NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[option=TEX] \getbuffer
+
+This feature will only be used in know situations and those seldom involve advanced
+typesetting. However, the following does work: \footnote {Should this be an option
+instead?}
+
+\startbuffer
+\starttabulate[|cG{d}|]
+\NC \smallcaps abcdefgh \NC \NR
+\NC              xdy    \NC \NR
+\NC \sl          xdy    \NC \NR
+\NC \tttf        xdy    \NC \NR
+\NC \tfd          d     \NC \NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[option=TEX] \getbuffer
+
+As always with such mechanisms, the question is \quotation {Where to stop?} But it
+makes for nice demos and as long as little code is needed it doesn't hurt.
+
+\stopsection
+
+\startsection[title=Pitfalls and tricks]
+
+The next example mixes bidirectional typesetting. It might look weird at first
+sight but the result conforms to what we discussed in previous paragraphs.
+
+\startbuffer
+\starttabulate[|lG{.}|lG{}|]
+\NC \righttoleft 1.1   \NC \righttoleft \setalignmentcharacter{.}1.1   \NC\NR
+\NC              1.1   \NC              \setalignmentcharacter{.}1.1   \NC\NR
+\NC \righttoleft 1.11  \NC \righttoleft \setalignmentcharacter{.}1.11  \NC\NR
+\NC              1.11  \NC              \setalignmentcharacter{.}1.11  \NC\NR
+\NC \righttoleft 1.111 \NC \righttoleft \setalignmentcharacter{.}1.111 \NC\NR
+\NC              1.111 \NC              \setalignmentcharacter{.}1.111 \NC\NR
+\stoptabulate
+\stopbuffer
+
+{\switchtobodyfont[8pt] \typebuffer[option=TEX]} \getbuffer
+
+In case of doubt, look at this:
+
+\startbuffer
+\starttabulate[|lG{.}|lG{}|lG{.}|lG{}|]
+\NC \righttoleft 1.1   \NC \righttoleft \setalignmentcharacter{.}1.1   \NC
+                 1.1   \NC              \setalignmentcharacter{.}1.1   \NC\NR
+\NC \righttoleft 1.11  \NC \righttoleft \setalignmentcharacter{.}1.11  \NC
+                 1.11  \NC              \setalignmentcharacter{.}1.11  \NC\NR
+\NC \righttoleft 1.111 \NC \righttoleft \setalignmentcharacter{.}1.111 \NC
+                 1.111 \NC              \setalignmentcharacter{.}1.111 \NC\NR
+\stoptabulate
+\stopbuffer
+
+{\switchtobodyfont[8pt] \typebuffer[option=TEX]} \getbuffer
+
+The next example shows the effect of \type {\omit} and \type {\span}. The first one
+makes that in this cell the preamble template is ignored.
+
+\startbuffer
+\halign \bgroup
+    \tabsize 2cm\relax     [\alignmark]\hss \aligntab
+    \tabsize 2cm\relax \hss[\alignmark]\hss \aligntab
+    \tabsize 2cm\relax \hss[\alignmark]\cr
+           1\aligntab       2\aligntab       3\cr
+     \omit 1\aligntab \omit 2\aligntab \omit 3\cr
+           1\aligntab       2\span           3\cr
+           1\span           2\aligntab       3\cr
+           1\span           2\span           3\cr
+           1\span \omit     2\span \omit     3\cr
+     \omit 1\span \omit     2\span \omit     3\cr
+\egroup
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+Spans are applied at the end so you see a mix of templates applied.
+
+{\showboxes\getbuffer[blownup-2]}
+
+When you define an alignment inside a macro, you need to duplicate the \type {\alignmark}
+signals. This is similar to embedded macro definitions. But in \LUAMETATEX\ we can get
+around that by using \type {\aligncontent}. Keep in mind that when the preamble is scanned there
+is no doesn't expand with the exception of the token after \type {\span}.
+
+\startbuffer
+\halign \bgroup
+    \tabsize 2cm\relax     \aligncontent\hss \aligntab
+    \tabsize 2cm\relax \hss\aligncontent\hss \aligntab
+    \tabsize 2cm\relax \hss\aligncontent\cr
+    1\aligntab 2\aligntab 3\cr
+    A\aligntab B\aligntab C\cr
+\egroup
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+\blank\getbuffer\blank
+
+In this example we still have to be verbose in the way we align but we can do this:
+
+\startbuffer
+\halign \bgroup
+    \tabsize 2cm\relax \aligncontentleft  \aligntab
+    \tabsize 2cm\relax \aligncontentmiddle\aligntab
+    \tabsize 2cm\relax \aligncontentright \cr
+    1\aligntab 2\aligntab 3\cr
+    A\aligntab B\aligntab C\cr
+\egroup
+\stopbuffer
+
+\typebuffer[option=TEX]
+
+Where the helpers are defined as:
+
+\starttyping[option=TEX]
+\noaligned\protected\def\aligncontentleft
+  {\ignorespaces\aligncontent\unskip\hss}
+
+\noaligned\protected\def\aligncontentmiddle
+  {\hss\ignorespaces\aligncontent\unskip\hss}
+
+\noaligned\protected\def\aligncontentright
+  {\hss\ignorespaces\aligncontent\unskip}
+\stoptyping
+
+The preamble scanner see such macros as candidates for a single level expansion
+so it will inject the meaning and see the \type {\aligncontent} eventually.
+
+\blank\getbuffer\blank
+
+The same effect could be achieved by using the \type {\span} prefix:
+
+\starttyping[option=TEX]
+\def\aligncontentleft{\ignorespaces\aligncontent\unskip\hss}
+
+\halign { ... \span\aligncontentleft ...}
+\stoptyping
+
+One of the reasons for not directly using the low level \type {\halign} command is
+that it's a lot of work but by providing a set of helpers like here might change
+that a bit. Keep in mind that much of the above is not new in the sense that we
+could not achieve the same already, it's just a bit programmer friendly.
+
+\stopsection
+
+\startsection[title=Remark]
+
+It can be that the way alignments are interfaced with respect to attributes is a bit
+different between \LUATEX\ and \LUAMETATEX\ but because the former is frozen (in
+order not to interfere with current usage patterns) this is something that we will
+deal with deep down in \CONTEXT\ \LMTX.
+
+In principle we can have hooks into the rows for pre and post material but it
+doesn't really pay of as grouping will still interfere. So for now I decided not
+to add these.
+
+\stopsection
+
+\stopdocument
diff --git a/doc/context/sources/general/manuals/lowlevel/lowlevel-expansion.tex b/doc/context/sources/general/manuals/lowlevel/lowlevel-expansion.tex
index 433632e2d..98f533859 100644
--- a/doc/context/sources/general/manuals/lowlevel/lowlevel-expansion.tex
+++ b/doc/context/sources/general/manuals/lowlevel/lowlevel-expansion.tex
@@ -1,5 +1,8 @@
 % language=us runpath=texruns:manuals/lowlevel
 
+% This is work in progress and after an initial draft got extended because of the
+% 2021 meeting. It will hopefully improve over time.
+
 \usemodule[system-tokens]
 
 \environment lowlevel-style
@@ -10,13 +13,15 @@
 
 \startsection[title=Preamble]
 
-% \startsubsection[title=Introduction]
-% \stopsubsection
-
 This short manual demonstrates a couple of properties of the macro language. It
-is not the in|-|depth philosophical expose about macro languages, tokens,
+is not an in|-|depth philosophical expose about macro languages, tokens,
 expansion and such that some \TEX ies like. I prefer to stick to the practical
-aspects.
+aspects. Occasionally it will be technical but you can just skip those paragraphs
+(or later return to them) when you can't follow the explanation. It's often not
+that relevant. I won't talk in terms of mouth, stomach and gut the way the \TEX
+book does and although there is no way to avoid the word \quote {token} I will do
+my best to not complicate matters by too much token speak. Examples show best
+what we mean.
 
 \stopsection
 
@@ -24,8 +29,8 @@ aspects.
 
 The \TEX\ language provides quite some commands and those built in are called
 primitives. User defined commands are called macros. A macro is a shortcut to a
-list of primitives, macro calls. All can be mixed with characters that are to
-be typeset somehow.
+list of primitives and|/|or macro calls. All can be mixed with characters that
+are to be typeset somehow.
 
 \starttyping[option=TEX]
 \def\MyMacro{b}
@@ -40,9 +45,79 @@ macro. In this case it sees just an \type {b} and it will give this the same
 treatment as the \type {a}. The macro ends, the input level decrements and the
 \type {c} gets its treatment.
 
-A macro can contain references to macros so in practice the input can go several
-levels up and some applications push back a lot so this is why your \TEX\ input
-stack can be configured.
+Before we move on to more examples and differences between engines, it is good to
+stress that \type {\MyMacro} is not a primitive command: we made our command
+here. The \type {b} actually can be seen as a sort of primitive because in this
+macro it gets stored as so called token with a primitive property. That primitive
+property can later on be used to determine what to do. More explicit examples of
+primitives are \type {\hbox}, \type {\advance} and \type {\relax}. It will be
+clear that \CONTEXT\ extends the repertoire of primitive commands with a lot of
+macro commands. When we typeset a source using module \type {m-scite} the
+primitives come out dark blue.
+
+The amount of primitives differs per engine. It all starts with \TEX\ as written
+by Don Knuth. Later \ETEX\ added some more primitives and these became official
+extensions adopted by other variants of \TEX. The \PDFTEX\ engine added quite
+some and as follow up on that \LUATEX\ added more but didn't add all of \PDFTEX.
+A few new primitives came from \OMEGA\ (\ALEPH). The \LUAMETATEX\ engine drops a
+set of primitives that comes with \LUATEX\ and adds plenty new ones. The nature
+of this engine (no backend and less frontend) makes that we need to implement
+some primitives as macros. But the basic set is what good old \TEX\ comes with.
+
+Internally these so called primitives are grouped in categories that relate to
+their nature. They can be directly expanded (a way of saying that they get
+immediately interpreted) or delayed (maybe stored for later usage). They can
+involve definitions, calculations, setting properties and values or they can
+result in some typesetting. This is what makes \TEX\ confusing to new users: it
+is a macro programming language, an interpreter but at the same time an executor
+of typesetting instructions.
+
+A group of primitives is internally identified as a command (they have a \type
+{cmd} code) and the sub commands are flagged by their \type {chr} code. This
+sounds confusing but just thing of the fact that most of what we input are
+characters and therefore they make up most sub commands. For instance the \quote
+{letter \type {cmd}} is used for characters that are seen as letters that can be
+used in the name of user commands, can be typeset, are valid for hyphenation
+etc.\ The letter related \type {cmd} can have many \type {chr} codes (all of
+\UNICODE). I'd like to remark that the grouping is to a large extend functional,
+so sometimes primitives that you expect to be similar in nature are in different
+groups. This has to do with the fact that \TEX\ needs to be a able to determine
+efficiently if a primitive is operating (or forbidden) in horizontal, vertical
+and|/|or math mode.
+
+There are more than 150 internal \type {cmd} groups. if we forget about the
+mentioned character related ones, some, have only a few sub commands (\type
+{chr}) and others many more (just consider all the \OPENTYPE\ math spacing
+related parameters). A handful of these commands deal with what we call macros:
+user defined combinations of primitives and other macros, consider them little
+programs. The \type {\MyMacro} example above is an example. There are differences
+between engines. In standard \TEX\ there are \type {\outer} and \type {\long}
+commands, and most engines have these. However, in \LUAMETATEX\ the later to be
+discussed \type {\protected} macros have their own specific \quote {call \type
+{cmd}}. Users don't need to bother about this.
+
+So, when from now on we talk about primitives, we mean the built in, hard coded
+commands, and when we talk about macros we mean user commands. Although
+internally there are less \type {cmd} categories than primitives, from the
+perspective of the user they are all unique. Users won't consult the source
+anyway but when they do they are warned. Also, when in \LUAMETATEX\ you use the
+low level interfacing to \TEX\ you have to figure out these subtle aspects
+because there this grouping does matter.
+
+Before we continue I want to make clear that expansion (as discussed in this
+document) can refer to a macro being expanded (read: its meaning gets injected
+into the input, so the engine kind of sidetracks from what is was doing) but also
+to direct consequences of running into a primitive. However, users only need to
+consider expansion in the perspective of macros. If a user has \type {\advance}
+in the input it immediately gets done. But when it's part of a macro definition
+it only is executed when the macro expands. A good check in (traditional) \TEX\
+is to compare what happens in \type {\def} and \type {\edef} which is why we will
+use these two in the upcoming examples. You put something in a macro and then
+check what \type {\meaning} or \type {\show} reports.
+
+Now back to user defined macros. A macro can contain references to macros so in
+practice the input can go several levels up and some applications push back a lot
+so this is why your \TEX\ input stack can be configured to be huge.
 
 \starttyping[option=TEX]
 \def\MyMacroA{ and }
@@ -52,11 +127,11 @@ a\MyMacroA b
 \stoptyping
 
 When \type {\MyMacroB} is defined, its body gets three so called tokens: the
-character token \type {a} with property \quote {other}, a token that is a
+character token \type {1} with property \quote {other}, a token that is a
 reference to the macro \type {\MyMacroB}, and a character token \type {2}, also
-with property \quote {other} The meaning of \type {\MyMacroA} became five tokens:
+with property \quote {other} The meaning of \type {\MyMacroA} is five tokens:
 a reference to a space token, then three character tokens with property \quote
-{letter}, and finally again a space token.
+{letter}, and finally a space token.
 
 \starttyping[option=TEX]
 \def \MyMacroA{ and }
@@ -65,8 +140,8 @@ a reference to a space token, then three character tokens with property \quote
 a\MyMacroA b
 \stoptyping
 
-In the previous example an \type {\edef} is used, where the \type {e} indicates
-expansion. This time the meaning gets expanded. So we get effectively the same
+In the second definition an \type {\edef} is used, where the \type {e} indicates
+expansion. This time the meaning gets expanded immediately. So we get effectively the same
 as in:
 
 \starttyping[option=TEX]
@@ -123,7 +198,7 @@ Let's assume that \type {\scratchcounter} is zero to start with and use \type
 \meaning\MyMacroC
 \stoplines
 
-So, this time the third macro has basically its meaning frozen, but we can
+So, this time the third macro has its meaning frozen, but we can
 prevent this by applying a \type {\noexpand} when we do this:
 
 \startbuffer
@@ -156,7 +231,7 @@ their meanings).
 \def\MyShow{\quotation {\strut \inlinebuffer \expandafter \typ \expandafter
 {\the\scratchtoks}\strut}}
 
-You can also store tokens in a so called token register. Here we use a predefined
+You can also store tokens in a so-called token register. Here we use a predefined
 scratch register:
 
 \startbuffer
@@ -199,12 +274,12 @@ Indeed the macro gets expanded but only one level: \MyShow. Compare this with:
 
 \typebuffer[option=TEX]
 
-The trick is to expand in two steps: \MyShow. Later we will see that other
+The trick is to expand in two steps with an intermediate \type {\edef}: \MyShow. Later we will see that other
 engines provide some more expansion tricks. The only way to get some grip on
 expansion is to just play with it.
 
-The \type {\expandafter} primitive expands the token (which can be a macro) after
-the next next one and injects its meaning into the stream. So:
+The \type {\expandafter} primitive expands the token (which can be a macro) standing after
+the next next one and then injects its meaning into the stream. So:
 
 \starttyping[option=TEX]
 \expandafter \MyMacroA \MyMacroB
@@ -271,7 +346,7 @@ These macros have the following meanings:
 \meaning\MyMacroABC
 \stoplines
 
-In \CONTEXT\ you will use the \type {\unexpanded} prefix instead because that one
+In \CONTEXT\ you will use the \type {\unexpanded} prefix instead, because that one
 did something similar in older versions of \CONTEXT. As we were early adopters of
 \ETEX, this later became a synonym to the \ETEX\ primitive.
 
@@ -286,17 +361,17 @@ did something similar in older versions of \CONTEXT. As we were early adopters o
 
 Here the wrapper around the token register assignment will expand the three
 macros, unless they are protected, so its content becomes \MyShow. This saves
-either a lot of more complex \type {\expandafter} usage or using an intermediate
+either a lot of more complex \type {\expandafter} usage or the need to use an intermediate
 \type {\edef}. In \CONTEXT\ the \type {\expanded} macro does something simpler
-but it doesn't expand the first token as it is meant as a wrapper around a command,
+but it doesn't expand the first token as this is meant as a wrapper around a command,
 like:
 
 \starttyping[option=TEX]
 \expanded{\chapter{....}} % a ConTeXt command
 \stoptyping
 
-where we do want to expand the title but not the \type {\chapter} command, not
-that this would happen actually because \type {\chapter} is a protected command.
+where we do want to expand the title but not the \type {\chapter} command (not
+that this would happen actually because \type {\chapter} is a protected command.)
 
 The counterpart of \type {\normalexpanded} is \type {\normalunexpanded}, as in:
 
@@ -325,7 +400,7 @@ close, with:
 \typebuffer[option=TEX]
 
 This (when typeset monospaced) is: {\tt \inlinebuffer}. The detokenizer is like
-\type {\string} applied to each token in its argument. Compare this:
+\type {\string} applied to each token in its argument. Compare this to:
 
 \startbuffer
 \normalexpanded {
@@ -447,8 +522,8 @@ We now get:
 \type{\TestD} : {\tttf \meaningless\TestD}
 \stoplines
 
-There are however cases where one wishes this to happen automatically but that
-will also make protected macros expand that create havoc, like switching fonts.
+There are however cases where one wishes this to happen automatically, but that
+will also make protected macros expand which will create havoc, like switching fonts.
 
 \startbuffer
                \def\TestA{A}
@@ -473,8 +548,8 @@ This time \type {\TestC} looses its protection:
 \stoplines
 
 Actually adding \type {\fullyexpanded} would be trivial but it makes not much
-sense to add that overhead (at least not now). This feature is experimental
-anyway so it might go away when I see no real advantage.
+sense to add the overhead (at least not now). This feature is experimental
+anyway so it might go away when I see no real advantage from it.
 
 When you store something in a macro or token register you always need to keep an
 eye on category codes. A dollar in the input is normally treated as math shift, a
@@ -494,19 +569,19 @@ but a character flagged as \quote {other}.
 A whole new category of commands has to do with so called local control. When
 \TEX\ scans and interprets the input, a process takes place that is called
 tokenizing: (sequences of) characters get a symbolic representation and travel
-through the system as tokens. Often they immediately get interpreted and then
-discarded, but when for instance you define a macro they end up as a linked list
+through the system as tokens. Often they immediately get interpreted and are then
+discarded. But when for instance you define a macro they end up as a linked list
 of tokens in the macro body. We already saw that expansion plays a role. In most
-cases, unless \TEX\ is collecting tokens, the main action is dealt with in the so
-called main loop. Something gets picked up from the input but can also be pushed
-back, for instance because of some lookahead that didn't result in some action.
-Quite some time is spent in pushing and popping from the so called input stack.
+cases, unless \TEX\ is collecting tokens, the main action is dealt with in the so-called
+main loop. Something gets picked up from the input but can also be pushed
+back, for instance because of some lookahead that didn't result in an action.
+Quite some time is spent in pushing and popping from the so-called input stack.
 
 When we are in \LUA, we can pipe back into the engine but all is collected till
 we're back in \TEX\ where the collected result is pushed into the input. Because
 \TEX\ is a mix of programming and action there basically is only that main loop.
 There is no real way to start a sub run in \LUA\ and do all kind of things
-independent of the current run. This makes sense when you consider the mix: it
+independent of the current one. This makes sense when you consider the mix: it
 would get too confusing.
 
 However, in \LUATEX\ and even better in \LUAMETATEX, we can enter a sort of local
@@ -662,7 +737,7 @@ When you want to add some grouping, it quickly can become less pretty:
 
 \getbuffer[def]\getbuffer[use]
 
-A single token alternative is available too and its usage us like this:
+A single token alternative is available too and its usage is like this:
 
 \startbuffer
  \def\TestA{\scratchcounter=100 }
@@ -752,7 +827,7 @@ context.runstring([[\setbox0\hbox{(Here \bf 1.2345)}]])
 context.runstring([[\setbox0\hbox{(Here \bf   %.3f)}]],1.2345)
 \stoptyping
 
-Before we had the string runner this was the way to do it when staying in \LUA\
+Before we had \type {runstring} this was the way to do it when staying in \LUA\
 was needed:
 
 \startbuffer[run]
@@ -780,7 +855,7 @@ context("[8: %p]",tex.box[0].width)
 \start \getbuffer[run] \stop
 
 The order of flushing matters because as soon as something is not stored in a
-token list or macro body, \TEX\ will typeset it. And as said, a lot this relates
+token list or macro body, \TEX\ will typeset it. And as said, a lot of this relates
 to pushing stuff into the input which is stacked. Compare:
 
 \startbuffer[run]
@@ -809,7 +884,7 @@ tex.pushlocal() context("[HERE 2]") tex.poplocal()
 
 You can expand a macro at the \LUA\ end with \type {token.expandmacro} which has
 a peculiar interface. The first argument has to be a string (the name of a macro)
-or a user data (a valid macro token). This macro can be fed with parameters by
+or a userdata (a valid macro token). This macro can be fed with parameters by
 passing more arguments:
 
 \starttabulate[|||]
@@ -840,15 +915,15 @@ primitive(s):
 
 \typebuffer[run][option=TEX]
 
-The \type {\tokenized} takes an optionally keyword and the examples above give: {\tttf
+The \type {\tokenized} takes an optional keyword and the examples above give: {\tttf
 \getbuffer[run]}. The \LUATEX\ primitive \type {\scantextokens} which is a
-variant of \ETEX's \type {\scantokens} operate under the current catcode regime
+variant of \ETEX's \type {\scantokens} operates under the current catcode regime
 (the last one honors \type {\everyeof}). The difference with \type {\tokenized}
 is that this one first serializes the given token list (just like \type
 {\detokenize}). \footnote {The \type {\scan*tokens} primitives now share the same
 helpers as \LUA, but they should behave the same as in \LUATEX.}
 
-With \type {\retokenized} the catcode table index is mandate (it saves a bit of
+With \type {\retokenized} the catcode table index is mandatory (it saves a bit of
 scanning and is easier on intermixed \type {\expandafter} usage. There
 often are several ways to accomplish the same:
 
@@ -865,10 +940,10 @@ often are several ways to accomplish the same:
 \getbuffer[run]
 
 Here the numbers show the relative performance of these methods. The \type
-{\detokenize} and \type {\meaningless} win because they already know that that a
+{\detokenize} and \type {\meaningless} win because they already know that a
 verbose serialization is needed. The last two first serialize and then
 reinterpret the resulting token list using the given catcode regime. The last one
-is slowest because has to scan the keyword.
+is slowest because it has to scan the keyword.
 
 There is however a pitfall here:
 
@@ -884,7 +959,7 @@ There is however a pitfall here:
 \typebuffer[run][option=TEX]
 
 The outcome is different now because we have an expandable embedded macro call.
-The fact that we expand in the last two primitives is also a reason why they are
+The fact that we expand in the last two primitives is also the reason why they are
 \quote {slower}.
 
 \getbuffer[run]
@@ -901,7 +976,7 @@ introduced in this section:
 \retokenized              \notcatcodes{\MyTitle}\crlf
 \retokenized              \notcatcodes{\semiexpanded{\MyTitle}}\crlf
 \tokenized   catcodetable \notcatcodes{\MyTitle}\crlf
-\tokenized   catcodetable \notcatcodes{\semiexpanded{\MyTitle}}\crlf
+\tokenized   catcodetable \notcatcodes{\semiexpanded{\MyTitle}}
 \stopbuffer
 
 \typebuffer[run][option=TEX]
@@ -911,7 +986,7 @@ This time compare the last four lines:
 \getbuffer[run]
 
 Of course the question remains to what extend we need this and eventually will
-apply it in \CONTEXT. The \type {\detokenize} is used already. History shows that
+apply in \CONTEXT. The \type {\detokenize} is used already. History shows that
 eventually there is a use for everything and given the way \LUAMETATEX\ is
 structured it was not that hard to provide the alternatives without sacrificing
 performance or bloating the source.
@@ -926,6 +1001,123 @@ performance or bloating the source.
 
 \stopsection
 
+\startsection[title=Dirty tricks]
+
+When I was updating this manual Hans vd Meer and I had some discussions about
+expansion and tokenization related issues when combining of \XML\ processing with
+\TEX\ macros where he did some manipulations in \LUA. In these mixed cases you
+can run into catcode related problems because in \XML\ you want for instance a
+\type {#} to be a hash mark (other character) and not an parameter identifier.
+Normally this is handled well in \CONTEXT\ but of course there are complex cases
+where you need to adapt.
+
+Say that you want to compare two strings (officially we should say token lists)
+with mixed catcodes. Let's also assume that you want to use the normal \type
+{\if} construct (which was part of the discussion). We start with defining
+a test set. The reason that we present this example here is that we use
+commands discussed in previous sections:
+
+\startbuffer[run]
+               \def\abc{abc}
+\semiprotected \def\xyz{xyz}
+              \edef\pqr{\expandtoken\notcatcodes`p%
+                        \expandtoken\notcatcodes`q%
+                        \expandtoken\notcatcodes`r}
+
+1: \ifcondition\similartokens{abc} {def}YES\else NOP\fi (NOP) \quad
+2: \ifcondition\similartokens{abc}{\abc}YES\else NOP\fi (YES)
+
+3: \ifcondition\similartokens{xyz} {pqr}YES\else NOP\fi (NOP) \quad
+4: \ifcondition\similartokens{xyz}{\xyz}YES\else NOP\fi (YES)
+
+5: \ifcondition\similartokens{pqr} {pqr}YES\else NOP\fi (YES) \quad
+6: \ifcondition\similartokens{pqr}{\pqr}YES\else NOP\fi (YES)
+\stopbuffer
+
+\typebuffer[run][option=TEX]
+
+So, we have a mix of expandable and semi expandable macros, and also a mix of
+catcodes. A naive approach would be:
+
+\startbuffer[def]
+\permanent\protected\def\similartokens#1#2%
+  {\iftok{#1}{#2}}
+\stopbuffer
+
+\typebuffer[def][option=TEX]
+
+but that will fail on some cases:
+
+\pushoverloadmode \startpacked \tttf \getbuffer[def,run]\stoppacked \popoverloadmode
+
+So how about:
+
+\startbuffer[def]
+\permanent\protected\def\similartokens#1#2%
+  {\iftok{\detokenize{#1}}{\detokenize{#2}}}
+\stopbuffer
+
+\typebuffer[def][option=TEX]
+
+That one is even worse:
+
+\pushoverloadmode \startpacked \tttf \getbuffer[def,run]\stoppacked \popoverloadmode
+
+We need to expand so we end up with this:
+
+\startbuffer[def]
+\permanent\protected\def\similartokens#1#2%
+  {\normalexpanded{\noexpand\iftok
+     {\noexpand\detokenize{#1}}
+     {\noexpand\detokenize{#2}}}}
+\stopbuffer
+
+\typebuffer[def][option=TEX]
+
+Better:
+
+\pushoverloadmode \startpacked \tttf \getbuffer[def,run]\stoppacked \popoverloadmode
+
+But that will still not deal with the mildly protected macro so in the end we
+have:
+
+\startbuffer[def]
+\permanent\protected\def\similartokens#1#2%
+  {\semiexpanded{\noexpand\iftok
+     {\noexpand\detokenize{#1}}
+     {\noexpand\detokenize{#2}}}}
+\stopbuffer
+
+\typebuffer[def][option=TEX]
+
+Now we're good:
+
+\pushoverloadmode \startpacked \tttf \getbuffer[def,run]\stoppacked \popoverloadmode
+
+Finally we wrap this one in the usual \type {\doifelse...} macro:
+
+\startbuffer[def]
+\permanent\protected\def\doifelsesimilartokens#1#2%
+  {\ifcondition\similartokens{#1}{#2}%
+     \expandafter\firstoftwoarguments
+   \else
+     \expandafter\secondoftwoarguments
+   \fi}
+\stopbuffer
+
+\typebuffer[def][option=TEX]
+
+so that we can do:
+
+\starttyping[option=TEX]
+\doifelsesimilartokens{pqr}{\pqr}{YES}{NOP}
+\stoptyping
+
+A companion macro of this is \type {\wipetoken} but for that one you need to look
+into the source.
+
+\stopsection
+
 \stopdocument
 
 % \aftergroups
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex.tex b/doc/context/sources/general/manuals/luametatex/luametatex.tex
index 3c02391ce..a179dcb4c 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex.tex
@@ -87,6 +87,9 @@
 
 % with mimalloc and msvc we get a better native performance than crosscompiled
 
+% \enableexperiments [tabulateusesize]
+% \enableexperiments [tabulatesparseskips]
+
 \enableexperiments[fonts.compact]
 % \enabledirectives[fonts.injections.method=advance] % tricky ... not all xoffsets are advance robust