2021-04-18 18:07:00

7 files changed, 262 insertions, 19 deletions

diff --git a/doc/context/documents/general/manuals/lowlevel-macros.pdf b/doc/context/documents/general/manuals/lowlevel-macros.pdf
index 63689f1c8..9894d0484 100644
--- a/doc/context/documents/general/manuals/lowlevel-macros.pdf
+++ b/doc/context/documents/general/manuals/lowlevel-macros.pdf
diff --git a/doc/context/documents/general/manuals/luametatex.pdf b/doc/context/documents/general/manuals/luametatex.pdf
index cc0e821fb..b65e6f6ba 100644
--- a/doc/context/documents/general/manuals/luametatex.pdf
+++ b/doc/context/documents/general/manuals/luametatex.pdf
diff --git a/doc/context/sources/general/manuals/lowlevel/lowlevel-macros.tex b/doc/context/sources/general/manuals/lowlevel/lowlevel-macros.tex
index ef527732b..033bdee80 100644
--- a/doc/context/sources/general/manuals/lowlevel/lowlevel-macros.tex
+++ b/doc/context/sources/general/manuals/lowlevel/lowlevel-macros.tex
@@ -14,7 +14,6 @@
 
 % \permanent\tolerant\protected\def\xx[#1]#*#;[#2]#:#3% loops .. todo
 
-
 \usemodule[system-tokens]
 
 \environment lowlevel-style
@@ -540,9 +539,30 @@ a use for \type {#<} and \type {#>}. A summary of all this is given here:
 \NC *   \NC ignore spaces \NC \NR
 \NC :   \NC pick up scanning here  \NC \NR
 \NC ;   \NC quit scanning \NC \NR
+\ML
+\NC .   \NC ignore pars and spaces \NC \NR
+\NC ,   \NC push back space when quit \NC \NR
 \LL
 \stoptabulate
 
+The last two have not been discussed and were added later. The period
+directive gobbles space and par tokens and discards them in the
+process. The comma directive is like \type {*} but it pushes back a space
+when the matching quits.
+
+\startbuffer
+\tolerant\def\FooA[#1]#*[#2]{(#1/#2)} % remove spaces
+\tolerant\def\FooB[#1]#,[#2]{(#1/#2)} % push back space
+
+/\FooA/ /\FooA / /\FooA[1]/ /\FooA[!] / /\FooA[1] [2]/ /\FooA[1] [2] /\par
+/\FooB/ /\FooB / /\FooB[1]/ /\FooB[!] / /\FooB[1] [2]/ /\FooB[1] [2] /\par
+\stopbuffer
+
+\typebuffer[example][option=TEX] \startpacked \getbuffer[example] \stoppacked
+
+Gobbling spaces versus pushing back is an interface design decision because it
+has to do with consistency.
+
 \stopsection
 
 \startsection[title=Runaway arguments]
@@ -639,6 +659,8 @@ macro is a special version of the online variant:
 That one is always available and shows a similar list on the console. Again, users
 normally don't want to know such details.
 
+\stopsection
+
 \startsection[title=nesting]
 
 You can nest macros, as in:
@@ -812,10 +834,10 @@ these can easily be fixed. So, it will be used but it will take a while to adapt
 the code base.
 
 One problem with frozen macros is that they don't play nice with for instance
-\type {\futurelet}. Also, there are places in \CONTEXT\ where we actually do
+\typ {\futurelet}. Also, there are places in \CONTEXT\ where we actually do
 redefine some core macro that we also want to protect from redefinition by a
-user. One can of course \type {\unletfrozen} such a command first but as a bonus
-we have a prefix \type {\overloaded} that can be used as prefix. So, one can easily
+user. One can of course \typ {\unletfrozen} such a command first but as a bonus
+we have a prefix \typ {\overloaded} that can be used as prefix. So, one can easily
 redefine a frozen macro but it takes a little effort. After all, this feature is
 mainly meant to protect a user for side effects of definitions, and not as final
 blocker. \footnote {As usual adding features like this takes some experimenting
@@ -824,7 +846,7 @@ The fact that we can apply such features in large macro package like \CONTEXT\
 helps figuring out the needs and best approaches.}
 
 A frozen macro can still be overloaded, so what if we want to prevent that? For
-this we have the \type {\permanent} prefix. Internally we also create primitives
+this we have the \typ {\permanent} prefix. Internally we also create primitives
 but we don't have a prefix for that. But we do have one for a very special case
 which we demonstrate with an example:
 
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex b/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
index 4681f6bea..b60079477 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
@@ -316,21 +316,29 @@ a bitset made from the following values, some of which we saw in the previous
 examples.
 
 \starttabulate[|l|p|]
-\NC \number \normalhyphenationmodecode           \NC honour (normal) \prm{discretionary}'s \NC \NR
-\NC \number \automatichyphenationmodecode        \NC turn \type {-} into (automatic) discretionaries \NC \NR
-\NC \number \explicithyphenationmodecode         \NC turn \type {\-} into (explicit) discretionaries \NC \NR
-\NC \number \syllablehyphenationmodecode         \NC hyphenate (syllable) according to language \NC \NR
-\NC \number \uppercasehyphenationmodecode        \NC hyphenate uppercase characters too (replaces \prm {uchyph} \NC \NR
-\NC \number \compoundhyphenationmodecode         \NC permit break at an explicit hyphen (border cases) \NC \NR
-\NC \number \strictstarthyphenationmodecode      \NC traditional \TEX\ compatibility wrt the start of a word \NC \NR
-\NC \number \strictendhyphenationmodecode        \NC traditional \TEX\ compatibility wrt the end of a word \NC \NR
-\NC \number \automaticpenaltyhyphenationmodecode \NC use \lpr {automatichyphenpenalty} \NC \NR
-\NC \number \explicitpenaltyhyphenationmodecode  \NC use \lpr {explicithyphenpenalty} \NC \NR
-\NC \number \permitgluehyphenationmodecode       \NC turn glue in discretionaries into kerns \NC \NR
-\NC \number \lazyligatureshyphenationmodecode    \NC controls how successive explicit discretionaries are handled in base mode \NC \NR
-\NC \number \forcecheckhyphenationmodecode       \NC treat all discretionaries equal when breaking lines (in all three passes) \NC \NR
+\NC \number \normalhyphenationmodecode            \NC honour (normal) \prm{discretionary}'s \NC \NR
+\NC \number \automatichyphenationmodecode         \NC turn \type {-} into (automatic) discretionaries \NC \NR
+\NC \number \explicithyphenationmodecode          \NC turn \type {\-} into (explicit) discretionaries \NC \NR
+\NC \number \syllablehyphenationmodecode          \NC hyphenate (syllable) according to language \NC \NR
+\NC \number \uppercasehyphenationmodecode         \NC hyphenate uppercase characters too (replaces \prm {uchyph} \NC \NR
+\NC \number \compoundhyphenationmodecode          \NC permit break at an explicit hyphen (border cases) \NC \NR
+\NC \number \strictstarthyphenationmodecode       \NC traditional \TEX\ compatibility wrt the start of a word \NC \NR
+\NC \number \strictendhyphenationmodecode         \NC traditional \TEX\ compatibility wrt the end of a word \NC \NR
+\NC \number \automaticpenaltyhyphenationmodecode  \NC use \lpr {automatichyphenpenalty} \NC \NR
+\NC \number \explicitpenaltyhyphenationmodecode   \NC use \lpr {explicithyphenpenalty} \NC \NR
+\NC \number \permitgluehyphenationmodecode        \NC turn glue in discretionaries into kerns \NC \NR
+\NC \number \permitallhyphenationmodecode         \NC okay, let's be even more tolerant in discretionaries \NC \NR
+\NC \number \permitmathreplacehyphenationmodecode \NC and again we're more permissive \NC \NR
+\NC \number \lazyligatureshyphenationmodecode     \NC controls how successive explicit discretionaries are handled in base mode \NC \NR
+\NC \number \forcecheckhyphenationmodecode        \NC treat all discretionaries equal when breaking lines (in all three passes) \NC \NR
+\NC \number \forcehandlerhyphenationmodecode      \NC kick in the handler (experiment) \NC \NR
+\NC \number \feedbackcompoundhyphenationmodecode  \NC feedback compound snippets \NC \NR
 \stoptabulate
 
+Some of these options are still experimental, simply because not all aspects and
+side effects have been explored. You can find some experimental use cases in
+\CONTEXT.
+
 \stopsection
 
 \startsection[title={The main control loop}]
diff --git a/doc/context/sources/general/manuals/math/math-grouping.tex b/doc/context/sources/general/manuals/math/math-grouping.tex
new file mode 100644
index 000000000..cc79ff198
--- /dev/null
+++ b/doc/context/sources/general/manuals/math/math-grouping.tex
@@ -0,0 +1,212 @@
+% language=uk
+
+\environment math-layout
+
+\startbuffer[oldcolor]
+    \disableexperiments[simplegroups]%
+    % when we no longer have it as experiment
+    \unprotect \pushoverloadmode
+    \permanent\protected\def\color[#1]%
+      {\bgroup % \beginsimplegroup
+       \edef\currentcolorname{#1}%
+       \ifempty\currentcolorprefix
+         \colo_helpers_activate_nop
+       \else
+         \colo_helpers_activate_yes
+       \fi
+       \let\nexttoken}%
+    \popoverloadmode \protect
+\stopbuffer
+
+\startbuffer[newcolor]
+    \enableexperiments[simplegroups]%
+\stopbuffer
+
+\startcomponent math-grouping
+
+\startchapter[title=Grouping]
+
+\startsection[title=Some details]
+
+In \TEX\ there are all kind of groups. When you start with a curly brace, you
+often enter a group but when you start a box or table cell you also do that. When
+you enter math a math group is started. Assignments are, unless explicitly done
+global, nearly always local to the group. Here we discuss the following two
+cases:
+
+\starttyping
+{ .... }
+\bgroup .... \egroup
+\begingroup .... \endgroup
+\stoptyping
+
+We say two cases, not three, because the first two are equivalent: the two macros
+in the second line are not primitives but aliases to the curly braces. There is
+however one fundamental difference between them. The verbose \type {\begingroup}
+starts a so called simple group so let's call the other complex. A complex group
+is bounded by equivalents to the two characters (braces) that have catcodes that
+begin and end these groups. So, the following is valid.
+
+\starttyping
+{ .... }
+{ .... \egroup
+\bgroup .... }
+\bgroup .... \egroup
+\stoptyping
+
+This means that a macro like this is okay:
+
+\starttyping
+\def\foo{\bgroup\bf\let\next}  \foo{text}
+\stoptyping
+
+The \type {\foo} will start a complex group, switch font and then pick up the
+brace. The group will be closed by the matching right brace or an equivalent.
+The following two cases are invalid:
+
+\starttyping
+\bgroup .... \endgroup
+\begingroup .... \egroup
+\stoptyping
+
+which means that:
+
+\starttyping
+\def\foo{\begingroup\bf\let\next}  \foo{text}
+\stoptyping
+
+will trigger an error message. This is rather unfortunate because using braces
+to wrap an argument in curly braces is rather convenient. The way out is this:
+
+\starttyping
+\def\foo#1{\begingroup\bf#1\endgroup}  \foo{text}
+\stoptyping
+
+which is perfectly valid apart from the fact that the argument is first picked up
+and then fed back into the input. Apart from a small performance hit and using
+more memory it also adds noise to tracing.
+
+\stopsection
+
+\startsection[title=Side effects]
+
+In math mode matters are complicates by the fact that complex groups (the ones
+started with the curly brace) start a math list. And that has side effects because
+the spacing between math elements depends on what we deal with: math symbol, lists,
+fenced material. The following example shows a whole lot of this:
+
+\startbuffer[example]
+\starttabulate[|j1|j1||]
+    \NC $\sin{\left(xxx\right)}$
+    \NC $f   {\left(xxx\right)}$
+    \NC $x   {\left(xxx\right)}$
+    \NR
+    \NC $\sin\begingroup\left(xxx\right)\endgroup$
+    \NC $f   \begingroup\left(xxx\right)\endgroup$
+    \NC $x   \begingroup\left(xxx\right)\endgroup$
+    \NR
+    \NC $\sin\left(xxx\right)$\par
+    \NC $f   \left(xxx\right)$\par
+    \NC $x   \left(xxx\right)$\par
+    \NR
+    \NC $\sin\color[darkgreen]{(xxx)}$\par
+    \NC $f   \color[darkgreen]{(xxx)}$\par
+    \NC $x   \color[darkgreen]{(xxx)}$\par
+    \NR
+    \NC $\sin\startcolor[darkblue](xxx)\stopcolor$\par
+    \NC $f   \startcolor[darkblue](xxx)\stopcolor$\par
+    \NC $x   \startcolor[darkblue](xxx)\stopcolor$\par
+    \NR
+    \NC $\sin(xxx)$\par
+    \NC $f   (xxx)$\par
+    \NC $x   (xxx)$\par
+    \NR
+    \NC $\sin{\color[darkyellow]{(xxx)}}$\par
+    \NC $f   {\color[darkyellow]{(xxx)}}$\par
+    \NC $x   {\color[darkyellow]{(xxx)}}$\par
+    \NR
+    \NC $\sin\begingroup\color[darkred]{(xxx)}\endgroup$\par
+    \NC $f   \begingroup\color[darkred]{(xxx)}\endgroup$\par
+    \NC $x   \begingroup\color[darkred]{(xxx)}\endgroup$\par
+    \NR
+\stoptabulate
+\stopbuffer
+
+\typebuffer[example]
+
+A valid question is why we would want to add curly braces. One reason is that we
+we want to apply something to a few characters, in this case color the argument
+to the sinus. Now, when a color command is defined as the \type {\foo} before, we
+start a complex group and that influences spacing! This is demonstrated in the
+left part of \in {figure} [fig:grouping:sin] (you can zoom in on the table to
+see the details; we also report the kind of spacing applied).
+
+\startplacefigure[reference=fig:grouping:sin,title=Grouping influencing math spacing (list).]
+    \startcombination[nx=2,ny=1,distance=2em,style=bold]
+        \startcontent
+            \showmakeup[math,fontkern,italic]% laps into the margin hence the -2em
+            \getbuffer[oldcolor]%
+            \scale[width=\dimexpr\measure{combination}-2em\relax]{\getbuffer[example]}%
+        \stopcontent
+        \startcaption
+            pseudo grouping
+        \stopcaption
+        \startcontent
+            \showmakeup[math,fontkern,italic]% laps into the margin hence the -2em
+            \getbuffer[newcolor]%
+            \scale[width=\dimexpr\measure{combination}-1em\relax]{\getbuffer[example]}%
+        \stopcontent
+        \startcaption
+            grouping
+        \stopcaption
+    \stopcombination
+\stopplacefigure
+
+The right variant in that figure uses a different way of grouping, one that is
+equivalent to:
+
+\starttyping
+\def\foo{\beginsimplegroup\bf\let\next}
+\stoptyping
+
+This time we effectively do \type {\begingroup} but permits both \type
+{\endgroup} or a curly brace (or \type {\egroup} to wrap up the group. That means
+that we don't get the side effect of starting a math list.
+
+\startbuffer[example]
+$ x = y \quad x \color[red]{=} y$
+\stopbuffer
+
+This example shows the effect of coloring a single character (the result is shown
+in \in {figure} [fig:grouping:char]):
+
+\typebuffer[example]
+
+\startplacefigure[reference=fig:grouping:char,title=Grouping influencing math spacing (symbol).]
+    \vkern4ex
+    \startcombination[nx=2,ny=1,distance=4em,style=bold]
+        \startcontent
+            \showmakeup[math,fontkern,italic]% laps into the margin hence the -2em
+            \getbuffer[oldcolor]%
+            \scale[width=\dimexpr\measure{combination}-2em\relax]{\getbuffer[example]}%
+        \stopcontent
+        \startcaption
+            pseudo grouping
+        \stopcaption
+        \startcontent
+            \showmakeup[math,fontkern,italic]% laps into the margin hence the -2em
+            \getbuffer[newcolor]%
+            \scale[width=\dimexpr\measure{combination}-2em\relax]{\getbuffer[example]}%
+        \stopcontent
+        \startcaption
+            grouping
+        \stopcaption
+    \stopcombination
+    \vkern2ex
+\stopplacefigure
+
+\stopsection
+
+\stopchapter
+
+\stopcomponent
diff --git a/doc/context/sources/general/manuals/math/math-layout.tex b/doc/context/sources/general/manuals/math/math-layout.tex
index 83137444d..81eb9da62 100644
--- a/doc/context/sources/general/manuals/math/math-layout.tex
+++ b/doc/context/sources/general/manuals/math/math-layout.tex
@@ -13,7 +13,7 @@
 \usebodyfont    [bonum]
 \usebodyfont   [schola]
 
-\setupbodyfont
+\setupbodyfont % don't change this, it's assumed in examples
   [pagella]
 
 \setupwhitespace
diff --git a/doc/context/sources/general/manuals/math/math-mkiv.tex b/doc/context/sources/general/manuals/math/math-mkiv.tex
index 11fb4ca1c..b09cfd2bb 100644
--- a/doc/context/sources/general/manuals/math/math-mkiv.tex
+++ b/doc/context/sources/general/manuals/math/math-mkiv.tex
@@ -42,6 +42,7 @@
     \component math-suboptimal
     \component math-tricks
     \component math-oddities
+    \component math-grouping
 \stopbodymatter
 
 \stopdocument