2023-04-11 22:47:00

14 files changed, 197 insertions, 31 deletions

diff --git a/doc/context/scripts/mkiv/mtx-pdf.html b/doc/context/scripts/mkiv/mtx-pdf.html
index 52c6b5608..3e4541a0c 100644
--- a/doc/context/scripts/mkiv/mtx-pdf.html
+++ b/doc/context/scripts/mkiv/mtx-pdf.html
@@ -44,7 +44,7 @@
         <tr><th>--pretty</th><td></td><td>replace newlines in metadata</td></tr>
         <tr><th>--fonts</th><td></td><td>show used fonts (--detail)</td></tr>
         <tr><th>--object</th><td></td><td>show object"/></td></tr>
-        <tr><th>--linkjs</th><td></td><td>show links"/></td></tr>
+        <tr><th>--links</th><td></td><td>show links"/></td></tr>
         <tr><th/><td/><td/></tr>
     </table>
 <br/>
diff --git a/doc/context/scripts/mkiv/mtx-pdf.man b/doc/context/scripts/mkiv/mtx-pdf.man
index a8fbade52..222261a81 100644
--- a/doc/context/scripts/mkiv/mtx-pdf.man
+++ b/doc/context/scripts/mkiv/mtx-pdf.man
@@ -26,7 +26,7 @@ show used fonts (--detail)
 .B --object
 show object"/>
 .TP
-.B --linkjs
+.B --links
 show links"/>
 .SH AUTHOR
 More information about ConTeXt and the tools that come with it can be found at:
diff --git a/doc/context/scripts/mkiv/mtx-pdf.xml b/doc/context/scripts/mkiv/mtx-pdf.xml
index c4779d876..6b2efdcb8 100644
--- a/doc/context/scripts/mkiv/mtx-pdf.xml
+++ b/doc/context/scripts/mkiv/mtx-pdf.xml
@@ -13,7 +13,7 @@
     <flag name="pretty"><short>replace newlines in metadata</short></flag>
     <flag name="fonts"><short>show used fonts (<ref name="detail)"/></short></flag>
     <flag name="object"><short>show object"/></short></flag>
-    <flag name="linkjs"><short>show links"/></short></flag>
+    <flag name="links"><short>show links"/></short></flag>
    </subcategory>
    <subcategory>
     <example><command>mtxrun --script pdf --info foo.pdf</command></example>
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-building.tex b/doc/context/sources/general/manuals/luametatex/luametatex-building.tex
index ca09c53f9..277d39251 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-building.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-building.tex
@@ -792,6 +792,60 @@ we can freeze them.
 {everybeforepar}, \prm {snapshotpar}, \prm {wrapuppar}, etc. For the moment the
 manuals that come with \CONTEXT\ have to do.}
 
+% The concept of paragraph in \TEX\ can be a bit confusing, and what follows here
+% is even more so. In \LUAMETATEX\ we distinguish three different cases:
+%
+% \startitemize[packed]
+% \starthead {normal}
+%     This state is entered when we have an explicit \type {\par}.
+% \stophead
+% \starthead {inserted}
+%     This state is entered when \TEX\ forces a new paragraph which can happen when
+%     we automatically change to horizontal mode.
+% \stophead
+% \starthead{newline}
+%     This state is entered when an empty line is encoduntered.
+% \stophead
+% \stopitemize
+%
+% When defining macros, a \type {\par} can be a delimiter and an empty line is then
+% equivalent to that. With \type {\autoparagraphmode} we can change this behaviour.
+% The bits set in this variable determines how \type {\par} tokens are interpreted
+% and processed in different situations. It is good to know that this experimental
+% feature is pretty much \CONTEXT\ specific. Here are a few characteristics:
+%
+% \startitemize[packed]
+%     \startitem
+%         When any bit is set, a par token is appended when with property inserted.
+%         The reason behind different properties (mentioned above) is that we can
+%         intercept them with callbacks.
+%     \stopitem
+%     \startitem
+%         When bit~1 (text) is set, \type {\par} will be appended to a string when
+%         a token list is serialized.
+%     \stopitem
+%     \startitem
+%         When bit~2 (macro) is set when a macro is defined there is an explicit
+%         check for a par token when \type {\par} is part of the preamble.
+%     \stopitem
+%     \startitem
+%         When bit~4 (go on) is not set we enter the same state as a new line.
+%         After that when bit~1 (text) is set, a regular par token command is
+%         injected (with an associated newline state), otherwise the meaning of
+%         \type {\par} kicks in (users can have redefined \type {\par}).
+%     \stopitem
+% \stopitemize
+%
+% In \CONTEXT\ we currently default to one, because we still have a few \type {\par}
+% delimited macros but these will go and then we will set the mode to two. This
+% means that in \CONTEXT\ such macro expects an explicit \type {\par} and not an
+% empty line which in turn encourages users to use the proper alternatives. We
+% anyway don't support redefined \type {\par} tokens. Therefore, when we load for
+% instance tikz, we set the mode to zero (normal \TEX) and afterwards back to one.
+% This is still an experimental feature that we occasionally review so don't bother
+% us with questions about it (just don't set the mode).
+
+
 \stopsubsection
 
 \startsubsection[title=Penalties]
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex b/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
index 13bc3a3e3..dd7428e25 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
@@ -248,7 +248,7 @@ input that would have generated an error when processed by older \TEX|-|based
 engines. The affected commands with an altered initial (left of the equal sign)
 or secondary (right of the equal sign) value are: \prm {char}, \prm {lccode},
 \prm {uccode}, \prm {hjcode}, \prm {catcode}, \prm {sfcode}, \prm {efcode}, \prm
-{lpcode}, \prm {rpcode}, \prm {chardef}.
+{cfcode}, \prm {lpcode}, \prm {rpcode}, \prm {chardef}.
 
 As far as the core engine is concerned, all input and output to text files is
 \UTF-8 encoded. Input files can be pre|-|processed using the \type {reader}
@@ -1914,6 +1914,16 @@ by the engine for the same reasons: less code and less tracing clutter.
 
 \stopsubsection
 
+\startsubsection[title={\prm{cdef}, \prm {cdefcsname} and \prm {constant}}]
+
+These primitives are like \prm {edef} and \prm {edefcsname} but tag the macro as
+being kind of simple, which means that in some scenarios they are serialized in a
+fast way, not going through the expansion machinery. This is actually an
+experiment but it will stay. The \prm {constant} prefix can be used with other
+definition primitives instead.
+
+\stopsubsection
+
 \startsubsection[title={\prm {csstring}, \prm {begincsname} and \prm {lastnamedcs}}]
 
 These are somewhat special. The \prm {csstring} primitive is like
@@ -2002,6 +2012,21 @@ defined (or used).
 
 \stopsubsection
 
+\startsubsection[title=Prefixes]
+
+Quite some primitive usage can be preceded by a prefix. For instance assignments
+can be \prm {global} and macros can be defined \prm {protected}. In \LUAMETATEX\
+we have more prefixes, like \prm {tolerant} that signals a different way of
+interpreting macro arguments and \type {permanent} that flags a definition in
+away that, when overload protection is enabled, will prevent redefinition.
+Prefixes like \prm {immediate} and its counterpart \prm {deferred} are backend
+related and, as we don't have one in the engine, are not doing much. They are
+just intercepted and passed to e.g.\ some \LUA\ function called so that one can
+use them to construct additional (\LUA\ based) pseudo primitives. Various
+prefixes are discussed elsewhere.
+
+\stopsubsection
+
 \startsubsection[title=Arguments]
 
 \topicindex {macros+arguments}
@@ -2421,6 +2446,45 @@ omits the unit.
 
 \stopsubsection
 
+\startsubsection[title={Units}]
+
+The familiar \TEX\ units like \type {pt} and \type {cm} are supported but since
+the 2021 \CONTEXT\ meeting we also support the Knuthian Potrzebie, cf.\ \typ
+{en.wikipedia.org/wiki/Potrzebie}. The two character acronym is \type {dk}. One
+\type {dk} is 6.43985pt. This unit is particularly suited for offsets in framed
+examples.
+
+In 2023 we added the Edith (\type {es}) and Tove (\type {ts}) as metric
+replacements for the inch (\type {in}). As with the \type {dk} more background
+information can be found in documents that come with \CONTEXT\ and user group
+journals. The \type {eu} unit starts out as one \type {es} but can be scaled with
+\prm {eufactor}.
+
+\startbuffer
+\localcontrolledloop -5 55 5 {
+    \eufactor=\currentloopiterator
+    \dontleavehmode\strut
+    \vrule height .1es depth .25ts width 1dk\relax\quad
+    \vrule height .1es depth .25ts width 1eu\relax\quad
+    \the\currentloopiterator
+    \par
+}
+\stopbuffer
+
+\typebuffer
+
+This example code shows all four new units. Watch how \prm {eufactor} is clipped
+to a value in the range $1-50$. The default factor of $10$ makes the European
+Unit equivalent to ten Toves or one Edith.
+
+\startpacked
+\startcolor[darkgray]
+\getbuffer
+\stopcolor
+\stoppacked
+
+\stopsubsection
+
 \stopsection
 
 \startsection[title=Expressions]
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex b/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex
index b4e692545..127fd16c1 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-fonts.tex
@@ -130,8 +130,14 @@ optical paragraph optimization:
 \NC \type {leftprotruding}  \NC number  \NC left protruding factor (\prm {lpcode}) \NC\NR
 \NC \type {rightprotruding} \NC number  \NC right protruding factor (\prm {rpcode}) \NC\NR
 \NC \type {expansion}       \NC number  \NC expansion factor (\prm {efcode}) \NC\NR
+\NC \type {compression}     \NC number  \NC compression factor (\prm {cfcode}) \NC\NR
 \stoptabulate
 
+The \prm {lpcode}, \prm {rpcode} and \prm {efcode} are comparable to the ones
+introduced by \PDFTEX, while \prm {cfcode} complements \prm {efcode}: a
+compression alongside expansion code per character. In \LUAMETATEX\ the expansion
+mechanism is also available in math.
+
 From \TEX\ we inherit the following tables. Ligatures are only used in so call
 base mode, when the engine does the font magic. Kerns are used in text and
 optionally in math: More details follow below.
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex b/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
index 83e19dcc9..daa4420c9 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
@@ -130,7 +130,7 @@ hyphenation happens is (normally) when the paragraph or a horizontal box is
 constructed. When \prm {savinghyphcodes} was zero when the language got
 initialized you start out with nothing, otherwise you already have a set.
 
-When a \prm {hjcode} is greater than 0 but less than 32 is indicates the
+When a \prm {hjcode} is greater than 0 but less than 32 the value indicates the
 to be used length. In the following example we map a character (\type {x}) onto
 another one in the patterns and tell the engine that \type {œ} counts as two
 characters. Because traditionally zero itself is reserved for inhibiting
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex b/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
index bccb899c0..df4da576e 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
@@ -24,21 +24,63 @@ such a user data object is represented as \syntax {<node>}.
 \blank
 
 You can ask for a list of fields with \type {node.fields} and for valid subtypes
-with \type {node.subtypes}. The \type {node.values} function reports some used
-values. Valid arguments are \type {glue}, \type {style} and \type {math}. Keep in
-mind that the setters normally expect a number, but this helper gives you a list
-of what numbers matter. For practical reason the \type {pagestate} values are
-also reported with this helper, but they are backend specific.
+with \type {node.subtypes}. There are plenty specific field values and you can
+some idea about them by calling \type {tex.get*values()} which returns a table if
+numbers (exclusive numbers or bits):
 
-\def\ShowValues#1{
+\starttexdefinition ShowTeXValues #1
     \blank
     \dontleavehmode
-    {\bf The return values of \type {node.values("#1")} are:}
+%     {\bf The return values of \type {tex.get#1values()} are:}\space
+    {\bf #1:}\space
     \showvalues{#1}
+%     \showhexvalues{#1}
     \blank
-}
-
-\ShowValues{glue} \ShowValues{style} \ShowValues{math} \ShowValues{pagestate}
+\stoptexdefinition
+
+\ShowTeXValues{fill}
+\ShowTeXValues{alignmentcontext}
+\ShowTeXValues{appendlinecontext}
+% \ShowTeXValues{automigration}
+\ShowTeXValues{charactertag}
+\ShowTeXValues{direction}
+\ShowTeXValues{discoption}
+\ShowTeXValues{discstate}
+% \ShowTeXValues{error}
+% \ShowTeXValues{flag}
+% \ShowTeXValues{frozenpar}
+\ShowTeXValues{glyphoption}
+\ShowTeXValues{group}
+\ShowTeXValues{hyphenation}
+\ShowTeXValues{io}
+\ShowTeXValues{kerneloption}
+\ShowTeXValues{listanchor}
+\ShowTeXValues{listgeometry}
+\ShowTeXValues{listsign}
+\ShowTeXValues{mathclassoption}
+\ShowTeXValues{mathcontrol}
+\ShowTeXValues{mathparameter}
+\ShowTeXValues{mathstylename}
+\ShowTeXValues{mathstyle}
+% \ShowTeXValues{mathvariant}
+% \ShowTeXValues{mode}
+\ShowTeXValues{noadoption}
+\ShowTeXValues{normalizeline}
+\ShowTeXValues{normalizepar}
+\ShowTeXValues{packtype}
+\ShowTeXValues{pagecontext}
+\ShowTeXValues{parbegin}
+\ShowTeXValues{parcontext}
+\ShowTeXValues{parmode}
+% \ShowTeXValues{runstate}
+% \ShowTeXValues{shapingpenalties}
+% \ShowTeXValues{specialmathclass}
+% \ShowTeXValues{textcontrol}
+
+There are a lot of helpers, especially for direct nodes. When possible they adapt
+to the kind of node they get passed. Often multiple values are returned which
+lessens the number of additional calls. It will take a while before all gets
+documented (which is no big deal as the main usage for them is in \CONTEXT).
 
 \stopsection
 
@@ -754,8 +796,6 @@ but don't rely on that for the future. Fields are: \showfields {style}.
 \LL
 \stoptabulate
 
-Valid styles are: \showvalues{style}.
-
 \stopsubsubsection
 
 \startsubsubsection[title={\nod {parameter} nodes}]
@@ -2398,9 +2438,9 @@ number, for a glue, kern, hlist, vlist, rule and math nodes as well as glyph
 nodes (although this last one is not used in native \SYNCTEX).
 
 \startfunctioncall
-node.setsynctexfields(<integer> f, <integer> l)
+node.direct.setsynctexfields(<integer> f, <integer> l)
 <integer> f, <integer> l =
-    node.getsynctexfields(<node> n)
+    node.direct.getsynctexfields(<node> n)
 \stopfunctioncall
 
 Of course you need to know what you're doing as no checking on sane values takes
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-style.tex b/doc/context/sources/general/manuals/luametatex/luametatex-style.tex
index 6586e6fc1..b3591344a 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-style.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-style.tex
@@ -84,13 +84,14 @@
         end
     end
 
-    function document.functions.showvalues(l)
+    function document.functions.showvalues(l,hex)
         if l then
             local n = table.count(l)
             local i = 0
+            local f = hex and "%s{\\tttf %s} (0x%02X)" or "%s{\\tttf %s} (%i)"
             for k, v in table.sortedhash(l) do
                 i = i + 1
-                context("%s{\\tttf %s} (%i)", (i == 1 and "") or (i == n and " and ") or ", ", v,k)
+                context(f, (i == 1 and "") or (i == n and " and ") or ", ", v,k)
             end
         end
     end
@@ -120,11 +121,11 @@
 \protected\def\showfields   #1{\ctxlua{document.functions.showfields("#1")}}
 \protected\def\showid       #1{\ctxlua{document.functions.showid("#1")}}
 \protected\def\showsubtypes #1{\ctxlua{document.functions.showsubtypes("#1")}}
-\protected\def\showvalues   #1{\ctxlua{document.functions.showvalues(node.values("#1"))}}
+\protected\def\showvalues   #1{\ctxlua{document.functions.showvalues(tex.get#1values())}}
+\protected\def\showhexvalues#1{\ctxlua{document.functions.showvalues(tex.get#1values(),true)}}
 \protected\def\showtypes      {\ctxlua{document.functions.showvalues(node.types())}}
 \protected\def\showvaluelist#1{\ctxlua{document.functions.showvalues(#1)}}
 
-
 \definecolor[blue]      [b=.5]
 \definecolor[red]       [r=.5]
 \definecolor[green]     [g=.5]
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex.tex b/doc/context/sources/general/manuals/luametatex/luametatex.tex
index f845fed17..e75053972 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex.tex
@@ -110,7 +110,7 @@
 % because the amount of output which is sensitive for refresh delays set. The new
 % target is nwo 50 pages per second for this manual but on this laptop that is
 % unlikely to happen any time soon. With tabulateusesize and tabulatesparseskips
-% experimends enabled we needed 8.1 second and 44.3 pps.
+% experiments enabled we needed 8.1 second and 44.3 pps.
 
 % \enableexperiments [tabulateusesize]
 % \enableexperiments [tabulatesparseskips]
diff --git a/doc/context/sources/general/manuals/luatex/luatex-languages.tex b/doc/context/sources/general/manuals/luatex/luatex-languages.tex
index cb9ca10f3..b3f96d247 100644
--- a/doc/context/sources/general/manuals/luatex/luatex-languages.tex
+++ b/doc/context/sources/general/manuals/luatex/luatex-languages.tex
@@ -140,7 +140,7 @@ hyphenation happens is (normally) when the paragraph or a horizontal box is
 constructed. When \prm {savinghyphcodes} was zero when the language got
 initialized you start out with nothing, otherwise you already have a set.
 
-When a \lpr {hjcode} is greater than 0 but less than 32 is indicates the
+When a \lpr {hjcode} is greater than 0 but less than 32 it indicates the
 to be used length. In the following example we map a character (\type {x}) onto
 another one in the patterns and tell the engine that \type {œ} counts as one
 character. Because traditionally zero itself is reserved for inhibiting
diff --git a/doc/context/sources/general/manuals/luatex/luatex-nodes.tex b/doc/context/sources/general/manuals/luatex/luatex-nodes.tex
index 6b9b0bd8c..2453edb2d 100644
--- a/doc/context/sources/general/manuals/luatex/luatex-nodes.tex
+++ b/doc/context/sources/general/manuals/luatex/luatex-nodes.tex
@@ -1321,8 +1321,8 @@ creates a node at the \TEX\ level.
 \libindex{flush_node}
 \libindex{flush_list}
 
-The next one the node \type {n} from \TEX's memory. Be careful: no checks are
-done on whether this node is still pointed to from a register or some \type
+The next one removes the node \type {n} from \TEX's memory. Be careful: no checks
+are done on whether this node is still pointed to from a register or some \type
 {next} field: it is up to you to make sure that the internal data structures
 remain correct.
 
@@ -2490,9 +2490,9 @@ number, for a glue, kern, hlist, vlist, rule and math nodes as well as glyph
 nodes (although this last one is not used in native \SYNCTEX).
 
 \startfunctioncall
-node.set_synctex_fields(<integer> f, <integer> l)
+node.direct.set_synctex_fields(<integer> f, <integer> l)
 <integer> f, <integer> l =
-    node.get_synctex_fields(<node> n)
+    node.direct.get_synctex_fields(<node> n)
 \stopfunctioncall
 
 Of course you need to know what you're doing as no checking on sane values takes
diff --git a/doc/context/sources/general/manuals/ontarget/ontarget-dk.tex b/doc/context/sources/general/manuals/ontarget/ontarget-dk.tex
index d1df3b8cc..bda07ecc9 100644
--- a/doc/context/sources/general/manuals/ontarget/ontarget-dk.tex
+++ b/doc/context/sources/general/manuals/ontarget/ontarget-dk.tex
@@ -9,8 +9,8 @@
 At the \CONTEXT\ 2021 meeting I mixed my \TEX\ talks with showing some of the
 (upcoming) \LUAMETATEX\ source code. One evening we had a extension party where a
 new unit was implemented, the \type {dk}. This event was triggered by a remark
-Hraban [Ramm] made on the participants list in advance of the meeting, where he pointed
-to a Wikipedia article from which we quote:
+Hraban [Ramm] made on the participants list in advance of the meeting, where he
+pointed to a Wikipedia article from which we quote:
 
 \startquotation
 In issue 33, Mad published a partial table of the \quotation {Potrzebie System of
diff --git a/doc/context/sources/general/manuals/ontarget/ontarget.tex b/doc/context/sources/general/manuals/ontarget/ontarget.tex
index 635ca5fb8..888a73f41 100644
--- a/doc/context/sources/general/manuals/ontarget/ontarget.tex
+++ b/doc/context/sources/general/manuals/ontarget/ontarget.tex
@@ -27,6 +27,7 @@
         \component ontarget-gettingridof
         \component ontarget-registers
         \component ontarget-fairphone
+      % \component ontarget-mathsizes
     \stopbodymatter
 
 \stopdocument