2020-08-14 18:27:00

13 files changed, 167 insertions, 143 deletions

diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex b/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex
index 36d4390aa..64ceaa353 100644
--- a/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex
+++ b/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex
@@ -62,26 +62,49 @@ The decision making is influenced by quite some factors, like:
 \stopitem
 \stopitemize
 
-In traditional \TEX\ it doesn't really matter how the resulting boxes look like,
-as long as the following steps can handle them, and those steps don't look into
-those boxes. In fact, unless you unpack a box, only the backend deals with the
-content. But in \LUATEX\ we have callbacks that hook into several stages and {\em
-can} look into the constructed boxes. In \LUATEX\ these boxes also have embedded
-directional information (needed by the backend) and (although that is seldom
-used) left or right boxed material, a features inherited from \ALEPH|/|\OMEGA.
-And when messing around with the content of boxes one has to know what can be
-seen there. In principle the code can be reorganized a it but adding additional
-functionality is not that trivial because we want to stay close to the original
-implementation, even if it has been messed up a bit by successive additions.
-Eventually I might give it a try to integrate all these features a bit better,
-but on the other hand: it works.
+% In traditional \TEX\ it doesn't really matter how the resulting boxes look like,
+% as long as the following steps can handle them, and those steps don't look into
+% those boxes. In fact, unless you unpack a box, only the backend deals with the
+% content. But in \LUATEX\ we have callbacks that hook into several stages and {\em
+% can} look into the constructed boxes. In \LUATEX\ these boxes also have embedded
+% directional information (needed by the backend) and (although that is seldom
+% used) left or right boxed material, a features inherited from \ALEPH|/|\OMEGA.
+% And when messing around with the content of boxes one has to know what can be
+% seen there. In principle the code can be reorganized a it but adding additional
+% functionality is not that trivial because we want to stay close to the original
+% implementation, even if it has been messed up a bit by successive additions.
+% Eventually I might give it a try to integrate all these features a bit better,
+% but on the other hand: it works.
+%
+% \starttexdefinition Sample #1#2
+%     \startluacode
+%         document.normalizestate = nodes.getnormalizeline()
+%         nodes.setnormalizeline(#1)
+%     \stopluacode
+%     \startsubsubject[title={normalization #1, #2}]
+%         \typebuffer[#2]
+%         \startlinecorrection
+%             \forgetall
+%             \start
+%                 \setupalign[verytolerant,stretch]
+%                 \showmakeup[line,hbox,vbox,glue]
+%                 \vbox{\getbuffer[#2]\samplefile{sapolsky}}
+%             \stop
+%             \par
+%         \stoplinecorrection
+%     \stopsubject
+%     \startluacode
+%         nodes.setnormalizeline(document.normalizestate)
+%     \stopluacode
+% \stoptexdefinition
+
+\newcount\OldNormalizeLineMode
 
 \starttexdefinition Sample #1#2
-    \startluacode
-        document.normalizestate = nodes.getnormalizeline()
-        nodes.setnormalizeline(#1)
-    \stopluacode
-    \startsubsubject[title={normalization #1, #2}]
+    \OldNormalizeLineMode\normalizelinemode
+    \bitwiseflip \normalizelinemode \normalizelinemodecode
+    \bitwiseflip \normalizelinemode \indentskipmodecode
+    \startsubsubject[title={#1}]
         \typebuffer[#2]
         \startlinecorrection
             \forgetall
@@ -93,9 +116,7 @@ but on the other hand: it works.
             \par
         \stoplinecorrection
     \stopsubject
-    \startluacode
-        nodes.setnormalizeline(document.normalizestate)
-    \stopluacode
+    \normalizelinemode\OldNormalizeLineMode
 \stoptexdefinition
 
 \startbuffer[sample-1]
@@ -130,108 +151,113 @@ but on the other hand: it works.
     \hangafter  =  3
 \stopbuffer
 
-In the next examples we show how the result of typesetting a paragraph looks
-like. We use the Sapolsky quote from the distribution. The cyan glue nodes are
-the left and right skip nodes, and the gray one at the end of the last line
-represents the parfill skip. The magenta ones at the edge are baseline skips. An
-indentation is shown in gray too. As experiment we have four normalization levels
-but in the end only the highest level makes sense, simply because normalization
-makes no sense unless one consistently normalizes all. We just keep the
-granularity because it makes it possible to explain what gets done.
-
-\texdefinition{Sample}{0}{sample-1}
-\texdefinition{Sample}{0}{sample-2}
-\texdefinition{Sample}{0}{sample-3}
-\texdefinition{Sample}{0}{sample-4}
-
-You might have noticed that the right skip is always there but the left skip is
-absent when it is zero. As said, as long as the result is okay, it does not
-really matter. But \unknown\ in \LUATEX\ (and therefore \CONTEXT) it can have
-consequences because there we can kick in a callback that does something with
-lines. Such a callback often has to deal with these specific glues and them being
-optional makes for more testing. The more predictable the order is, the better.
-Although we can easily normalize lines (in a callback) to always have a left skip
-too it is also an option in the engine.
-
-\texdefinition{Sample}{1}{sample-1}
-\texdefinition{Sample}{1}{sample-2}
-\texdefinition{Sample}{1}{sample-3}
-\texdefinition{Sample}{1}{sample-4}
-
-In the previous examples there are always left skips as well as right skips. It
-makes no sense to have an option to omit both zero left and right skips, because
-that again is unpredictable. But we can go further.
-
-\texdefinition{Sample}{2}{sample-1}
-\texdefinition{Sample}{2}{sample-2}
-\texdefinition{Sample}{2}{sample-3}
-\texdefinition{Sample}{2}{sample-4}
-
-In these examples the indentation has been turned into a glue as well (actually
-it is more a kern but using a glue makes more sense). The hanging indentation
-however is not seen here: it is not represented by glue but instead sort of
-hidden in the width of the box and a shift of its content.
-
-\texdefinition{Sample}{3}{sample-1}
-\texdefinition{Sample}{3}{sample-2}
-\texdefinition{Sample}{3}{sample-3}
-\texdefinition{Sample}{3}{sample-4}
-
-In the previous examples the hanging indentation is turned into left and right
-hang skips. These cannot be set at the \TEX\ end, but are injected when we
-instruct the normalizer to do so.
-
-\texdefinition{Sample}{4}{sample-1}
-\texdefinition{Sample}{4}{sample-2}
-\texdefinition{Sample}{4}{sample-3}
-\texdefinition{Sample}{4}{sample-4}
-
-The previous examples differ from the previous set in that they push these hang
-related glue nodes before the left and after the right skip. As I couldn't make
-up my mind yet, I let \LUAMETATEX\ just provide both variants.
-
-The option to keep hang related information explicitly in the line has some
-consequences. First of all, we now have glue and not some shift|/|width
-combination. Second, we have introduced an incompatibility: the lines now always
-have the proper width. You might have noticed that but we can show it more
-explicitly. We use two parameter sets
-
-\startbuffer[sample-5]
-    \hangindent = 20pt
-    \hangafter  =  0
-\stopbuffer
-
-\startbuffer[sample-6]
-    \hangindent =-20pt
-    \hangafter  =  0
-\stopbuffer
-
-\Sample{0}{sample-5}
-\Sample{4}{sample-5}
-
-\Sample{0}{sample-6}
-\Sample{4}{sample-6}
-
-A not yet mention part of the normalization is that, because they are no longer
-of relevance, the special local par nodes have been removed. The one that starts
-a paragraph is turned into a normal directional node if needed, so that we get
-properly balanced pairs of directional nodes. It must been said that the code
-that does all this is a bit of a mess. We want to stay close to the original
-code, but we also need to deal with all these extensions, like directions,
-protrusion, extra boxes, etc.
-
-Not shown here is that there is a fifth mode of operation. When we enable that
-level an overfull box will get a correction skip so that the right skip etc are
-properly aligned. How useful this is: we'll see.
-
-Now, when I decide to keep this feature, which can be set at the \LUA\ end to do
-the previously mentioned tasks, depending on a feature level ranging from zero to
-four, I also need to check the impact on existing \CONTEXT\ code, which
-(currently) is complicated by the fact that most is shared between \MKIV\ and
-\LMTX, and only \LUAMETATEX\ has this normalization feature. I will probably
-enable it for a while locally in order to see if there are side effects. Then,
-when the code base gets adapted, we have to assume that normalization happens, so
-there is no way back.
+% In the next examples we show how the result of typesetting a paragraph looks
+% like. We use the Sapolsky quote from the distribution. The cyan glue nodes are
+% the left and right skip nodes, and the gray one at the end of the last line
+% represents the parfill skip. The magenta ones at the edge are baseline skips. An
+% indentation is shown in gray too. As experiment we have four normalization levels
+% but in the end only the highest level makes sense, simply because normalization
+% makes no sense unless one consistently normalizes all. We just keep the
+% granularity because it makes it possible to explain what gets done.
+%
+% \texdefinition{Sample}{0}{sample-1}
+% \texdefinition{Sample}{0}{sample-2}
+% \texdefinition{Sample}{0}{sample-3}
+% \texdefinition{Sample}{0}{sample-4}
+%
+% You might have noticed that the right skip is always there but the left skip is
+% absent when it is zero. As said, as long as the result is okay, it does not
+% really matter. But \unknown\ in \LUATEX\ (and therefore \CONTEXT) it can have
+% consequences because there we can kick in a callback that does something with
+% lines. Such a callback often has to deal with these specific glues and them being
+% optional makes for more testing. The more predictable the order is, the better.
+% Although we can easily normalize lines (in a callback) to always have a left skip
+% too it is also an option in the engine.
+%
+% \texdefinition{Sample}{1}{sample-1}
+% \texdefinition{Sample}{1}{sample-2}
+% \texdefinition{Sample}{1}{sample-3}
+% \texdefinition{Sample}{1}{sample-4}
+%
+% In the previous examples there are always left skips as well as right skips. It
+% makes no sense to have an option to omit both zero left and right skips, because
+% that again is unpredictable. But we can go further.
+%
+% \texdefinition{Sample}{2}{sample-1}
+% \texdefinition{Sample}{2}{sample-2}
+% \texdefinition{Sample}{2}{sample-3}
+% \texdefinition{Sample}{2}{sample-4}
+%
+% In these examples the indentation has been turned into a glue as well (actually
+% it is more a kern but using a glue makes more sense). The hanging indentation
+% however is not seen here: it is not represented by glue but instead sort of
+% hidden in the width of the box and a shift of its content.
+%
+% \texdefinition{Sample}{3}{sample-1}
+% \texdefinition{Sample}{3}{sample-2}
+% \texdefinition{Sample}{3}{sample-3}
+% \texdefinition{Sample}{3}{sample-4}
+%
+% In the previous examples the hanging indentation is turned into left and right
+% hang skips. These cannot be set at the \TEX\ end, but are injected when we
+% instruct the normalizer to do so.
+%
+% \texdefinition{Sample}{4}{sample-1}
+% \texdefinition{Sample}{4}{sample-2}
+% \texdefinition{Sample}{4}{sample-3}
+% \texdefinition{Sample}{4}{sample-4}
+%
+% The previous examples differ from the previous set in that they push these hang
+% related glue nodes before the left and after the right skip. As I couldn't make
+% up my mind yet, I let \LUAMETATEX\ just provide both variants.
+%
+% The option to keep hang related information explicitly in the line has some
+% consequences. First of all, we now have glue and not some shift|/|width
+% combination. Second, we have introduced an incompatibility: the lines now always
+% have the proper width. You might have noticed that but we can show it more
+% explicitly. We use two parameter sets
+%
+% \startbuffer[sample-5]
+%     \hangindent = 20pt
+%     \hangafter  =  0
+% \stopbuffer
+%
+% \startbuffer[sample-6]
+%     \hangindent =-20pt
+%     \hangafter  =  0
+% \stopbuffer
+%
+% \Sample{0}{sample-5}
+% \Sample{4}{sample-5}
+%
+% \Sample{0}{sample-6}
+% \Sample{4}{sample-6}
+
+\texdefinition{Sample}{Sample 1}{sample-1}
+\texdefinition{Sample}{Sample 2}{sample-2}
+\texdefinition{Sample}{Sample 3}{sample-3}
+\texdefinition{Sample}{Sample 4}{sample-4}
+
+% A not yet mention part of the normalization is that, because they are no longer
+% of relevance, the special local par nodes have been removed. The one that starts
+% a paragraph is turned into a normal directional node if needed, so that we get
+% properly balanced pairs of directional nodes. It must been said that the code
+% that does all this is a bit of a mess. We want to stay close to the original
+% code, but we also need to deal with all these extensions, like directions,
+% protrusion, extra boxes, etc.
+%
+% Not shown here is that there is a fifth mode of operation. When we enable that
+% level an overfull box will get a correction skip so that the right skip etc are
+% properly aligned. How useful this is: we'll see.
+%
+% Now, when I decide to keep this feature, which can be set at the \LUA\ end to do
+% the previously mentioned tasks, depending on a feature level ranging from zero to
+% four, I also need to check the impact on existing \CONTEXT\ code, which
+% (currently) is complicated by the fact that most is shared between \MKIV\ and
+% \LMTX, and only \LUAMETATEX\ has this normalization feature. I will probably
+% enable it for a while locally in order to see if there are side effects. Then,
+% when the code base gets adapted, we have to assume that normalization happens, so
+% there is no way back.
 
 \stopchapter
 
diff --git a/doc/context/sources/general/manuals/evenmore/evenmore.tex b/doc/context/sources/general/manuals/evenmore/evenmore.tex
index 9f67f13f0..87044b34d 100644
--- a/doc/context/sources/general/manuals/evenmore/evenmore.tex
+++ b/doc/context/sources/general/manuals/evenmore/evenmore.tex
@@ -28,6 +28,7 @@
       % \component evenmore-paragraphs
       % \component evenmore-hyphenation
       % \component evenmore-bitwise
+      % \component evenmore-offloading
     \stopbodymatter
 
 \stopdocument
diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii
index 939c9d616..1c75e5664 100644
--- a/tex/context/base/mkii/cont-new.mkii
+++ b/tex/context/base/mkii/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2020.08.13 19:42}
+\newcontextversion{2020.08.14 18:24}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii
index fede76b63..5c11d41d3 100644
--- a/tex/context/base/mkii/context.mkii
+++ b/tex/context/base/mkii/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.08.13 19:42}
+\edef\contextversion{2020.08.14 18:24}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index 0f6ad84f5..20b7f5554 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2020.08.13 19:42}
+\newcontextversion{2020.08.14 18:24}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index cee7d0ebc..29513dad7 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -45,7 +45,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.08.13 19:42}
+\edef\contextversion{2020.08.14 18:24}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/context.mkxl b/tex/context/base/mkiv/context.mkxl
index c344a48d1..b54d1c46e 100644
--- a/tex/context/base/mkiv/context.mkxl
+++ b/tex/context/base/mkiv/context.mkxl
@@ -29,7 +29,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.08.13 19:42}
+\edef\contextversion{2020.08.14 18:24}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/spac-chr.lua b/tex/context/base/mkiv/spac-chr.lua
index 087bff89a..ac04a4c8e 100644
--- a/tex/context/base/mkiv/spac-chr.lua
+++ b/tex/context/base/mkiv/spac-chr.lua
@@ -38,7 +38,6 @@ local setchar            = nuts.setchar
 local setattrlist        = nuts.setattrlist
 local getfont            = nuts.getfont
 local setsubtype         = nuts.setsubtype
-local setdisc            = nuts.setdisc
 local isglyph            = nuts.isglyph
 
 local setcolor           = nodes.tracers.colors.set
@@ -58,7 +57,6 @@ local new_penalty        = nodepool.penalty
 local new_glue           = nodepool.glue
 local new_kern           = nodepool.kern
 local new_rule           = nodepool.rule
-local new_disc           = nodepool.disc
 
 local nodecodes          = nodes.nodecodes
 local gluecodes          = nodes.gluecodes
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index d652c1143..0b5a3fd8c 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index b1af2f3e8..53da6b027 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
diff --git a/tex/context/base/mkiv/strc-mat.mkiv b/tex/context/base/mkiv/strc-mat.mkiv
index 2c8ed8ddf..fe25e236e 100644
--- a/tex/context/base/mkiv/strc-mat.mkiv
+++ b/tex/context/base/mkiv/strc-mat.mkiv
@@ -1,5 +1,4 @@
 %D \module
-%D \module
 %D   [       file=strc-mat,
 %D        version=2008.10.20,
 %D          title=\CONTEXT\ Structure Macros,
diff --git a/tex/context/base/mkiv/supp-box.lmt b/tex/context/base/mkiv/supp-box.lmt
index 98bf828b5..61ec9b328 100644
--- a/tex/context/base/mkiv/supp-box.lmt
+++ b/tex/context/base/mkiv/supp-box.lmt
@@ -967,13 +967,15 @@ do
     local setprop            = nuts.setprop
     local getprop            = nuts.getprop
 
-    local line_code          = nodes.listcodes.line
-    local unknown_code       = nodes.listcodes.unknown
+    local listcodes          = nodes.listcodes
+    local line_code          = listcodes.line
+    local equation_code      = listcodes.equation
+    local unknown_code       = listcodes.unknown
 
     local values             = tokens.values
     local dimension_value    = values.dimension
     local cardinal_value     = values.cardinal
-    local list_value         = values.list
+    local direct_value       = values.direct
 
     -- todo: make helper that formats
 
@@ -992,7 +994,7 @@ do
             prop = { }
             if list then
                 for n, subtype in nexthlist, list do
-                    if subtype == line_code then
+                    if subtype == line_code then -- or subtype == equation_code then
                         line = line + 1
                         prop[line] = n
                     end
@@ -1063,7 +1065,7 @@ do
     local function getline(props,line,found,box,value)
         local p, n = getboth(found)
         local temp = new_hlist()
-        setsubtype(temp,line_code)
+        setsubtype(temp,getsubtype(found))
         setwhd(temp,getwhd(found))
         if found == getlist(box) then
             setlink(temp,n)
@@ -1074,22 +1076,20 @@ do
         getprop(box,"boxlines")[line] = temp
         setboth(found)
         setsubtype(found, unknown_code)
-        found = tonode(found)
         if value then
-            return list_value, found
+            return direct_value, found
         else
-            context(found)
+            context(tonode(found))
         end
     end
 
     local function copyline(props,line,found,box,value)
         found = copy_node(found)
         setsubtype(found, unknown_code)
-        found = tonode(found)
         if value then
-            return list_value, found
+            return direct_value, found
         else
-            context(found)
+            context(tonode(found))
         end
     end
 
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index b08cccbf6..21a775128 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date  : 2020-08-13 19:42
+-- merge date  : 2020-08-14 18:24
 
 do -- begin closure to overcome local limits and interference