summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHans Hagen <pragma@wxs.nl>2020-01-26 19:35:43 +0100
committerContext Git Mirror Bot <phg@phi-gamma.net>2020-01-26 19:35:43 +0100
commit43fc66771a0c9d27cc0b7fe7a69392ea313bd0ca (patch)
tree9b339c63cd28528e5062fe980e964808df619374
parent5189b2143a30a39cd3533569cbef3f06422cc1d9 (diff)
downloadcontext-43fc66771a0c9d27cc0b7fe7a69392ea313bd0ca.tar.gz
2020-01-26 18:37:00
-rw-r--r--doc/context/documents/general/manuals/luametatex.pdfbin1095262 -> 1186001 bytes
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-differences.tex6
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex2
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-languages.tex1113
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-lua.tex6
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex16
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex2
-rw-r--r--doc/context/sources/general/manuals/luametatex/luametatex-tex.tex15
-rw-r--r--scripts/context/lua/mtx-grep.lua20
-rw-r--r--scripts/context/lua/mtxrun.lua8
-rw-r--r--scripts/context/stubs/mswin/mtxrun.lua8
-rw-r--r--scripts/context/stubs/unix/mtxrun8
-rw-r--r--scripts/context/stubs/win64/mtxrun.lua8
-rw-r--r--tex/context/base/mkii/cont-new.mkii2
-rw-r--r--tex/context/base/mkii/context.mkii2
-rw-r--r--tex/context/base/mkiv/attr-ini.mkiv6
-rw-r--r--tex/context/base/mkiv/cont-new.mkiv2
-rw-r--r--tex/context/base/mkiv/context.mkiv2
-rw-r--r--tex/context/base/mkiv/context.mkxl2
-rw-r--r--tex/context/base/mkiv/data-exp.lua2
-rw-r--r--tex/context/base/mkiv/data-res.lua2
-rw-r--r--tex/context/base/mkiv/lpdf-emb.lua2
-rw-r--r--tex/context/base/mkiv/luat-cnf.lua6
-rw-r--r--tex/context/base/mkiv/luat-cod.lua3
-rw-r--r--tex/context/base/mkiv/luat-sto.lua2
-rw-r--r--tex/context/base/mkiv/mlib-mat.lua4
-rw-r--r--tex/context/base/mkiv/mult-sys.mkiv4
-rw-r--r--tex/context/base/mkiv/node-fin.lua110
-rw-r--r--tex/context/base/mkiv/node-res.lua13
-rw-r--r--tex/context/base/mkiv/page-ffl.mkiv58
-rw-r--r--tex/context/base/mkiv/page-flt.mkiv4
-rw-r--r--tex/context/base/mkiv/page-ini.lua32
-rw-r--r--tex/context/base/mkiv/page-ini.mkiv5
-rw-r--r--tex/context/base/mkiv/page-one.mkiv8
-rw-r--r--tex/context/base/mkiv/page-otr.mkvi4
-rw-r--r--tex/context/base/mkiv/page-run.lua4
-rw-r--r--tex/context/base/mkiv/status-files.pdfbin26724 -> 26743 bytes
-rw-r--r--tex/context/base/mkiv/status-lua.pdfbin269636 -> 269677 bytes
-rw-r--r--tex/context/base/mkiv/strc-itm.lua4
-rw-r--r--tex/context/base/mkiv/syst-lua.mkiv2
-rw-r--r--tex/context/base/mkiv/syst-lua.mkxl2
-rw-r--r--tex/context/base/mkiv/typo-chr.lua14
-rw-r--r--tex/context/base/mkiv/typo-chr.mkiv3
-rw-r--r--tex/context/modules/common/s-abbreviations-logos.tex3
-rw-r--r--tex/generic/context/luatex/luatex-fonts-merged.lua2
45 files changed, 1404 insertions, 117 deletions
diff --git a/doc/context/documents/general/manuals/luametatex.pdf b/doc/context/documents/general/manuals/luametatex.pdf
index ec46a3723..b128ed767 100644
--- a/doc/context/documents/general/manuals/luametatex.pdf
+++ b/doc/context/documents/general/manuals/luametatex.pdf
Binary files differ
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex b/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex
index 3e164c711..19a4078c1 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-differences.tex
@@ -170,7 +170,7 @@ really needs to be set because there is no built|-|in font loader.
define_font
\stoptyping
-There are all kind of subtle differences in the implementation, for instance we
+There are all kinds of subtle differences in the implementation, for instance we
no longer intercept \type {*} and \type {&} as these were already replaced long
ago in \TEX\ engines by command line options. Talking of options, only a few are
left.
@@ -186,11 +186,11 @@ natural with everything new and beta, but we have a fast update cycle.
The same is true for \LUAMETATEX\ and \CONTEXT\ \LMTX: it can be used for
production as usual and in practice \CONTEXT\ users tend to use the beta
-releases, which proves this. Of course, of you use low level features that are
+releases, which proves this. Of course, if you use low level features that are
experimental you're on your own. Also, as with \LUATEX\ it might take many years
before a long term stable is defined. The good news is that, the source code
being part of the \CONTEXT\ distribution, there is always a properly working,
-more of less long term stable, snapshot.
+more or less long term stable, snapshot.
The error reporting subsystem has been redone a little but is still fundamentally
the same. We don't really assume interactive usage but if someone uses it, it
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex b/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
index 34e717a72..06e0ce327 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex
@@ -59,7 +59,7 @@ to stay, their exact behaviour might evolve. Because \LUAMETATEX\ is also used
for experiments, this is not a problem. We can always decide to also add some of
what is discussed here to \LUATEX, but it will happen with a delay.
-There are all kind of small improvements that might find their way into stock
+There are all kinds of small improvements that might find their way into stock
\LUATEX: a few more helpers, some cleanup of code, etc. We'll see. In any case,
if you play with these before they are declared stable, unexpected side effects
are what you have to accept.
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex b/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
new file mode 100644
index 000000000..19112a7f1
--- /dev/null
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-languages.tex
@@ -0,0 +1,1113 @@
+% language=uk
+
+\environment luametatex-style
+
+\startcomponent luametatex-languages
+
+\startchapter[reference=languages,title={Languages, characters, fonts and glyphs}]
+
+\startsection[title={Introduction}]
+
+\topicindex {languages}
+
+\LUATEX's internal handling of the characters and glyphs that eventually become
+typeset is quite different from the way \TEX82 handles those same objects. The
+easiest way to explain the difference is to focus on unrestricted horizontal mode
+(i.e.\ paragraphs) and hyphenation first. Later on, it will be easy to deal
+with the differences that occur in horizontal and math modes.
+
+In \TEX82, the characters you type are converted into \type {char} node records
+when they are encountered by the main control loop. \TEX\ attaches and processes
+the font information while creating those records, so that the resulting \quote
+{horizontal list} contains the final forms of ligatures and implicit kerning.
+This packaging is needed because we may want to get the effective width of for
+instance a horizontal box.
+
+When it becomes necessary to hyphenate words in a paragraph, \TEX\ converts (one
+word at time) the \type {char} node records into a string by replacing ligatures
+with their components and ignoring the kerning. Then it runs the hyphenation
+algorithm on this string, and converts the hyphenated result back into a \quote
+{horizontal list} that is consecutively spliced back into the paragraph stream.
+Keep in mind that the paragraph may contain unboxed horizontal material, which
+then already contains ligatures and kerns and the words therein are part of the
+hyphenation process.
+
+Those \type {char} node records are somewhat misnamed, as they are glyph
+positions in specific fonts, and therefore not really \quote {characters} in the
+linguistic sense. There is no language information inside the \type {char} node
+records at all. Instead, language information is passed along using \type
+{language whatsit} nodes inside the horizontal list.
+
+In \LUATEX, the situation is quite different. The characters you type are always
+converted into \nod {glyph} node records with a special subtype to identify them
+as being intended as linguistic characters. \LUATEX\ stores the needed language
+information in those records, but does not do any font|-|related processing at
+the time of node creation. It only stores the index of the current font and a
+reference to a character in that font.
+
+When it becomes necessary to typeset a paragraph, \LUATEX\ first inserts all
+hyphenation points right into the whole node list. Next, it processes all the
+font information in the whole list (creating ligatures and adjusting kerning),
+and finally it adjusts all the subtype identifiers so that the records are \quote
+{glyph nodes} from now on.
+
+\stopsection
+
+\startsection[title={Characters, glyphs and discretionaries},reference=charsandglyphs]
+
+\topicindex {characters}
+\topicindex {glyphs}
+\topicindex {hyphenation}
+
+\TEX82 (including \PDFTEX) differentiates between \type {char} nodes and \type
+{lig} nodes. The former are simple items that contained nothing but a \quote
+{character} and a \quote {font} field, and they lived in the same memory as
+tokens did. The latter also contained a list of components, and a subtype
+indicating whether this ligature was the result of a word boundary, and it was
+stored in the same place as other nodes like boxes and kerns and glues.
+
+In \LUATEX, these two types are merged into one, somewhat larger structure called
+a \nod {glyph} node. Besides having the old character, font, and component
+fields there are a few more, like \quote {attr} that we will see in \in {section}
+[glyphnodes], these nodes also contain a subtype, that codes four main types and
+two additional ghost types. For ligatures, multiple bits can be set at the same
+time (in case of a single|-|glyph word).
+
+\startitemize
+ \startitem
+ \type {character}, for characters to be hyphenated: the lowest bit
+ (bit 0) is set to 1.
+ \stopitem
+ \startitem
+ \nod {glyph}, for specific font glyphs: the lowest bit (bit 0) is
+ not set.
+ \stopitem
+ \startitem
+ \type {ligature}, for constructed ligatures bit 1 is set.
+ \stopitem
+ \startitem
+ \type {ghost}, for so called \quote {ghost objects} bit 2 is set.
+ \stopitem
+ \startitem
+ \type {left}, for ligatures created from a left word boundary and for
+ ghosts created from \lpr {leftghost} bit 3 gets set.
+ \stopitem
+ \startitem
+ \type {right}, for ligatures created from a right word boundary and
+ for ghosts created from \lpr {rightghost} bit 4 is set.
+ \stopitem
+\stopitemize
+
+The \nod {glyph} nodes also contain language data, split into four items that
+were current when the node was created: the \prm {setlanguage} (15~bits), \prm
+{lefthyphenmin} (8~bits), \prm {righthyphenmin} (8~bits), and \prm {uchyph}
+(1~bit).
+
+Incidentally, \LUATEX\ allows 16383 separate languages, and words can be 256
+characters long. The language is stored with each character. You can set
+\prm {firstvalidlanguage} to for instance~1 and make thereby language~0
+an ignored hyphenation language.
+
+The new primitive \lpr {hyphenationmin} can be used to signal the minimal length
+of a word. This value is stored with the (current) language.
+
+Because the \prm {uchyph} value is saved in the actual nodes, its handling is
+subtly different from \TEX82: changes to \prm {uchyph} become effective
+immediately, not at the end of the current partial paragraph.
+
+Typeset boxes now always have their language information embedded in the nodes
+themselves, so there is no longer a possible dependency on the surrounding
+language settings. In \TEX82, a mid|-|paragraph statement like \type {\unhbox0}
+would process the box using the current paragraph language unless there was a
+\prm {setlanguage} issued inside the box. In \LUATEX, all language variables
+are already frozen.
+
+In traditional \TEX\ the process of hyphenation is driven by \type {lccode}s. In
+\LUATEX\ we made this dependency less strong. There are several strategies
+possible. When you do nothing, the currently used \type {lccode}s are used, when
+loading patterns, setting exceptions or hyphenating a list.
+
+When you set \prm {savinghyphcodes} to a value greater than zero the current set
+of \type {lccode}s will be saved with the language. In that case changing a \type
+{lccode} afterwards has no effect. However, you can adapt the set with:
+
+\starttyping
+\hjcode`a=`a
+\stoptyping
+
+This change is global which makes sense if you keep in mind that the moment that
+hyphenation happens is (normally) when the paragraph or a horizontal box is
+constructed. When \prm {savinghyphcodes} was zero when the language got
+initialized you start out with nothing, otherwise you already have a set.
+
+When a \lpr {hjcode} is greater than 0 but less than 32 is indicates the
+to be used length. In the following example we map a character (\type {x}) onto
+another one in the patterns and tell the engine that \type {œ} counts as one
+character. Because traditionally zero itself is reserved for inhibiting
+hyphenation, a value of 32 counts as zero.
+
+Here are some examples (we assume that French patterns are used):
+
+\starttabulate[||||]
+\NC \NC \type{foobar} \NC \type{foo-bar} \NC \NR
+\NC \type{\hjcode`x=`o} \NC \type{fxxbar} \NC \type{fxx-bar} \NC \NR
+\NC \type{\lefthyphenmin3} \NC \type{œdipus} \NC \type{œdi-pus} \NC \NR
+\NC \type{\lefthyphenmin4} \NC \type{œdipus} \NC \type{œdipus} \NC \NR
+\NC \type{\hjcode`œ=2} \NC \type{œdipus} \NC \type{œdi-pus} \NC \NR
+\NC \type{\hjcode`i=32 \hjcode`d=32} \NC \type{œdipus} \NC \type{œdipus} \NC \NR
+\NC
+\stoptabulate
+
+Carrying all this information with each glyph would give too much overhead and
+also make the process of setting up these codes more complex. A solution with
+\type {hjcode} sets was considered but rejected because in practice the current
+approach is sufficient and it would not be compatible anyway.
+
+Beware: the values are always saved in the format, independent of the setting
+of \prm {savinghyphcodes} at the moment the format is dumped.
+
+A boundary node normally would mark the end of a word which interferes with for
+instance discretionary injection. For this you can use the \prm {wordboundary}
+as a trigger. Here are a few examples of usage:
+
+\startbuffer
+ discrete---discrete
+\stopbuffer
+\typebuffer \startnarrower \dontcomplain \hsize 1pt \getbuffer \par \stopnarrower
+\startbuffer
+ discrete\discretionary{}{}{---}discrete
+\stopbuffer
+\typebuffer \startnarrower \dontcomplain \hsize 1pt \getbuffer \par \stopnarrower
+\startbuffer
+ discrete\wordboundary\discretionary{}{}{---}discrete
+\stopbuffer
+\typebuffer \startnarrower \dontcomplain \hsize 1pt \getbuffer \par \stopnarrower
+\startbuffer
+ discrete\wordboundary\discretionary{}{}{---}\wordboundary discrete
+\stopbuffer
+\typebuffer \startnarrower \dontcomplain \hsize 1pt \getbuffer \par \stopnarrower
+\startbuffer
+ discrete\wordboundary\discretionary{---}{}{}\wordboundary discrete
+\stopbuffer
+\typebuffer \startnarrower \dontcomplain \hsize 1pt \getbuffer \par \stopnarrower
+
+We only accept an explicit hyphen when there is a preceding glyph and we skip a
+sequence of explicit hyphens since that normally indicates a \type {--} or \type
+{---} ligature in which case we can in a worse case usage get bad node lists
+later on due to messed up ligature building as these dashes are ligatures in base
+fonts. This is a side effect of separating the hyphenation, ligaturing and
+kerning steps.
+
+The start and end of a sequence of characters is signalled by a \nod {glue}, \nod
+{penalty}, \nod {kern} or \nod {boundary} node. But by default also a \nod
+{hlist}, \nod {vlist}, \nod {rule}, \nod {dir}, \nod {whatsit}, \nod {ins}, and
+\nod {adjust} node indicate a start or end. You can omit the last set from the
+test by setting \lpr {hyphenationbounds} to a non|-|zero value:
+
+\starttabulate[|c|l|]
+\DB value \BC behaviour \NC \NR
+\TB
+\NC \type{0} \NC not strict \NC \NR
+\NC \type{1} \NC strict start \NC \NR
+\NC \type{2} \NC strict end \NC \NR
+\NC \type{3} \NC strict start and strict end \NC \NR
+\LL
+\stoptabulate
+
+The word start is determined as follows:
+
+\starttabulate[|l|l|]
+\DB node \BC behaviour \NC \NR
+\TB
+\BC boundary \NC yes when wordboundary \NC \NR
+\BC hlist \NC when hyphenationbounds 1 or 3 \NC \NR
+\BC vlist \NC when hyphenationbounds 1 or 3 \NC \NR
+\BC rule \NC when hyphenationbounds 1 or 3 \NC \NR
+\BC dir \NC when hyphenationbounds 1 or 3 \NC \NR
+\BC whatsit \NC when hyphenationbounds 1 or 3 \NC \NR
+\BC glue \NC yes \NC \NR
+\BC math \NC skipped \NC \NR
+\BC glyph \NC exhyphenchar (one only) : yes (so no -- ---) \NC \NR
+\BC otherwise \NC yes \NC \NR
+\LL
+\stoptabulate
+
+The word end is determined as follows:
+
+\starttabulate[|l|l|]
+\DB node \BC behaviour \NC \NR
+\TB
+\BC boundary \NC yes \NC \NR
+\BC glyph \NC yes when different language \NC \NR
+\BC glue \NC yes \NC \NR
+\BC penalty \NC yes \NC \NR
+\BC kern \NC yes when not italic (for some historic reason) \NC \NR
+\BC hlist \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC vlist \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC rule \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC dir \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC whatsit \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC ins \NC when hyphenationbounds 2 or 3 \NC \NR
+\BC adjust \NC when hyphenationbounds 2 or 3 \NC \NR
+\LL
+\stoptabulate
+
+\in {Figures} [hb:1] upto \in [hb:5] show some examples. In all cases we set the
+min values to 1 and make sure that the words hyphenate at each character.
+
+\hyphenation{o-n-e t-w-o}
+
+\def\SomeTest#1#2%
+ {\lefthyphenmin \plusone
+ \righthyphenmin \plusone
+ \parindent \zeropoint
+ \everypar \emptytoks
+ \dontcomplain
+ \hbox to 2cm {%
+ \vtop {%
+ \hsize 1pt
+ \hyphenationbounds#1
+ #2
+ \par}}}
+
+\startplacefigure[reference=hb:1,title={\type{one}}]
+ \startcombination[4*1]
+ {\SomeTest{0}{one}} {\type{0}}
+ {\SomeTest{1}{one}} {\type{1}}
+ {\SomeTest{2}{one}} {\type{2}}
+ {\SomeTest{3}{one}} {\type{3}}
+ \stopcombination
+\stopplacefigure
+
+\startplacefigure[reference=hb:2,title={\type{one\null two}}]
+ \startcombination[4*1]
+ {\SomeTest{0}{one\null two}} {\type{0}}
+ {\SomeTest{1}{one\null two}} {\type{1}}
+ {\SomeTest{2}{one\null two}} {\type{2}}
+ {\SomeTest{3}{one\null two}} {\type{3}}
+ \stopcombination
+\stopplacefigure
+
+\startplacefigure[reference=hb:3,title={\type{\null one\null two}}]
+ \startcombination[4*1]
+ {\SomeTest{0}{\null one\null two}} {\type{0}}
+ {\SomeTest{1}{\null one\null two}} {\type{1}}
+ {\SomeTest{2}{\null one\null two}} {\type{2}}
+ {\SomeTest{3}{\null one\null two}} {\type{3}}
+ \stopcombination
+\stopplacefigure
+
+\startplacefigure[reference=hb:4,title={\type{one\null two\null}}]
+ \startcombination[4*1]
+ {\SomeTest{0}{one\null two\null}} {\type{0}}
+ {\SomeTest{1}{one\null two\null}} {\type{1}}
+ {\SomeTest{2}{one\null two\null}} {\type{2}}
+ {\SomeTest{3}{one\null two\null}} {\type{3}}
+ \stopcombination
+\stopplacefigure
+
+\startplacefigure[reference=hb:5,title={\type{\null one\null two\null}}]
+ \startcombination[4*1]
+ {\SomeTest{0}{\null one\null two\null}} {\type{0}}
+ {\SomeTest{1}{\null one\null two\null}} {\type{1}}
+ {\SomeTest{2}{\null one\null two\null}} {\type{2}}
+ {\SomeTest{3}{\null one\null two\null}} {\type{3}}
+ \stopcombination
+\stopplacefigure
+
+% (Future versions of \LUATEX\ might provide more granularity.)
+
+In traditional \TEX\ ligature building and hyphenation are interwoven with the
+line break mechanism. In \LUATEX\ these phases are isolated. As a consequence we
+deal differently with (a sequence of) explicit hyphens. We already have added
+some control over aspects of the hyphenation and yet another one concerns
+automatic hyphens (e.g.\ \type {-} characters in the input).
+
+When \lpr {automatichyphenmode} has a value of 0, a hyphen will be turned into
+an automatic discretionary. The snippets before and after it will not be
+hyphenated. A side effect is that a leading hyphen can lead to a split but one
+will seldom run into that situation. Setting a pre and post character makes this
+more prominent. A value of 1 will prevent this side effect and a value of 2 will
+not turn the hyphen into a discretionary. Experiments with other options, like
+permitting hyphenation of the words on both sides were discarded.
+
+\startbuffer[a]
+before-after \par
+before--after \par
+before---after \par
+\stopbuffer
+
+\startbuffer[b]
+-before \par
+after- \par
+--before \par
+after-- \par
+---before \par
+after--- \par
+\stopbuffer
+
+\startbuffer[c]
+before-after \par
+before--after \par
+before---after \par
+\stopbuffer
+
+\startbuffer[demo]
+\startcombination[nx=4,ny=3,location=top]
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize6em \getbuffer[a]}} {A~0~6em}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize2pt \getbuffer[a]}} {A~0~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plusone \hsize2pt \getbuffer[a]}} {A~1~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plustwo \hsize2pt \getbuffer[a]}} {A~2~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize6em \getbuffer[b]}} {B~0~6em}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize2pt \getbuffer[b]}} {B~0~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plusone \hsize2pt \getbuffer[b]}} {B~1~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plustwo \hsize2pt \getbuffer[b]}} {B~2~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize6em \getbuffer[c]}} {C~0~6em}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\zerocount \hsize2pt \getbuffer[c]}} {C~0~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plusone \hsize2pt \getbuffer[c]}} {C~1~2pt}
+ {\framed[align=normal,strut=no,top=\vskip.5ex,bottom=\vskip.5ex]{\automatichyphenmode\plustwo \hsize2pt \getbuffer[c]}} {C~2~2pt}
+\stopcombination
+\stopbuffer
+
+\startplacefigure[locationreference=automatichyphenmode:1,title={The automatic modes \type {0} (default), \type {1} and \type {2}, with a \prm {hsize}
+of 6em and 2pt (which triggers a linebreak).}]
+ \dontcomplain \tt \getbuffer[demo]
+\stopplacefigure
+
+\startplacefigure[reference=automatichyphenmode:2,title={The automatic modes \type {0} (default), \type {1} and \type {2}, with \lpr {preexhyphenchar} and \lpr {postexhyphenchar} set to characters \type {A} and \type {B}.}]
+ \postexhyphenchar`A\relax
+ \preexhyphenchar `B\relax
+ \dontcomplain \tt \getbuffer[demo]
+\stopplacefigure
+
+In \in {figure} [automatichyphenmode:1] \in {and} [automatichyphenmode:2] we show
+what happens with three samples:
+
+Input A: \typebuffer[a]
+Input B: \typebuffer[b]
+Input C: \typebuffer[c]
+
+As with primitive companions of other single character commands, the \prm {-}
+command has a more verbose primitive version in \lpr {explicitdiscretionary}
+and the normally intercepted in the hyphenator character \type {-} (or whatever
+is configured) is available as \lpr {automaticdiscretionary}.
+
+\stopsection
+
+\startsection[title={The main control loop}]
+
+\topicindex {main loop}
+\topicindex {hyphenation}
+
+In \LUATEX's main loop, almost all input characters that are to be typeset are
+converted into \nod {glyph} node records with subtype \quote {character}, but
+there are a few exceptions.
+
+\startitemize[n]
+
+\startitem
+ The \prm {accent} primitive creates nodes with subtype \quote {glyph}
+ instead of \quote {character}: one for the actual accent and one for the
+ accentee. The primary reason for this is that \prm {accent} in \TEX82 is
+ explicitly dependent on the current font encoding, so it would not make much
+ sense to attach a new meaning to the primitive's name, as that would
+ invalidate many old documents and macro packages. A secondary reason is that
+ in \TEX82, \prm {accent} prohibits hyphenation of the current word. Since
+ in \LUATEX\ hyphenation only takes place on \quote {character} nodes, it is
+ possible to achieve the same effect. Of course, modern \UNICODE\ aware macro
+ packages will not use the \prm {accent} primitive at all but try to map
+ directly on composed characters.
+
+ This change of meaning did happen with \prm {char}, that now generates
+ \quote {glyph} nodes with a character subtype. In traditional \TEX\ there was
+ a strong relationship between the 8|-|bit input encoding, hyphenation and
+ glyphs taken from a font. In \LUATEX\ we have \UTF\ input, and in most cases
+ this maps directly to a character in a font, apart from glyph replacement in
+ the font engine. If you want to access arbitrary glyphs in a font directly
+ you can always use \LUA\ to do so, because fonts are available as \LUA\
+ table.
+\stopitem
+
+\startitem
+ All the results of processing in math mode eventually become nodes with
+ \quote {glyph} subtypes. In fact, the result of processing math is just
+ a regular list of glyphs, kerns, glue, penalties, boxes etc.
+\stopitem
+
+\startitem
+ The \ALEPH|-|derived commands \lpr {leftghost} and \lpr {rightghost}
+ create nodes of a third subtype: \quote {ghost}. These nodes are ignored
+ completely by all further processing until the stage where inter|-|glyph
+ kerning is added.
+\stopitem
+
+\startitem
+ Automatic discretionaries are handled differently. \TEX82 inserts an empty
+ discretionary after sensing an input character that matches the \prm
+ {hyphenchar} in the current font. This test is wrong in our opinion: whether
+ or not hyphenation takes place should not depend on the current font, it is a
+ language property. \footnote {When \TEX\ showed up we didn't have \UNICODE\
+ yet and being limited to eight bits meant that one sometimes had to
+ compromise between supporting character input, glyph rendering, hyphenation.}
+
+ In \LUATEX, it works like this: if \LUATEX\ senses a string of input
+ characters that matches the value of the new integer parameter \prm
+ {exhyphenchar}, it will insert an explicit discretionary after that series of
+ nodes. Initially \TEX\ sets the \type {\exhyphenchar=`\-}. Incidentally, this
+ is a global parameter instead of a language-specific one because it may be
+ useful to change the value depending on the document structure instead of the
+ text language.
+
+ The insertion of discretionaries after a sequence of explicit hyphens happens
+ at the same time as the other hyphenation processing, {\it not\/} inside the
+ main control loop.
+
+ The only use \LUATEX\ has for \prm {hyphenchar} is at the check whether a
+ word should be considered for hyphenation at all. If the \prm {hyphenchar}
+ of the font attached to the first character node in a word is negative, then
+ hyphenation of that word is abandoned immediately. This behaviour is added
+ for backward compatibility only, and the use of \type {\hyphenchar=-1} as a
+ means of preventing hyphenation should not be used in new \LUATEX\ documents.
+\stopitem
+
+\startitem
+ The \prm {setlanguage} command no longer creates whatsits. The meaning of
+ \prm {setlanguage} is changed so that it is now an integer parameter like all
+ others. That integer parameter is used in \type {\glyph_node} creation to add
+ language information to the glyph nodes. In conjunction, the \prm {language}
+ primitive is extended so that it always also updates the value of \prm
+ {setlanguage}.
+\stopitem
+
+\startitem
+ The \prm {noboundary} command (that prohibits word boundary processing
+ where that would normally take place) now does create nodes. These nodes are
+ needed because the exact place of the \prm {noboundary} command in the
+ input stream has to be retained until after the ligature and font processing
+ stages.
+\stopitem
+
+\startitem
+ There is no longer a \type {main_loop} label in the code. Remember that
+ \TEX82 did quite a lot of processing while adding \type {char_nodes} to the
+ horizontal list? For speed reasons, it handled that processing code outside
+ of the \quote {main control} loop, and only the first character of any \quote
+ {word} was handled by that \quote {main control} loop. In \LUATEX, there is
+ no longer a need for that (all hard work is done later), and the (now very
+ small) bits of character|-|handling code have been moved back inline. When
+ \prm {tracingcommands} is on, this is visible because the full word is
+ reported, instead of just the initial character.
+\stopitem
+
+\stopitemize
+
+Because we tend to make hard coded behaviour configurable a few new primitives
+have been added:
+
+\starttyping
+\hyphenpenaltymode
+\automatichyphenpenalty
+\explicithyphenpenalty
+\stoptyping
+
+The first parameter has the following consequences for automatic discs (the ones
+resulting from an \prm {exhyphenchar}:
+
+\starttabulate[|c|l|l|]
+\DB mode \BC automatic disc \type {-} \BC explicit disc \prm{-} \NC \NR
+\TB
+\NC \type{0} \NC \prm {exhyphenpenalty} \NC \prm {exhyphenpenalty} \NC \NR
+\NC \type{1} \NC \prm {hyphenpenalty} \NC \prm {hyphenpenalty} \NC \NR
+\NC \type{2} \NC \prm {exhyphenpenalty} \NC \prm {hyphenpenalty} \NC \NR
+\NC \type{3} \NC \prm {hyphenpenalty} \NC \prm {exhyphenpenalty} \NC \NR
+\NC \type{4} \NC \lpr {automatichyphenpenalty} \NC \lpr {explicithyphenpenalty} \NC \NR
+\NC \type{5} \NC \prm {exhyphenpenalty} \NC \lpr {explicithyphenpenalty} \NC \NR
+\NC \type{6} \NC \prm {hyphenpenalty} \NC \lpr {explicithyphenpenalty} \NC \NR
+\NC \type{7} \NC \lpr {automatichyphenpenalty} \NC \prm {exhyphenpenalty} \NC \NR
+\NC \type{8} \NC \lpr {automatichyphenpenalty} \NC \prm {hyphenpenalty} \NC \NR
+\LL
+\stoptabulate
+
+other values do what we always did in \LUATEX: insert \prm {exhyphenpenalty}.
+
+\stopsection
+
+\startsection[title={Loading patterns and exceptions},reference=patternsexceptions]
+
+\topicindex {hyphenation}
+\topicindex {hyphenation+patterns}
+\topicindex {hyphenation+exceptions}
+\topicindex {patterns}
+\topicindex {exceptions}
+
+Although we keep the traditional approach towards hyphenation (which is still
+superior) the implementation of the hyphenation algorithm in \LUATEX\ is quite
+different from the one in \TEX82.
+
+After expansion, the argument for \prm {patterns} has to be proper \UTF8 with
+individual patterns separated by spaces, no \prm {char} or \prm {chardef}d
+commands are allowed. The current implementation is quite strict and will reject
+all non|-|\UNICODE\ characters. Likewise, the expanded argument for \prm
+{hyphenation} also has to be proper \UTF8, but here a bit of extra syntax is
+provided:
+
+\startitemize[n]
+\startitem
+ Three sets of arguments in curly braces (\type {{}{}{}}) indicate a desired
+ complex discretionary, with arguments as in \prm {discretionary}'s command in
+ normal document input.
+\stopitem
+\startitem
+ A \type {-} indicates a desired simple discretionary, cf.\ \type {\-} and
+ \type {\discretionary{-}{}{}} in normal document input.
+\stopitem
+\startitem
+ Internal command names are ignored. This rule is provided especially for \prm
+ {discretionary}, but it also helps to deal with \prm {relax} commands that
+ may sneak in.
+\stopitem
+\startitem
+ An \type {=} indicates a (non|-|discretionary) hyphen in the document input.
+\stopitem
+\stopitemize
+
+The expanded argument is first converted back to a space|-|separated string while
+dropping the internal command names. This string is then converted into a
+dictionary by a routine that creates key|-|value pairs by converting the other
+listed items. It is important to note that the keys in an exception dictionary
+can always be generated from the values. Here are a few examples:
+
+\starttabulate[|l|l|l|]
+\DB value \BC implied key (input) \BC effect \NC\NR
+\TB
+\NC \type {ta-ble} \NC table \NC \type {ta\-ble} ($=$ \type {ta\discretionary{-}{}{}ble}) \NC\NR
+\NC \type {ba{k-}{}{c}ken} \NC backen \NC \type {ba\discretionary{k-}{}{c}ken} \NC\NR
+\LL
+\stoptabulate
+
+The resultant patterns and exception dictionary will be stored under the language
+code that is the present value of \prm {language}.
+
+In the last line of the table, you see there is no \prm {discretionary} command
+in the value: the command is optional in the \TEX-based input syntax. The
+underlying reason for that is that it is conceivable that a whole dictionary of
+words is stored as a plain text file and loaded into \LUATEX\ using one of the
+functions in the \LUA\ \type {lang} library. This loading method is quite a bit
+faster than going through the \TEX\ language primitives, but some (most?) of that
+speed gain would be lost if it had to interpret command sequences while doing so.
+
+It is possible to specify extra hyphenation points in compound words by using
+\type {{-}{}{-}} for the explicit hyphen character (replace \type {-} by the
+actual explicit hyphen character if needed). For example, this matches the word
+\quote {multi|-|word|-|boundaries} and allows an extra break inbetween \quote
+{boun} and \quote {daries}:
+
+\starttyping
+\hyphenation{multi{-}{}{-}word{-}{}{-}boun-daries}
+\stoptyping
+
+The motivation behind the \ETEX\ extension \prm {savinghyphcodes} was that
+hyphenation heavily depended on font encodings. This is no longer true in
+\LUATEX, and the corresponding primitive is basically ignored. Because we now
+have \lpr {hjcode}, the case relate codes can be used exclusively for \prm
+{uppercase} and \prm {lowercase}.
+
+The three curly brace pair pattern in an exception can be somewhat unexpected so
+we will try to explain it by example. The pattern \type {foo{}{}{x}bar} pattern
+creates a lookup \type {fooxbar} and the pattern \type {foo{}{}{}bar} creates
+\type {foobar}. Then, when a hit happens there is a replacement text (\type {x})
+or none. Because we introduced penalties in discretionary nodes, the exception
+syntax now also can take a penalty specification. The value between square brackets
+is a multiplier for \lpr {exceptionpenalty}. Here we have set it to 10000 so
+effectively we get 30000 in the example.
+
+\def\ShowSample#1#2%
+ {\startlinecorrection[blank]
+ \hyphenation{#1}%
+ \exceptionpenalty=10000
+ \bTABLE[foregroundstyle=type]
+ \bTR
+ \bTD[align=middle,nx=4] \type{#1} \eTD
+ \eTR
+ \bTR
+ \bTD[align=middle] \type{10em} \eTD
+ \bTD[align=middle] \type {3em} \eTD
+ \bTD[align=middle] \type {0em} \eTD
+ \bTD[align=middle] \type {6em} \eTD
+ \eTR
+ \bTR
+ \bTD[width=10em]\vtop{\hsize 10em 123 #2 123\par}\eTD
+ \bTD[width=10em]\vtop{\hsize 3em 123 #2 123\par}\eTD
+ \bTD[width=10em]\vtop{\hsize 0em 123 #2 123\par}\eTD
+ \bTD[width=10em]\vtop{\setupalign[verytolerant,stretch]\rmtf\hsize 6em 123 #2 #2 #2 #2 123\par}\eTD
+ \eTR
+ \eTABLE
+ \stoplinecorrection}
+
+\ShowSample{x{a-}{-b}{}x{a-}{-b}{}x{a-}{-b}{}x{a-}{-b}{}xx}{xxxxxx}
+\ShowSample{x{a-}{-b}{}x{a-}{-b}{}[3]x{a-}{-b}{}[1]x{a-}{-b}{}xx}{xxxxxx}
+
+\ShowSample{z{a-}{-b}{z}{a-}{-b}{z}{a-}{-b}{z}{a-}{-b}{z}z}{zzzzzz}
+\ShowSample{z{a-}{-b}{z}{a-}{-b}{z}[3]{a-}{-b}{z}[1]{a-}{-b}{z}z}{zzzzzz}
+
+\stopsection
+
+\startsection[title={Applying hyphenation}]
+
+\topicindex {hyphenation+how it works}
+\topicindex {hyphenation+discretionaries}
+\topicindex {discretionaries}
+
+The internal structures \LUATEX\ uses for the insertion of discretionaries in
+words is very different from the ones in \TEX82, and that means there are some
+noticeable differences in handling as well.
+
+First and foremost, there is no \quote {compressed trie} involved in hyphenation.
+The algorithm still reads pattern files generated by \PATGEN, but \LUATEX\ uses a
+finite state hash to match the patterns against the word to be hyphenated. This
+algorithm is based on the \quote {libhnj} library used by \OPENOFFICE, which in
+turn is inspired by \TEX.
+
+There are a few differences between \LUATEX\ and \TEX82 that are a direct result
+of the implementation:
+
+\startitemize
+\startitem
+ \LUATEX\ happily hyphenates the full \UNICODE\ character range.
+\stopitem
+\startitem
+ Pattern and exception dictionary size is limited by the available memory
+ only, all allocations are done dynamically. The trie|-|related settings in
+ \type {texmf.cnf} are ignored.
+\stopitem
+\startitem
+ Because there is no \quote {trie preparation} stage, language patterns never
+ become frozen. This means that the primitive \prm {patterns} (and its \LUA\
+ counterpart \type {lang.patterns}) can be used at any time, not only in
+ ini\TEX.
+\stopitem
+\startitem
+ Only the string representation of \prm {patterns} and \prm {hyphenation} is
+ stored in the format file. At format load time, they are simply
+ re|-|evaluated. It follows that there is no real reason to preload languages
+ in the format file. In fact, it is usually not a good idea to do so. It is
+ much smarter to load patterns no sooner than the first time they are actually
+ needed.
+\stopitem
+\startitem
+ \LUATEX\ uses the language-specific variables \lpr {prehyphenchar} and \lpr
+ {posthyphenchar} in the creation of implicit discretionaries, instead of
+ \TEX82's \prm {hyphenchar}, and the values of the language|-|specific
+ variables \lpr {preexhyphenchar} and \lpr {postexhyphenchar} for explicit
+ discretionaries (instead of \TEX82's empty discretionary).
+\stopitem
+\startitem
+ The value of the two counters related to hyphenation, \prm {hyphenpenalty}
+ and \prm {exhyphenpenalty}, are now stored in the discretionary nodes. This
+ permits a local overload for explicit \prm {discretionary} commands. The
+ value current when the hyphenation pass is applied is used. When no callbacks
+ are used this is compatible with traditional \TEX. When you apply the \LUA\
+ \type {lang.hyphenate} function the current values are used.
+\stopitem
+\startitem
+ The hyphenation exception dictionary is maintained as key|-|value hash, and
+ that is also dynamic, so the \type {hyph_size} setting is not used either.
+\stopitem
+\stopitemize
+
+Because we store penalties in the disc node the \prm {discretionary} command has
+been extended to accept an optional penalty specification, so you can do the
+following:
+
+\startbuffer
+\hsize1mm
+1:foo{\hyphenpenalty 10000\discretionary{}{}{}}bar\par
+2:foo\discretionary penalty 10000 {}{}{}bar\par
+3:foo\discretionary{}{}{}bar\par
+\stopbuffer
+
+\typebuffer
+
+This results in:
+
+\blank \start \getbuffer \stop \blank
+
+Inserted characters and ligatures inherit their attributes from the nearest glyph
+node item (usually the preceding one, but the following one for the items
+inserted at the left-hand side of a word).
+
+Word boundaries are no longer implied by font switches, but by language switches.
+One word can have two separate fonts and still be hyphenated correctly (but it
+can not have two different languages, the \prm {setlanguage} command forces a
+word boundary).
+
+All languages start out with \type {\prehyphenchar=`\-}, \type {\posthyphenchar=0},
+\type {\preexhyphenchar=0} and \type {\postexhyphenchar=0}. When you assign the
+values of one of these four parameters, you are actually changing the settings
+for the current \prm {language}, this behaviour is compatible with \prm {patterns}
+and \prm {hyphenation}.
+
+\LUATEX\ also hyphenates the first word in a paragraph. Words can be up to 256
+characters long (up from 64 in \TEX82). Longer words are ignored right now, but
+eventually either the limitation will be removed or perhaps it will become
+possible to silently ignore the excess characters (this is what happens in
+\TEX82, but there the behaviour cannot be controlled).
+
+If you are using the \LUA\ function \type {lang.hyphenate}, you should be aware
+that this function expects to receive a list of \quote {character} nodes. It will
+not operate properly in the presence of \quote {glyph}, \quote {ligature}, or
+\quote {ghost} nodes, nor does it know how to deal with kerning.
+
+\stopsection
+
+\startsection[title={Applying ligatures and kerning}]
+
+\topicindex {ligatures}
+\topicindex {kerning}
+
+After all possible hyphenation points have been inserted in the list, \LUATEX\
+will process the list to convert the \quote {character} nodes into \quote {glyph}
+and \quote {ligature} nodes. This is actually done in two stages: first all
+ligatures are processed, then all kerning information is applied to the result
+list. But those two stages are somewhat dependent on each other: If the used font
+makes it possible to do so, the ligaturing stage adds virtual \quote {character}
+nodes to the word boundaries in the list. While doing so, it removes and
+interprets \prm {noboundary} nodes. The kerning stage deletes those word
+boundary items after it is done with them, and it does the same for \quote
+{ghost} nodes. Finally, at the end of the kerning stage, all remaining \quote
+{character} nodes are converted to \quote {glyph} nodes.
+
+This word separation is worth mentioning because, if you overrule from \LUA\ only
+one of the two callbacks related to font handling, then you have to make sure you
+perform the tasks normally done by \LUATEX\ itself in order to make sure that the
+other, non|-|overruled, routine continues to function properly.
+
+Although we could improve the situation the reality is that in modern \OPENTYPE\
+fonts ligatures can be constructed in many ways: by replacing a sequence of
+characters by one glyph, or by selectively replacing individual glyphs, or by
+kerning, or any combination of this. Add to that contextual analysis and it will
+be clear that we have to let \LUA\ do that job instead. The generic font handler
+that we provide (which is part of \CONTEXT) distinguishes between base mode
+(which essentially is what we describe here and which delegates the task to \TEX)
+and node mode (which deals with more complex fonts.
+
+Let's look at an example. Take the word \type {office}, hyphenated \type
+{of-fice}, using a \quote {normal} font with all the \type {f}-\type {f} and
+\type {f}-\type {i} type ligatures:
+
+\starttabulate[|l|l|]
+\NC initial \NC \type {{o}{f}{f}{i}{c}{e}} \NC\NR
+\NC after hyphenation \NC \type {{o}{f}{{-},{},{}}{f}{i}{c}{e}} \NC\NR
+\NC first ligature stage \NC \type {{o}{{f-},{f},{<ff>}}{i}{c}{e}} \NC\NR
+\NC final result \NC \type {{o}{{f-},{<fi>},{<ffi>}}{c}{e}} \NC\NR
+\stoptabulate
+
+That's bad enough, but let us assume that there is also a hyphenation point
+between the \type {f} and the \type {i}, to create \type {of-f-ice}. Then the
+final result should be:
+
+\starttyping
+{o}{{f-},
+ {{f-},
+ {i},
+ {<fi>}},
+ {{<ff>-},
+ {i},
+ {<ffi>}}}{c}{e}
+\stoptyping
+
+with discretionaries in the post-break text as well as in the replacement text of
+the top-level discretionary that resulted from the first hyphenation point.
+
+Here is that nested solution again, in a different representation:
+
+\testpage[4]
+
+\starttabulate[|l|c|c|c|c|c|c|]
+\DB \BC pre \BC \BC post \BC \BC replace \BC \NC \NR
+\TB
+\NC topdisc \NC \type {f-} \NC (1) \NC \NC sub 1 \NC \NC sub 2 \NC \NR
+\NC sub 1 \NC \type {f-} \NC (2) \NC \type {i} \NC (3) \NC \type {<fi>} \NC (4) \NC \NR
+\NC sub 2 \NC \type {<ff>-} \NC (5) \NC \type {i} \NC (6) \NC \type {<ffi>} \NC (7) \NC \NR
+\LL
+\stoptabulate
+
+When line breaking is choosing its breakpoints, the following fields will
+eventually be selected:
+
+\starttabulate[|l|c|c|]
+\NC \type {of-f-ice} \NC \type {f-} \NC (1) \NC \NR
+\NC \NC \type {f-} \NC (2) \NC \NR
+\NC \NC \type {i} \NC (3) \NC \NR
+\NC \type {of-fice} \NC \type {f-} \NC (1) \NC \NR
+\NC \NC \type {<fi>} \NC (4) \NC \NR
+\NC \type {off-ice} \NC \type {<ff>-} \NC (5) \NC \NR
+\NC \NC \type {i} \NC (6) \NC \NR
+\NC \type {office} \NC \type {<ffi>} \NC (7) \NC \NR
+\stoptabulate
+
+The current solution in \LUATEX\ is not able to handle nested discretionaries,
+but it is in fact smart enough to handle this fictional \type {of-f-ice} example.
+It does so by combining two sequential discretionary nodes as if they were a
+single object (where the second discretionary node is treated as an extension of
+the first node).
+
+One can observe that the \type {of-f-ice} and \type {off-ice} cases both end with
+the same actual post replacement list (\type {i}), and that this would be the
+case even if \type {i} was the first item of a potential following ligature like
+\type {ic}. This allows \LUATEX\ to do away with one of the fields, and thus make
+the whole stuff fit into just two discretionary nodes.
+
+The mapping of the seven list fields to the six fields in this discretionary node
+pair is as follows:
+
+\starttabulate[|l|c|c|]
+\DB field \BC description \NC \NC \NR
+\TB
+\NC \type {disc1.pre} \NC \type {f-} \NC (1) \NC \NR
+\NC \type {disc1.post} \NC \type {<fi>} \NC (4) \NC \NR
+\NC \type {disc1.replace} \NC \type {<ffi>} \NC (7) \NC \NR
+\NC \type {disc2.pre} \NC \type {f-} \NC (2) \NC \NR
+\NC \type {disc2.post} \NC \type {i} \NC (3,6) \NC \NR
+\NC \type {disc2.replace} \NC \type {<ff>-} \NC (5) \NC \NR
+\LL
+\stoptabulate
+
+What is actually generated after ligaturing has been applied is therefore:
+
+\starttyping
+{o}{{f-},
+ {<fi>},
+ {<ffi>}}
+ {{f-},
+ {i},
+ {<ff>-}}{c}{e}
+\stoptyping
+
+The two discretionaries have different subtypes from a discretionary appearing on
+its own: the first has subtype 4, and the second has subtype 5. The need for
+these special subtypes stems from the fact that not all of the fields appear in
+their \quote {normal} location. The second discretionary especially looks odd,
+with things like the \type {<ff>-} appearing in \type {disc2.replace}. The fact
+that some of the fields have different meanings (and different processing code
+internally) is what makes it necessary to have different subtypes: this enables
+\LUATEX\ to distinguish this sequence of two joined discretionary nodes from the
+case of two standalone discretionaries appearing in a row.
+
+Of course there is still that relationship with fonts: ligatures can be implemented by
+mapping a sequence of glyphs onto one glyph, but also by selective replacement and
+kerning. This means that the above examples are just representing the traditional
+approach.
+
+\stopsection
+
+\startsection[title={Breaking paragraphs into lines}]
+
+\topicindex {linebreaks}
+\topicindex {paragraphs}
+\topicindex {discretionaries}
+
+This code is almost unchanged, but because of the above|-|mentioned changes
+with respect to discretionaries and ligatures, line breaking will potentially be
+different from traditional \TEX. The actual line breaking code is still based on
+the \TEX82 algorithms, and it does not expect there to be discretionaries inside
+of discretionaries. But, as patterns evolve and font handling can influence
+discretionaries, you need to be aware of the fact that long term consistency is not
+an engine matter only.
+
+But that situation is now fairly common in \LUATEX, due to the changes to the
+ligaturing mechanism. And also, the \LUATEX\ discretionary nodes are implemented
+slightly different from the \TEX82 nodes: the \type {no_break} text is now
+embedded inside the disc node, where previously these nodes kept their place in
+the horizontal list. In traditional \TEX\ the discretionary node contains a
+counter indicating how many nodes to skip, but in \LUATEX\ we store the pre, post
+and replace text in the discretionary node.
+
+The combined effect of these two differences is that \LUATEX\ does not always use
+all of the potential breakpoints in a paragraph, especially when fonts with many
+ligatures are used. Of course kerning also complicates matters here.
+
+\stopsection
+
+\startsection[title={The \type {lang} library}][library=lang]
+
+\subsection {\type {new} and \type {id}}
+
+\topicindex {languages+library}
+
+\libindex {new}
+\libindex {id}
+
+This library provides the interface to \LUATEX's structure representing a
+language, and the associated functions.
+
+\startfunctioncall
+<language> l = lang.new()
+<language> l = lang.new(<number> id)
+\stopfunctioncall
+
+This function creates a new userdata object. An object of type \type {<language>}
+is the first argument to most of the other functions in the \type {lang} library.
+These functions can also be used as if they were object methods, using the colon
+syntax. Without an argument, the next available internal id number will be
+assigned to this object. With argument, an object will be created that links to
+the internal language with that id number.
+
+\startfunctioncall
+<number> n = lang.id(<language> l)
+\stopfunctioncall
+
+The number returned is the internal \prm {language} id number this object refers to.
+
+\subsection {\type {hyphenation}}
+
+\libindex {hyphenation}
+
+You can hyphenate a string directly with:
+
+\startfunctioncall
+<string> n = lang.hyphenation(<language> l)
+lang.hyphenation(<language> l, <string> n)
+\stopfunctioncall
+
+\subsection {\type {clear_hyphenation} and \type {clean}}
+
+\libindex {clear_hyphenation}
+\libindex {clean}
+
+This either returns the current hyphenation exceptions for this language, or adds
+new ones. The syntax of the string is explained in~\in {section}
+[patternsexceptions].
+
+\startfunctioncall
+lang.clear_hyphenation(<language> l)
+\stopfunctioncall
+
+This call clears the exception dictionary (string) for this language.
+
+\startfunctioncall
+<string> n = lang.clean(<language> l, <string> o)
+<string> n = lang.clean(<string> o)
+\stopfunctioncall
+
+This function creates a hyphenation key from the supplied hyphenation value. The
+syntax of the argument string is explained in \in {section} [patternsexceptions].
+This function is useful if you want to do something else based on the words in a
+dictionary file, like spell|-|checking.
+
+\subsection {\type {patterns} and \type {clear_patterns}}
+
+\libindex {patterns}
+\libindex {clear_patterns}
+
+\startfunctioncall
+<string> n = lang.patterns(<language> l)
+lang.patterns(<language> l, <string> n)
+\stopfunctioncall
+
+This adds additional patterns for this language object, or returns the current
+set. The syntax of this string is explained in \in {section}
+[patternsexceptions].
+
+\startfunctioncall
+lang.clear_patterns(<language> l)
+\stopfunctioncall
+
+This can be used to clear the pattern dictionary for a language.
+
+\subsection {\type {hyphenationmin}}
+
+\libindex {hyphenationmin}
+
+This function sets (or gets) the value of the \TEX\ parameter
+\type {\hyphenationmin}.
+
+\startfunctioncall
+n = lang.hyphenationmin(<language> l)
+lang.hyphenationmin(<language> l, <number> n)
+\stopfunctioncall
+
+\subsection {\type {[pre|post][ex|]hyphenchar}}
+
+\libindex {prehyphenchar}
+\libindex {posthyphenchar}
+\libindex {preexhyphenchar}
+\libindex {postexhyphenchar}
+
+\startfunctioncall
+<number> n = lang.prehyphenchar(<language> l)
+lang.prehyphenchar(<language> l, <number> n)
+
+<number> n = lang.posthyphenchar(<language> l)
+lang.posthyphenchar(<language> l, <number> n)
+\stopfunctioncall
+
+These two are used to get or set the \quote {pre|-|break} and \quote
+{post|-|break} hyphen characters for implicit hyphenation in this language. The
+intial values are decimal 45 (hyphen) and decimal~0 (indicating emptiness).
+
+\startfunctioncall
+<number> n = lang.preexhyphenchar(<language> l)
+lang.preexhyphenchar(<language> l, <number> n)
+
+<number> n = lang.postexhyphenchar(<language> l)
+lang.postexhyphenchar(<language> l, <number> n)
+\stopfunctioncall
+
+These gets or set the \quote {pre|-|break} and \quote {post|-|break} hyphen
+characters for explicit hyphenation in this language. Both are initially
+decimal~0 (indicating emptiness).
+
+\subsection {\type {hyphenate}}
+
+\libindex {hyphenate}
+
+The next call inserts hyphenation points (discretionary nodes) in a node list. If
+\type {tail} is given as argument, processing stops on that node. Currently,
+\type {success} is always true if \type {head} (and \type {tail}, if specified)
+are proper nodes, regardless of possible other errors.
+
+\startfunctioncall
+<boolean> success = lang.hyphenate(<node> head)
+<boolean> success = lang.hyphenate(<node> head, <node> tail)
+\stopfunctioncall
+
+Hyphenation works only on \quote {characters}, a special subtype of all the glyph
+nodes with the node subtype having the value \type {1}. Glyph modes with
+different subtypes are not processed. See \in {section} [charsandglyphs] for
+more details.
+
+\subsection {\type {[set|get]hjcode}}
+
+\libindex {sethjcode}
+\libindex {gethjcode}
+
+The following two commands can be used to set or query hj codes:
+
+\startfunctioncall
+lang.sethjcode(<language> l, <number> char, <number> usedchar)
+<number> usedchar = lang.gethjcode(<language> l, <number> char)
+\stopfunctioncall
+
+When you set a hjcode the current sets get initialized unless the set was already
+initialized due to \prm {savinghyphcodes} being larger than zero.
+
+\stopsection
+
+\stopchapter
+
+\stopcomponent
+
+% \parindent0pt \hsize=1.1cm
+% 12-34-56 \par
+% 12-34-\hbox{56} \par
+% 12-34-\vrule width 1em height 1.5ex \par
+% 12-\hbox{34}-56 \par
+% 12-\vrule width 1em height 1.5ex-56 \par
+% \hjcode`\1=`\1 \hjcode`\2=`\2 \hjcode`\3=`\3 \hjcode`\4=`\4 \vskip.5cm
+% 12-34-56 \par
+% 12-34-\hbox{56} \par
+% 12-34-\vrule width 1em height 1.5ex \par
+% 12-\hbox{34}-56 \par
+% 12-\vrule width 1em height 1.5ex-56 \par
+
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex b/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex
index 50db9593b..a126a95dc 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-lua.tex
@@ -50,7 +50,7 @@ path as the binary itself
When the \LUAMETATEX\ executable starts, it looks for the \type {--lua} command line
option. If there is no \type {--lua} option, the command line is interpreted in a
similar fashion as the other \TEX\ engines. All options are accepted but only some
-have are understood by \LUAMETATEX\ itself:
+are understood by \LUAMETATEX\ itself:
\starttabulate[|l|p|]
\DB commandline argument \BC explanation \NC \NR
@@ -149,7 +149,7 @@ normally with some delay. Therefore the user needs to keep an eye on (subtle)
differences in successive versions of the language. Here is an example of one
aspect.
-\LUA s \type {tostring} function (and \type {string.format} may return values in
+\LUA s \type {tostring} function (and \type {string.format}) may return values in
scientific notation, thereby confusing the \TEX\ end of things when it is used as
the right|-|hand side of an assignment to a \prm {dimen} or \prm {count}. The
output of these serializers also depend on the \LUA\ version, so in \LUA\ 5.3 you
@@ -187,7 +187,7 @@ language specific conversions needed at the \TEX\ end.
Of course the regular \LUA\ modules are present. In addition we provide the \type
{lpeg} library by Roberto Ierusalimschy, This library is not \UNICODE|-|aware,
but interprets strings on a byte|-|per|-|byte basis. This mainly means that \type
-{lpeg.S} cannot be used with \UTF8 characters encoded in more than two bytes, and
+{lpeg.S} cannot be used with \UTF8 characters that need more than one byte, and
thus \type {lpeg.S} will look for one of those two bytes when matching, not the
combination of the two. The same is true for \type {lpeg.R}, although the latter
will display an error message if used with multibyte characters. Therefore \type
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex b/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex
index dca6c3781..6448f2b01 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-modifications.tex
@@ -22,7 +22,7 @@ many years, when the engine had become more stable, the decision was made to
clean up the rather hybrid nature of the program. This means that some primitives
were promoted to core primitives, often with a different name, and that others
were removed. This also made it possible to start cleaning up the code base. In
-\in {chapter} [enhancements] we discussed some new primitives, here we will cover
+\in {chapter} [enhancements] we discuss some new primitives, here we will cover
most of the adapted ones.
During more than a decade stepwise new functionality was added and after 10 years
@@ -56,13 +56,13 @@ most still comes from original Knuthian \TEX. But we divert a bit.
files organized in categories like \type {tex}, \type {luaf}, \type
{languages}, \type {fonts}, \type {libraries}, etc. There are some artifacts
of the conversion to \CCODE, but these got (and get) removed stepwise. The
- documentation, which is actually comes from the mix of engines (via so called
- change files) is kept as much as possible. Of course we want to stay as close
- as possible to the original so that the documentation of the fundamentals
- behind \TEX\ by Don Knuth still applies. However, because we use \CCODE, some
- documentation is a bit off. Also, most global variables are now collected in
- structures, but the original names were kept. There are lots of so called
- macros too.
+ documentation, which actually comes from the mix of engines (via so called
+ change files), is kept as much as possible. Of course we want to stay as
+ close as possible to the original so that the documentation of the
+ fundamentals behind \TEX\ by Don Knuth still applies. However, because we use
+ \CCODE, some documentation is a bit off. Also, most global variables are now
+ collected in structures, but the original names were kept. There are lots of
+ so called macros too.
\stopitem
\startitem
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex b/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
index 1d408ba6c..8b4f6101c 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-nodes.tex
@@ -2513,7 +2513,7 @@ demands a pretty strong agreement over what bit represents what, and this is
unlikely to succeed in the \TEX\ community. It doesn't pay off.
Just in case one wonders why properties make sense: it is not so much speed that
-we gain, but more convenience: storing all kind of (temporary) data in attributes
+we gain, but more convenience: storing all kinds of (temporary) data in attributes
is no fun and this mechanism makes sure that properties are cleaned up when a
node is freed. Also, the advantage of a more or less global properties table is
that we stay at the \LUA\ end. An alternative is to store a reference in the node
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex b/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
index d02829bea..839bae409 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
@@ -1656,7 +1656,10 @@ primitives \quote{\tex{ }}, \quote{\tex {/}} and \quote{\type {-}} are defined.
\startluacode
function document.showprimitives(tag)
- for k, v in table.sortedpairs(tex.extraprimitives(tag)) do
+ local t = tex.extraprimitives(tag)
+ table.sort(t)
+ for i=1,#t do
+ local v = t[i]
if v ~= ' ' and v ~= "/" and v ~= "-" then
context.type(v)
context.space()
@@ -1666,12 +1669,12 @@ end
\stopluacode
\starttabulate[|l|pl|]
-\DB name \BC values \NC \NR
+\DB name \BC values \NC \NR
\TB
-\NC tex \NC \ctxlua{document.showprimitives('tex') } \NC \NR
-\NC core \NC \ctxlua{document.showprimitives('core') } \NC \NR
-\NC etex \NC \ctxlua{document.showprimitives('etex') } \NC \NR
-\NC luatex \NC \ctxlua{document.showprimitives('luatex') } \NC \NR
+\NC tex \NC \ctxlua{document.showprimitives('tex') } \NC \NR
+\NC core \NC \ctxlua{document.showprimitives('core') } \NC \NR
+\NC etex \NC \ctxlua{document.showprimitives('etex') } \NC \NR
+\NC luatex \NC \ctxlua{document.showprimitives('luatex') } \NC \NR
\LL
\stoptabulate
diff --git a/scripts/context/lua/mtx-grep.lua b/scripts/context/lua/mtx-grep.lua
index 9a4237737..e4a2a8d2f 100644
--- a/scripts/context/lua/mtx-grep.lua
+++ b/scripts/context/lua/mtx-grep.lua
@@ -173,15 +173,17 @@ function scripts.grep.find(pattern, files, offset)
local globbed = dir.glob(files[i])
for i=1,#globbed do
name = globbed[i]
- local data = io.loaddata(name)
- if data then
- n, m, noffiles = 0, 0, noffiles + 1
- lpegmatch(capture,data)
- if count and m > 0 then
- nofmatches = nofmatches + m
- nofmatchedfiles = nofmatchedfiles + 1
- write_nl(format("%5i %s",m,name))
- io.flush()
+ if not find(name,"/%.") then
+ local data = io.loaddata(name)
+ if data then
+ n, m, noffiles = 0, 0, noffiles + 1
+ lpegmatch(capture,data)
+ if count and m > 0 then
+ nofmatches = nofmatches + m
+ nofmatchedfiles = nofmatchedfiles + 1
+ write_nl(format("%5i %s",m,name))
+ io.flush()
+ end
end
end
end
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 5f48eb803..bcc3bb243 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -20896,7 +20896,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-exp"] = package.loaded["data-exp"] or true
--- original size: 18173, stripped down to: 10432
+-- original size: 18179, stripped down to: 10432
if not modules then modules={} end modules ['data-exp']={
version=1.001,
@@ -22117,7 +22117,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-res"] = package.loaded["data-res"] or true
--- original size: 69575, stripped down to: 44470
+-- original size: 69576, stripped down to: 44470
if not modules then modules={} end modules ['data-res']={
version=1.001,
@@ -26189,8 +26189,8 @@ end -- of closure
-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 1047135
--- stripped bytes : 416869
+-- original bytes : 1047142
+-- stripped bytes : 416876
-- end library merge
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 5f48eb803..bcc3bb243 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -20896,7 +20896,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-exp"] = package.loaded["data-exp"] or true
--- original size: 18173, stripped down to: 10432
+-- original size: 18179, stripped down to: 10432
if not modules then modules={} end modules ['data-exp']={
version=1.001,
@@ -22117,7 +22117,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-res"] = package.loaded["data-res"] or true
--- original size: 69575, stripped down to: 44470
+-- original size: 69576, stripped down to: 44470
if not modules then modules={} end modules ['data-res']={
version=1.001,
@@ -26189,8 +26189,8 @@ end -- of closure
-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 1047135
--- stripped bytes : 416869
+-- original bytes : 1047142
+-- stripped bytes : 416876
-- end library merge
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 5f48eb803..bcc3bb243 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -20896,7 +20896,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-exp"] = package.loaded["data-exp"] or true
--- original size: 18173, stripped down to: 10432
+-- original size: 18179, stripped down to: 10432
if not modules then modules={} end modules ['data-exp']={
version=1.001,
@@ -22117,7 +22117,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-res"] = package.loaded["data-res"] or true
--- original size: 69575, stripped down to: 44470
+-- original size: 69576, stripped down to: 44470
if not modules then modules={} end modules ['data-res']={
version=1.001,
@@ -26189,8 +26189,8 @@ end -- of closure
-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 1047135
--- stripped bytes : 416869
+-- original bytes : 1047142
+-- stripped bytes : 416876
-- end library merge
diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua
index 5f48eb803..bcc3bb243 100644
--- a/scripts/context/stubs/win64/mtxrun.lua
+++ b/scripts/context/stubs/win64/mtxrun.lua
@@ -20896,7 +20896,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-exp"] = package.loaded["data-exp"] or true
--- original size: 18173, stripped down to: 10432
+-- original size: 18179, stripped down to: 10432
if not modules then modules={} end modules ['data-exp']={
version=1.001,
@@ -22117,7 +22117,7 @@ do -- create closure to overcome 200 locals limit
package.loaded["data-res"] = package.loaded["data-res"] or true
--- original size: 69575, stripped down to: 44470
+-- original size: 69576, stripped down to: 44470
if not modules then modules={} end modules ['data-res']={
version=1.001,
@@ -26189,8 +26189,8 @@ end -- of closure
-- used libraries : l-bit32.lua l-lua.lua l-macro.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-sha.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua util-soc-imp-reset.lua util-soc-imp-socket.lua util-soc-imp-copas.lua util-soc-imp-ltn12.lua util-soc-imp-mime.lua util-soc-imp-url.lua util-soc-imp-headers.lua util-soc-imp-tp.lua util-soc-imp-http.lua util-soc-imp-ftp.lua util-soc-imp-smtp.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua util-zip.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua
-- skipped libraries : -
--- original bytes : 1047135
--- stripped bytes : 416869
+-- original bytes : 1047142
+-- stripped bytes : 416876
-- end library merge
diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii
index 20cc26b93..9e0025e0f 100644
--- a/tex/context/base/mkii/cont-new.mkii
+++ b/tex/context/base/mkii/cont-new.mkii
@@ -11,7 +11,7 @@
%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
%C details.
-\newcontextversion{2020.01.15 19:09}
+\newcontextversion{2020.01.26 18:34}
%D This file is loaded at runtime, thereby providing an
%D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii
index 87441059d..aa7c02cf4 100644
--- a/tex/context/base/mkii/context.mkii
+++ b/tex/context/base/mkii/context.mkii
@@ -20,7 +20,7 @@
%D your styles an modules.
\edef\contextformat {\jobname}
-\edef\contextversion{2020.01.15 19:09}
+\edef\contextversion{2020.01.26 18:34}
%D For those who want to use this:
diff --git a/tex/context/base/mkiv/attr-ini.mkiv b/tex/context/base/mkiv/attr-ini.mkiv
index e4215ff00..49a1126b9 100644
--- a/tex/context/base/mkiv/attr-ini.mkiv
+++ b/tex/context/base/mkiv/attr-ini.mkiv
@@ -185,6 +185,12 @@
\unexpanded\def\showattributes{\clf_showattributes}
+%D Todo:
+
+% \appendtoks
+% \clf_cleanupattributes
+% \to \everyafterpagebreak
+
\protect \endinput
% for the luatex list:
diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index ce1abd040..2cc2a1098 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -13,7 +13,7 @@
% \normalend % uncomment this to get the real base runtime
-\newcontextversion{2020.01.15 19:09}
+\newcontextversion{2020.01.26 18:34}
%D This file is loaded at runtime, thereby providing an excellent place for
%D hacks, patches, extensions and new features.
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index f75556d26..d4206a4e8 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -45,7 +45,7 @@
%D {YYYY.MM.DD HH:MM} format.
\edef\contextformat {\jobname}
-\edef\contextversion{2020.01.15 19:09}
+\edef\contextversion{2020.01.26 18:34}
\edef\contextkind {beta}
%D Kind of special:
diff --git a/tex/context/base/mkiv/context.mkxl b/tex/context/base/mkiv/context.mkxl
index c3954baee..1b4da4e96 100644
--- a/tex/context/base/mkiv/context.mkxl
+++ b/tex/context/base/mkiv/context.mkxl
@@ -29,7 +29,7 @@
%D {YYYY.MM.DD HH:MM} format.
\edef\contextformat {\jobname}
-\edef\contextversion{2020.01.15 19:09}
+\edef\contextversion{2020.01.26 18:34}
\edef\contextkind {beta}
%D Kind of special:
diff --git a/tex/context/base/mkiv/data-exp.lua b/tex/context/base/mkiv/data-exp.lua
index 1ebdeb1e5..98c2f6839 100644
--- a/tex/context/base/mkiv/data-exp.lua
+++ b/tex/context/base/mkiv/data-exp.lua
@@ -198,7 +198,7 @@ local function expandedhome()
return usedhomedir
end
-local dohome = ((P("~")+P("$HOME")+P("%HOME%"))/expandedhome)^0
+local dohome = ((P("~") + P("$HOME") + P("%HOME%")) / expandedhome)^0
local cleanup = Cs(donegation * dohome * doslashes)
resolvers.cleanpath = function(str)
diff --git a/tex/context/base/mkiv/data-res.lua b/tex/context/base/mkiv/data-res.lua
index d8ad81656..8afc09b97 100644
--- a/tex/context/base/mkiv/data-res.lua
+++ b/tex/context/base/mkiv/data-res.lua
@@ -744,7 +744,7 @@ variable = function(name)
end
expansion = function(name)
- local expansions =instance.expansions
+ local expansions = instance.expansions
local name = name and lpegmatch(dollarstripper,name)
local result = name and expansions[name]
return result ~= nil and result or ""
diff --git a/tex/context/base/mkiv/lpdf-emb.lua b/tex/context/base/mkiv/lpdf-emb.lua
index 8fb211a2d..8db2ecaf1 100644
--- a/tex/context/base/mkiv/lpdf-emb.lua
+++ b/tex/context/base/mkiv/lpdf-emb.lua
@@ -51,8 +51,6 @@ local pdfreserveobject = lpdf.reserveobject
local pdfflushobject = lpdf.flushobject
local pdfflushstreamobject = lpdf.flushstreamobject
-local fontstreams = fonts.hashes.streams
-
local report_fonts = logs.reporter("backend","fonts")
local trace_fonts = false
diff --git a/tex/context/base/mkiv/luat-cnf.lua b/tex/context/base/mkiv/luat-cnf.lua
index a39106744..4939b05c3 100644
--- a/tex/context/base/mkiv/luat-cnf.lua
+++ b/tex/context/base/mkiv/luat-cnf.lua
@@ -31,7 +31,7 @@ texconfig.param_size = 25000
texconfig.save_size = 100000
texconfig.stack_size = 10000
texconfig.function_size = 32768
-texconfig.properties_size = 262144 -- after that, we're a hash
+texconfig.properties_size = 10000
texconfig.fix_mem_init = 750000
local stub = [[
@@ -145,7 +145,7 @@ function texconfig.init()
local getbytecode = lua.getbytecode
local callbytecode = lua.callbytecode or function(i)
local b = getbytecode(i)
- if b then
+ if type(b) == "function" then
b()
return true
else
@@ -160,7 +160,7 @@ function texconfig.init()
-- local b = callbytecode(i)
local e, b = pcall(callbytecode,i)
if not e then
- print("\nfatal error : unable to load bytecode, maybe wipe the cache first\n")
+ print(string.format("\nfatal error : unable to load bytecode register %%i, maybe wipe the cache first\n",i))
os.exit()
end
if b then
diff --git a/tex/context/base/mkiv/luat-cod.lua b/tex/context/base/mkiv/luat-cod.lua
index f38966443..51eab5a1d 100644
--- a/tex/context/base/mkiv/luat-cod.lua
+++ b/tex/context/base/mkiv/luat-cod.lua
@@ -31,12 +31,11 @@ texconfig.param_size = 25000
texconfig.save_size = 100000
texconfig.stack_size = 10000
texconfig.function_size = 32768
-texconfig.properties_size = 262144 -- nuts are actually kind of pointers (so we jump by size)
+texconfig.properties_size = 10000
texconfig.fix_mem_init = 750000
-- registering bytecode chunks
------ bytecode = lua.bytecode or { } -- we use functions
local bytedata = lua.bytedata or { }
local bytedone = lua.bytedone or { }
diff --git a/tex/context/base/mkiv/luat-sto.lua b/tex/context/base/mkiv/luat-sto.lua
index ce891765a..620de9a15 100644
--- a/tex/context/base/mkiv/luat-sto.lua
+++ b/tex/context/base/mkiv/luat-sto.lua
@@ -72,7 +72,7 @@ if environment.initex then
end
local dumped = serialize(original,target)
if trace_storage then
- report_storage('saving %a in slot %a, size %s',message,max,#dumped)
+ report_storage("saving %a in slot %a, size %s",message,max,#dumped)
end
-- we don't need tracing in such tables
dumped = concat({ definition, comment, dumped },"\n")
diff --git a/tex/context/base/mkiv/mlib-mat.lua b/tex/context/base/mkiv/mlib-mat.lua
index 4646a8979..6c82a4658 100644
--- a/tex/context/base/mkiv/mlib-mat.lua
+++ b/tex/context/base/mkiv/mlib-mat.lua
@@ -80,6 +80,10 @@ local m_y0 = m.y0 registerscript("m_y0", function() return
local m_y1 = m.y1 registerscript("m_y1", function() return m_y1 (scannumeric()) end)
local m_yn = m.yn registerscript("m_yn", function() return m_yn (scanpair ()) end)
+if not (c and c.sin) then
+ return
+end
+
local c_topair = c.topair
local c_new = c.new
diff --git a/tex/context/base/mkiv/mult-sys.mkiv b/tex/context/base/mkiv/mult-sys.mkiv
index 14619316e..8f8794df7 100644
--- a/tex/context/base/mkiv/mult-sys.mkiv
+++ b/tex/context/base/mkiv/mult-sys.mkiv
@@ -460,6 +460,10 @@
\definesystemconstant {xoffset}
\definesystemconstant {yoffset}
+\definesystemconstant {topfloat}
+\definesystemconstant {bottomfloat}
+\definesystemconstant {pagefloat}
+
%D As the name of their define command states, the next set of constants is used in
%D the message macro's.
diff --git a/tex/context/base/mkiv/node-fin.lua b/tex/context/base/mkiv/node-fin.lua
index 02c359174..a848cef69 100644
--- a/tex/context/base/mkiv/node-fin.lua
+++ b/tex/context/base/mkiv/node-fin.lua
@@ -12,10 +12,12 @@ if not modules then modules = { } end modules ['node-fin'] = {
-- leaders are also triggers ... see colo-ext for an example (negate a box)
local next, type, format = next, type, string.format
+local setmetatableindex = table.setmetatableindex
local attributes, nodes, node = attributes, nodes, node
local nuts = nodes.nuts
+local tonut = nodes.tonut
local getnext = nuts.getnext
local getid = nuts.getid
@@ -26,6 +28,7 @@ local getwidth = nuts.getwidth
local getwhd = nuts.getwhd
local getorientation = nuts.getorientation
local has_dimensions = nuts.has_dimensions
+local getbox = nuts.getbox
local setlist = nuts.setlist
local setleader = nuts.setleader
@@ -39,16 +42,9 @@ local nextnode = nuts.traversers.node
local nodecodes = nodes.nodecodes
local rulecodes = nodes.rulecodes
------ normalrule_code = rulecodes.normal
local boxrule_code = rulecodes.box
local imagerule_code = rulecodes.image
local emptyrule_code = rulecodes.empty
------ userrule_code = rulecodes.user
------ overrule_code = rulecodes.over
------ underrule_code = rulecodes.under
------ fractionrule_code = rulecodes.fraction
------ radicalrule_code = rulecodes.radical
------ outlinerule_code = rulecodes.outline
local glyph_code = nodecodes.glyph
local disc_code = nodecodes.disc
@@ -57,6 +53,9 @@ local rule_code = nodecodes.rule
local hlist_code = nodecodes.hlist
local vlist_code = nodecodes.vlist
+local texlists = tex.lists
+local texgetnest = tex.getnest
+
local states = attributes.states
local numbers = attributes.numbers
local a_trigger = attributes.private('trigger')
@@ -635,3 +634,100 @@ end
statistics.register("attribute processing time", function()
return statistics.elapsedseconds(attributes,"front- and backend")
end)
+
+-- -- --
+
+do
+
+ local cleaners = { }
+ local trace = true -- false
+
+ function attributes.cleanup()
+ if next(cleaners) then
+ local values = setmetatableindex("table")
+
+ if trace then
+ starttiming(values)
+ end
+
+ local function check(l)
+ for n, id in nextnode, l do
+ if id == hlist_code or id == vlist_code or id == glue_code then
+ local l = getlist(n)
+ if l then
+ check(l)
+ end
+ end
+ for a in next, cleaners do
+ local v = getattr(n,a)
+ if v then
+ -- values[a] = values[a] + 1
+ values[a][v] = true
+ end
+ end
+ end
+ end
+
+ local top = texgetnest("ptr")
+ for i=1,top do
+ local l = texgetnest(i)
+ if l then
+ check(tonut(l.head))
+ end
+ end
+
+ do local l
+ l = tonut(texlists.page_ins_head) if l then check(l) end
+ l = tonut(texlists.contrib_head) if l then check(l) end
+ l = tonut(texlists.page_discards_head) if l then check(l) end
+ l = tonut(texlists.split_discards_head) if l then check(l) end
+ l = tonut(texlists.page_head) if l then check(l) end
+ end
+
+ -- todo: traverseboxes
+
+ for i=0,65535 do
+ local b = getbox(i)
+ if b then
+ local l = getlist(b)
+ if l then
+ check(l)
+ end
+ end
+ end
+
+ for a, t in next, values do
+ cleaners[a](a,t)
+ end
+
+ if trace then
+ stoptiming(values)
+ local a = table.sortedkeys(values)
+ local t = statistics.elapsedtime(values)
+ local r = tex.getcount("realpageno")
+ if #a == 0 then
+ logs.report("attributes","cleaning up at page %i took %s seconds, no attributes",r,t)
+ else
+ logs.report("attributes","cleaning up at page %i took %s seconds, attributes: % t",r,t,a)
+ end
+ end
+ end
+ end
+
+ -- not yet used but when we do ... delay a call till we enable it (attr-ini.mkiv)
+
+ -- local function show(a,t) for k, v in next, t do print(a,k) end end
+ --
+ -- attributes.registercleaner( 1, show)
+ -- attributes.registercleaner( 2, show)
+
+ function attributes.registercleaner(a,f)
+ cleaners[a] = f
+ end
+
+ implement {
+ name = "cleanupattributes",
+ actions = attributes.cleanup,
+ }
+
+end
diff --git a/tex/context/base/mkiv/node-res.lua b/tex/context/base/mkiv/node-res.lua
index 4347fad47..a27f5c4e5 100644
--- a/tex/context/base/mkiv/node-res.lua
+++ b/tex/context/base/mkiv/node-res.lua
@@ -559,8 +559,7 @@ local function cleanup(nofboxes) -- todo
return nr, nl, nofboxes -- can be nil
end
-
-local function usage()
+local usage = CONTEXTLMTXMODE > 0 and node.inuse or function()
local t = { }
for n, tag in gmatch(status.node_mem_usage,"(%d+) ([a-z_]+)") do
t[tag] = tonumber(n) or 0
@@ -581,9 +580,13 @@ statistics.register("cleaned up reserved nodes", function()
end) -- \topofboxstack
statistics.register("node memory usage", function() -- comes after cleanup !
- local usage = status.node_mem_usage
- if usage ~= "" then
- return usage
+ local used = usage()
+ if next(used) then
+ local t, n = { }, 0
+ for k, v in table.sortedhash(used) do
+ n = n + 1 ; t[n] = format("%s %s",v,k)
+ end
+ return table.concat(t,", ")
end
end)
diff --git a/tex/context/base/mkiv/page-ffl.mkiv b/tex/context/base/mkiv/page-ffl.mkiv
index 306c19790..176c4a185 100644
--- a/tex/context/base/mkiv/page-ffl.mkiv
+++ b/tex/context/base/mkiv/page-ffl.mkiv
@@ -65,7 +65,7 @@
\c!inbetween={\blank[\v!big]},
%\c!style,
%\c!color,
- \c!page=\v!yes]
+ \c!page=\v!left]
\appendtoks
\ifx\currentfacingfloatparent\empty
@@ -74,8 +74,8 @@
\fi
\to \everydefinefacingfloat
-\newcount\c_strc_floats_saved
-\newcount\c_strc_floats_flushed
+\newcount\c_strc_floats_facing_saved
+\newcount\c_strc_floats_facing_flushed
\newbox\b_strc_floats_facing_l
\newbox\b_strc_floats_facing_r
@@ -83,12 +83,24 @@
\let\m_strc_floats_state\relax
\def\strc_floats_facing_flush
- {\ifnum\c_strc_floats_flushed<\c_strc_floats_saved
- \strc_floats_facing_flush_indeed % less tracing
+ {\ifnum\c_strc_floats_facing_flushed<\c_strc_floats_facing_saved
+ \ifodd\c_strc_floats_facing_flushed
+ \ifodd\realpageno
+ \strc_floats_facing_flush_indeed
+ \doifelsependingpagecontent\relax{\null\page}%
+ \else
+ \fi
+ \else
+ \ifodd\realpageno
+ \else
+ \strc_floats_facing_flush_indeed
+ \doifelsependingpagecontent\relax{\null\page}%
+ \fi
+ \fi
\fi}
\def\strc_floats_facing_flush_indeed
- {\global\advance\c_strc_floats_flushed\plusone
+ {\global\advance\c_strc_floats_facing_flushed\plusone
\floatingpenalty\zerocount
\insert\namedinsertionnumber\s!topfloat\bgroup
\forgetall
@@ -100,12 +112,12 @@
\prevdepth\maxdimen
\fi
\fi
- \directboxfromcache{\currentfacingfloat}{\number\c_strc_floats_flushed}%
+ \directboxfromcache{\currentfacingfloat}{\number\c_strc_floats_facing_flushed}%
\vkern\s_page_one_between_top_insert
\egroup
- \ifnum\c_strc_floats_saved=\c_strc_floats_flushed
- \global\c_strc_floats_saved \zerocount
- \global\c_strc_floats_flushed\zerocount
+ \ifnum\c_strc_floats_facing_saved=\c_strc_floats_facing_flushed
+ \global\c_strc_floats_facing_saved \zerocount
+ \global\c_strc_floats_facing_flushed\zerocount
\resetboxesincache{\currentfacingfloat}%
\fi}
@@ -145,34 +157,34 @@
\fi\fi}
\unexpanded\def\strc_floats_facing_handle
- {\strc_floats_facing_collect
- \doifnextbgroupelse
+ {\doifnextbgroupelse
\strc_floats_facing_handle_indeed
- \strc_floats_wrap_up}
+ \strc_floats_facing_wrap_up}
\unexpanded\def\strc_floats_facing_handle_indeed
{\dowithnextboxcontent
\strc_floats_facing_setup
- \strc_floats_facing_handle
+ {\strc_floats_facing_collect\strc_floats_facing_handle}
\vbox}
\unexpanded\def\startfacingfloat[#1]%
{\begingroup
% todo: \usepageparameter
- \edef\p_page{\facingfloatparameter\c!page}%
- \ifx\p_page\empty
- \page[\p_page]%
- \fi
+% \edef\p_page{\facingfloatparameter\c!page}%
+% \ifx\p_page\empty\else
+% \page[\p_page]%
+% \fi
%
\let\startcontent\bgroup
\let\stopcontent\egroup
\def\currentfacingfloat{#1}%
+ \let\m_strc_floats_state\v!left
\strc_floats_facing_handle}
\unexpanded\def\stopfacingfloat
{\endgroup}
-\unexpanded\def\strc_floats_wrap_up
+\unexpanded\def\strc_floats_facing_wrap_up
{\edef\p_spaceinbetween{\facingfloatparameter\c!spaceinbetween}%
\ifx\p_spaceinbetween\empty
\scratchdimen\zeropoint
@@ -208,13 +220,9 @@
\else
\setbox\scratchbox\vbox to \textheight{\box\scratchbox\vss}%
\fi
- \global\advance\c_strc_floats_saved\plusone
- \putboxincache{\currentfacingfloat}{\number\c_strc_floats_saved}\scratchbox
+ \global\advance\c_strc_floats_facing_saved\plusone
+ \putboxincache{\currentfacingfloat}{\number\c_strc_floats_facing_saved}\scratchbox
\endgroup
\fi}
-\appendtoks
- \strc_floats_facing_flush
-\to \everyafteroutput
-
\protect \endinput
diff --git a/tex/context/base/mkiv/page-flt.mkiv b/tex/context/base/mkiv/page-flt.mkiv
index e72ae314a..01ee8b689 100644
--- a/tex/context/base/mkiv/page-flt.mkiv
+++ b/tex/context/base/mkiv/page-flt.mkiv
@@ -20,10 +20,6 @@
\unprotect
-\ifdefined\s!topfloat \else \def\s!topfloat {topfloat} \fi
-\ifdefined\s!bottomfloat \else \def\s!bottomfloat{bottomfloat} \fi
-\ifdefined\s!pagefloat \else \def\s!pagefloat {pagefloat} \fi
-
\defineinsertion[\s!topfloat]
\defineinsertion[\s!bottomfloat]
\defineinsertion[\s!pagefloat]
diff --git a/tex/context/base/mkiv/page-ini.lua b/tex/context/base/mkiv/page-ini.lua
index 3d5534128..17f4c44da 100644
--- a/tex/context/base/mkiv/page-ini.lua
+++ b/tex/context/base/mkiv/page-ini.lua
@@ -16,6 +16,8 @@ local texgetcount = tex.getcount
local context = context
local ctx_doifelse = commands.doifelse
+local implement = interfaces.implement
+
local data = table.setmetatableindex("table")
local last = 0
local pages = structures.pages
@@ -177,19 +179,19 @@ luatex.registerpageactions(function()
end
end)
-interfaces.implement {
+implement {
name = "markpage",
arguments = "2 strings",
actions = pages.mark
}
-interfaces.implement {
+implement {
name = "doifelsemarkedpage",
arguments = "string",
actions = { marked, ctx_doifelse }
}
-interfaces.implement {
+implement {
name = "markedpages",
arguments = "string",
actions = function(name)
@@ -200,7 +202,7 @@ interfaces.implement {
end
}
-interfaces.implement {
+implement {
name = "startmarkpages",
arguments = "string",
actions = function(name)
@@ -208,7 +210,7 @@ interfaces.implement {
end
}
-interfaces.implement {
+implement {
name = "stopmarkpages",
arguments = "string",
actions = function(name)
@@ -217,3 +219,23 @@ interfaces.implement {
end
end
}
+
+local tonut = nodes.tonut
+local nextlist = nodes.nuts.traversers.list
+local texlists = tex.lists
+
+implement {
+ name = "doifelsependingpagecontent",
+ actions = function()
+ local h = texlists.contrib_head
+ -- local t = texlists.contrib_tail
+ local p = false
+ if h then
+ for n in nextlist, tonut(h) do
+ p = true
+ break
+ end
+ end
+ ctx_doifelse(p)
+ end,
+}
diff --git a/tex/context/base/mkiv/page-ini.mkiv b/tex/context/base/mkiv/page-ini.mkiv
index b1e3e7f8a..e56c3ac04 100644
--- a/tex/context/base/mkiv/page-ini.mkiv
+++ b/tex/context/base/mkiv/page-ini.mkiv
@@ -162,6 +162,7 @@
\strc_pagenumbers_increment_counters % should hook into an every
\page_adapts_synchronize
\page_otr_check_for_pending_inserts
+ \page_otr_command_flush_facing_floats
\page_floats_flush_page_floats % before postponed blocks
\page_spread_flush % defined later
\ifnum#3=\plusone
@@ -375,4 +376,8 @@
% \unexpanded\def\forcestrutdepth
% {\par\ifvmode\ifinner\else\doforcestrutdepth\fi\fi}
+% Also experimental:
+
+\unexpanded\def\doifelsependingpagecontent{\clf_doifelsependingpagecontent}
+
\protect \endinput
diff --git a/tex/context/base/mkiv/page-one.mkiv b/tex/context/base/mkiv/page-one.mkiv
index 31b4e22d9..85ed76da1 100644
--- a/tex/context/base/mkiv/page-one.mkiv
+++ b/tex/context/base/mkiv/page-one.mkiv
@@ -680,6 +680,9 @@
\endgroup
\fi}
+\unexpanded\def\page_one_command_flush_facing_floats
+ {\strc_floats_facing_flush}
+
\defineoutputroutine
[\s!singlecolumn]
[\s!page_otr_command_routine =\page_one_command_routine,
@@ -703,8 +706,9 @@
\s!page_otr_command_flush_saved_floats =\page_one_command_flush_saved_floats,
\s!page_otr_command_flush_all_floats =\page_one_command_flush_all_floats,
\s!page_otr_command_flush_margin_blocks =\page_one_command_flush_margin_blocks,
- \s!page_otr_command_test_column =\page_one_command_test_page
- ]
+ \s!page_otr_command_test_column =\page_one_command_test_page,
+ \s!page_otr_command_flush_facing_floats =\page_one_command_flush_facing_floats
+]
% \setupoutputroutine
% [\s!singlecolumn]
diff --git a/tex/context/base/mkiv/page-otr.mkvi b/tex/context/base/mkiv/page-otr.mkvi
index 909f5cd4d..6feb15a9d 100644
--- a/tex/context/base/mkiv/page-otr.mkvi
+++ b/tex/context/base/mkiv/page-otr.mkvi
@@ -285,6 +285,7 @@
\definesystemconstant{page_otr_command_flush_all_floats}
\definesystemconstant{page_otr_command_flush_margin_blocks}
\definesystemconstant{page_otr_command_test_column}
+\definesystemconstant{page_otr_command_flush_facing_floats}
\definesystemconstant{singlecolumn}
\definesystemconstant{multicolumn} % will move
@@ -313,7 +314,8 @@
\s!page_otr_command_flush_saved_floats,
\s!page_otr_command_flush_all_floats,
\s!page_otr_command_flush_margin_blocks,
- \s!page_otr_command_test_column]
+ \s!page_otr_command_test_column,
+ \s!page_otr_command_flush_facing_floats]
\appendtoks
\setupoutputroutine[\s!singlecolumn]%
diff --git a/tex/context/base/mkiv/page-run.lua b/tex/context/base/mkiv/page-run.lua
index 53331fc0e..cb8cf0311 100644
--- a/tex/context/base/mkiv/page-run.lua
+++ b/tex/context/base/mkiv/page-run.lua
@@ -200,8 +200,8 @@ function commands.showusage()
report(" string pointer : %s of %s", status.str_ptr, status.max_strings + status.init_str_ptr)
report(" pool size : %s", status.pool_size)
report("")
- report(" node memory usage : %s of %s", status.node_mem_usage, status.var_mem_max)
- report(" fixex memory end : %s of %s", status.fix_mem_end, status.fix_mem_max)
+ report(" node memory usage : %s of %s", status.var_used, status.var_mem_max)
+ report(" token memory usage : %s of %s", status.dyn_used, status.fix_mem_max)
report("")
report(" cs count : %s of %s", status.cs_count, status.hash_size + status.hash_extra)
report("")
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index f0a77a5a9..b7c571975 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
Binary files differ
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index eeda6094f..453fc01db 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
Binary files differ
diff --git a/tex/context/base/mkiv/strc-itm.lua b/tex/context/base/mkiv/strc-itm.lua
index adec591c1..e2fbdc960 100644
--- a/tex/context/base/mkiv/strc-itm.lua
+++ b/tex/context/base/mkiv/strc-itm.lua
@@ -36,8 +36,8 @@ local function analyzeitemgroup(name,level)
local stamp = f_stamp(name,n)
local n = getvariable(stamp,level,1,0)
local w = getvariable(stamp,level,2,0)
- texsetcount("local","c_strc_itemgroups_max_items",n)
- texsetdimen("local","d_strc_itemgroups_max_width",w)
+ texsetcount("c_strc_itemgroups_max_items",n)
+ texsetdimen("d_strc_itemgroups_max_width",w)
end
local function registeritemgroup(name,level,nofitems,maxwidth)
diff --git a/tex/context/base/mkiv/syst-lua.mkiv b/tex/context/base/mkiv/syst-lua.mkiv
index 0e2769d04..d275acd7d 100644
--- a/tex/context/base/mkiv/syst-lua.mkiv
+++ b/tex/context/base/mkiv/syst-lua.mkiv
@@ -118,4 +118,6 @@
%D The \MKIV\ version is a bit more limited than the \LMTX\ version because it only
%D supports the standard math functions.
+% maybe: \let\texexpr\expression
+
\protect \endinput
diff --git a/tex/context/base/mkiv/syst-lua.mkxl b/tex/context/base/mkiv/syst-lua.mkxl
index cceb4f160..cf20c3341 100644
--- a/tex/context/base/mkiv/syst-lua.mkxl
+++ b/tex/context/base/mkiv/syst-lua.mkxl
@@ -95,4 +95,6 @@
%D The \LMTX\ version supports te \type {xmath} and \type {xcomplex} functions and
%D therefore one can have expressions that don't work in \MKIV.
+% maybe: \let\texexpr\expression
+
\protect \endinput
diff --git a/tex/context/base/mkiv/typo-chr.lua b/tex/context/base/mkiv/typo-chr.lua
index 966df6098..7a0f1fd75 100644
--- a/tex/context/base/mkiv/typo-chr.lua
+++ b/tex/context/base/mkiv/typo-chr.lua
@@ -188,6 +188,14 @@ local function pickup(head,tail,str)
end
end
+local function found(str)
+ local list = texgetnest()
+ if list then
+ local tail = list.tail
+ return tail and tail[a_marked] == marked[str]
+ end
+end
+
local actions = {
remove = function(specification)
local list = texgetnest()
@@ -254,6 +262,12 @@ interfaces.implement {
arguments = "string",
}
+interfaces.implement {
+ name = "doifelsemarkedcontent",
+ actions = function(str) ctx_doifelse(found(str)) end,
+ arguments = "string",
+}
+
-- We just put these here.
interfaces.implement {
diff --git a/tex/context/base/mkiv/typo-chr.mkiv b/tex/context/base/mkiv/typo-chr.mkiv
index 7da783ceb..a648d4588 100644
--- a/tex/context/base/mkiv/typo-chr.mkiv
+++ b/tex/context/base/mkiv/typo-chr.mkiv
@@ -78,6 +78,9 @@
\def\typo_marked_remove[#1]%
{\clf_pickupmarkedcontent action{remove}mark{#1}\relax}
+\unexpanded\def\doifelsemarkedcontent#1%
+ {\clf_doifelsemarkedcontent{#1}}
+
%D A few helpers (put here for convenience):
%D
%D \starttyping
diff --git a/tex/context/modules/common/s-abbreviations-logos.tex b/tex/context/modules/common/s-abbreviations-logos.tex
index cafc296b5..9f1d5599e 100644
--- a/tex/context/modules/common/s-abbreviations-logos.tex
+++ b/tex/context/modules/common/s-abbreviations-logos.tex
@@ -57,7 +57,7 @@
\logo [CDROM] {cdrom}
\logo [CID] {cid}
\logo [CJK] {cjk}
-\logo [CLD] {cld}
+\logo [CLANG] {clang}
\logo [CLD] {cld}
\logo [CMAKE] {cmake}
\logo [CMR] {cmr}
@@ -204,6 +204,7 @@
\logo [MPTOPDF] {mptopdf}
\logo [MSDOS] {msdos}
\logo [MSEXCEL] {MS~Excel}
+\logo [MSVC] {MSVC}
\logo [MSWINDOWS] {MS~Windows}
\logo [MSWORD] {MS~Word}
\logo [MTXRUN] {mtxrun}
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 2882584ec..ce2794cf3 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
-- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
-- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date : 01/15/20 19:09:43
+-- merge date : 01/26/20 18:34:44
do -- begin closure to overcome local limits and interference