From 6db2cd924d26ade933812f90701343f06c8653f2 Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Fri, 30 Jul 2021 01:27:40 +0200 Subject: 2021-07-30 00:43:00 --- .../scite/context/lexers/scite-context-lexer.lua | 20 +- .../data/scite/context/scite-context.properties | 2 +- context/data/textadept/context/lexers/lexer.lua | 20 +- .../context/lexers/scite-context-lexer.lua | 20 +- .../documents/general/manuals/charts-mkiv.pdf | Bin 144092 -> 123926 bytes .../documents/general/manuals/followingup.pdf | Bin 439529 -> 404791 bytes doc/context/documents/general/manuals/notnow.pdf | Bin 103814 -> 61024 bytes doc/context/documents/general/manuals/ontarget.pdf | Bin 0 -> 29193 bytes .../documents/general/manuals/primitives.pdf | Bin 143967 -> 145244 bytes .../documents/general/manuals/templates-mkiv.pdf | Bin 92572 -> 44094 bytes .../documents/general/manuals/xtables-mkiv.pdf | Bin 157802 -> 97369 bytes .../sources/general/manuals/charts/charts-mkiv.tex | 2 + .../general/manuals/evenmore/evenmore-bitwise.tex | 194 +++++++++ .../general/manuals/evenmore/evenmore-contents.tex | 2 + .../manuals/evenmore/evenmore-expansion.tex | 2 +- .../manuals/evenmore/evenmore-expressions.tex | 336 +++++++++++++++ .../general/manuals/evenmore/evenmore-fonts.tex | 2 +- .../general/manuals/evenmore/evenmore-formats.tex | 362 ++++++++++++++++ .../manuals/evenmore/evenmore-hyphenation.tex | 426 +++++++++++++++++++ .../general/manuals/evenmore/evenmore-inserts.tex | 198 +++++++++ .../manuals/evenmore/evenmore-introduction.tex | 2 +- .../general/manuals/evenmore/evenmore-keywords.tex | 2 +- .../manuals/evenmore/evenmore-libraries.tex | 2 +- .../manuals/evenmore/evenmore-normalization.tex | 6 +- .../general/manuals/evenmore/evenmore-numbers.tex | 2 +- .../manuals/evenmore/evenmore-offloading.tex | 127 ++++++ .../manuals/evenmore/evenmore-paragraphs.tex | 397 ++++++++++++++++++ .../manuals/evenmore/evenmore-parameters.tex | 2 +- .../general/manuals/evenmore/evenmore-parsing.tex | 2 +- .../general/manuals/evenmore/evenmore-pi.tex | 2 +- .../general/manuals/evenmore/evenmore-style.tex | 2 + .../general/manuals/evenmore/evenmore-threesix.tex | 2 +- .../manuals/evenmore/evenmore-titlepage.tex | 2 + .../general/manuals/evenmore/evenmore-tokens.tex | 2 +- .../general/manuals/evenmore/evenmore-whattex.tex | 2 +- .../sources/general/manuals/evenmore/evenmore.tex | 22 +- .../followingup/followingup-expressions.tex | 333 --------------- .../manuals/followingup/followingup-formats.tex | 359 ---------------- .../general/manuals/followingup/followingup.tex | 2 - .../sources/general/manuals/graphics/graphics.tex | 2 +- .../general/manuals/lowlevel/lowlevel-boxes.tex | 14 + .../manuals/lowlevel/lowlevel-paragraphs.tex | 53 ++- .../sources/general/manuals/lua/lua-mkiv.tex | 2 +- .../manuals/luametatex/luametatex-enhancements.tex | 9 + .../manuals/luatex/luatex-modifications.tex | 10 +- .../manuals/metafun/metafun-backgrounds.tex | 2 +- .../general/manuals/metafun/metafun-backpage.tex | 2 +- .../general/manuals/metafun/metafun-basics.tex | 2 +- .../manuals/metafun/metafun-colofon-paper.tex | 2 +- .../manuals/metafun/metafun-colofon-screen.tex | 2 +- .../general/manuals/metafun/metafun-colofon.tex | 2 +- .../general/manuals/metafun/metafun-contents.tex | 2 +- .../manuals/metafun/metafun-conventions.tex | 2 +- .../general/manuals/metafun/metafun-debugging.tex | 2 +- .../general/manuals/metafun/metafun-document.tex | 2 +- .../general/manuals/metafun/metafun-effects.tex | 2 +- .../general/manuals/metafun/metafun-embedding.tex | 2 +- .../manuals/metafun/metafun-environment-layout.tex | 2 + .../metafun/metafun-environment-samples.tex | 2 + .../manuals/metafun/metafun-environment-screen.tex | 2 +- .../manuals/metafun/metafun-environment.tex | 2 +- .../general/manuals/metafun/metafun-examples.tex | 2 +- .../general/manuals/metafun/metafun-functions.tex | 2 +- .../general/manuals/metafun/metafun-gadgets.tex | 2 +- .../general/manuals/metafun/metafun-graphics.tex | 2 +- .../general/manuals/metafun/metafun-index.tex | 2 +- .../manuals/metafun/metafun-introduction.tex | 2 +- .../general/manuals/metafun/metafun-layout.tex | 2 +- .../general/manuals/metafun/metafun-lua.tex | 2 +- .../general/manuals/metafun/metafun-macros.tex | 2 +- .../manuals/metafun/metafun-positioning.tex | 2 +- .../general/manuals/metafun/metafun-reference.tex | 2 +- .../general/manuals/metafun/metafun-sneaky.tex | 2 +- .../general/manuals/metafun/metafun-styles.tex | 2 +- .../general/manuals/metafun/metafun-syntax.tex | 2 +- .../general/manuals/metafun/metafun-text-lmtx.tex | 2 +- .../general/manuals/metafun/metafun-text.tex | 2 +- .../manuals/metafun/metafun-titlepage-paper.tex | 2 +- .../sources/general/manuals/metafun/metafun.tex | 2 +- .../sources/general/manuals/metafun/mfun-700.tex | 2 +- .../sources/general/manuals/metafun/mfun-771.tex | 2 +- .../sources/general/manuals/metafun/mfun-772.tex | 2 +- .../sources/general/manuals/metafun/mfun-773.tex | 2 +- .../sources/general/manuals/metafun/mfun-774.tex | 2 +- .../sources/general/manuals/metafun/mfun-775.tex | 2 +- .../sources/general/manuals/metafun/mfun-776.tex | 2 +- .../sources/general/manuals/metafun/mfun-800.tex | 2 +- .../sources/general/manuals/metafun/mfun-900.tex | 2 +- .../sources/general/manuals/metafun/mfun-901.tex | 2 +- .../sources/general/manuals/metafun/mfun-902.tex | 2 +- .../manuals/notnow/notnow-columns-and-notes.tex | 2 + .../general/manuals/notnow/notnow-sidefloats.tex | 2 + .../sources/general/manuals/notnow/notnow.tex | 42 +- .../general/manuals/ontarget/ontarget-contents.tex | 13 + .../manuals/ontarget/ontarget-introduction.tex | 104 +++++ .../general/manuals/ontarget/ontarget-style.tex | 68 +++ .../manuals/ontarget/ontarget-titlepage.tex | 73 ++++ .../sources/general/manuals/ontarget/ontarget.tex | 19 + .../general/manuals/primitives/primitives.tex | 2 +- .../general/manuals/templates/templates-mkiv.tex | 6 +- .../sources/general/manuals/units/units-mkiv.tex | 2 +- .../manuals/workflows/workflows-contents.tex | 2 +- .../manuals/workflows/workflows-graphics.tex | 2 +- .../manuals/workflows/workflows-injectors.tex | 2 +- .../manuals/workflows/workflows-introduction.tex | 2 +- .../general/manuals/workflows/workflows-mkiv.tex | 2 + .../manuals/workflows/workflows-parallel.tex | 2 +- .../manuals/workflows/workflows-resources.tex | 2 +- .../manuals/workflows/workflows-running.tex | 2 +- .../general/manuals/workflows/workflows-setups.tex | 2 +- .../general/manuals/workflows/workflows-style.tex | 2 + .../manuals/workflows/workflows-suspects.tex | 2 +- .../manuals/workflows/workflows-synctex.tex | 2 +- .../manuals/workflows/workflows-titlepage.tex | 2 +- .../general/manuals/workflows/workflows-xml.tex | 2 +- .../general/manuals/xtables/xtables-mkiv.tex | 9 +- tex/context/base/mkii/cont-new.mkii | 2 +- tex/context/base/mkii/context.mkii | 2 +- tex/context/base/mkii/mult-en.mkii | 5 + tex/context/base/mkii/mult-it.mkii | 1 + tex/context/base/mkii/mult-nl.mkii | 4 + tex/context/base/mkiv/cont-new.mkiv | 2 +- tex/context/base/mkiv/context.mkiv | 2 +- tex/context/base/mkiv/file-res.lua | 2 +- tex/context/base/mkiv/grph-fil.lua | 1 + tex/context/base/mkiv/lpdf-ano.lua | 20 +- tex/context/base/mkiv/lpdf-mov.lua | 42 +- tex/context/base/mkiv/mult-def.lua | 3 + tex/context/base/mkiv/scrn-wid.lua | 16 +- tex/context/base/mkiv/status-files.pdf | Bin 23606 -> 24854 bytes tex/context/base/mkiv/status-lua.pdf | Bin 248677 -> 250063 bytes tex/context/base/mkxl/buff-ver.mkxl | 42 +- tex/context/base/mkxl/colo-ini.mkxl | 7 + tex/context/base/mkxl/cont-new.mkxl | 2 +- tex/context/base/mkxl/context.mkxl | 2 +- tex/context/base/mkxl/lang-ini.mkxl | 8 +- tex/context/base/mkxl/lang-rep.lmt | 467 +++++++++++++++++++++ tex/context/base/mkxl/lang-rep.mkxl | 2 +- tex/context/base/mkxl/lpdf-ano.lmt | 39 +- tex/context/base/mkxl/lpdf-emb.lmt | 24 +- tex/context/base/mkxl/lpdf-fmt.lmt | 7 + tex/context/base/mkxl/lpdf-lmt.lmt | 48 ++- tex/context/base/mkxl/lpdf-mov.lmt | 42 +- tex/context/base/mkxl/lpdf-tag.lmt | 37 +- tex/context/base/mkxl/node-ref.lmt | 2 +- tex/context/base/mkxl/scrn-wid.mklx | 60 +-- tex/context/base/mkxl/syst-ini.mkxl | 2 +- tex/context/base/mkxl/typo-duc.lmt | 2 +- tex/context/interface/mkii/keys-en.xml | 5 + tex/context/interface/mkii/keys-it.xml | 1 + tex/context/interface/mkii/keys-nl.xml | 4 + tex/generic/context/luatex/luatex-fonts-merged.lua | 2 +- 152 files changed, 3327 insertions(+), 986 deletions(-) create mode 100644 doc/context/documents/general/manuals/ontarget.pdf create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-bitwise.tex create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-expressions.tex create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-formats.tex create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-hyphenation.tex create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-inserts.tex create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-offloading.tex create mode 100644 doc/context/sources/general/manuals/evenmore/evenmore-paragraphs.tex delete mode 100644 doc/context/sources/general/manuals/followingup/followingup-expressions.tex delete mode 100644 doc/context/sources/general/manuals/followingup/followingup-formats.tex create mode 100644 doc/context/sources/general/manuals/ontarget/ontarget-contents.tex create mode 100644 doc/context/sources/general/manuals/ontarget/ontarget-introduction.tex create mode 100644 doc/context/sources/general/manuals/ontarget/ontarget-style.tex create mode 100644 doc/context/sources/general/manuals/ontarget/ontarget-titlepage.tex create mode 100644 doc/context/sources/general/manuals/ontarget/ontarget.tex create mode 100644 tex/context/base/mkxl/lang-rep.lmt diff --git a/context/data/scite/context/lexers/scite-context-lexer.lua b/context/data/scite/context/lexers/scite-context-lexer.lua index 8451abc85..289697b72 100644 --- a/context/data/scite/context/lexers/scite-context-lexer.lua +++ b/context/data/scite/context/lexers/scite-context-lexer.lua @@ -12,6 +12,24 @@ local info = { -- before version 10 but I can't figure out what else to do. It looks like there -- is some loading of lexer.lua but I can't see where. +-- For a while it looked like we're stuck with scite 3 because there would be no +-- update of scintillua for the newer versions (c++ changes) but now it looks that +-- there will be updates (2021). There is a dll for scite >= 5 but it doesn't +-- work (yet). In version 5.20+ the scintillua dll makes scite crash (alsl when I +-- use the recommended import). In an early 5.02 loading the (shipped) lpeg lexer +-- does nothing at all. There have been changes in the lua interface too but I need +-- to compare the old and new lib. For now I gave up and got back to version 3+. It +-- would be nice if error messages would go to the log pane so that wget an idea +-- what happens. After all the code involved (below) is not that much and not that +-- complex either. +-- +-- Actually, scite 5.22 also crashed when a program was launched so better wait +-- for a while. (In the worst case, when it all stops working, we need to migrate +-- to visual code, which is out backup/fallback plan.) I didn't test if the latest +-- textadept still works with our lexer variant. In the meantime that editor has +-- grown to some 30 MB so it is no longer a lightweight option (scite with scintilla +-- is still quite small). + if lpeg.setmaxstack then lpeg.setmaxstack(1000) end local log = false @@ -71,7 +89,7 @@ local inspect = false -- can save some 15% (maybe easier on scintilla) -- big deal as we already had that in place (ConTeXt used lpeg from the day it -- showed up so we have several lexing options there too). -- --- Keep in mind that in ConTeXt (typesetting) lexing can follow several approached: +-- Keep in mind that in ConTeXt (typesetting) lexing can follow several approaches: -- line based (which is handy for verbatim mode), syntax mode (which is nice for -- tutorials), and tolerant mode (so that one can also show bad examples or errors). -- These demands can clash. diff --git a/context/data/scite/context/scite-context.properties b/context/data/scite/context/scite-context.properties index 1d61727d6..4e8f69334 100644 --- a/context/data/scite/context/scite-context.properties +++ b/context/data/scite/context/scite-context.properties @@ -476,7 +476,7 @@ style.*.33=$(font.base),back:$(colors.linepanel) style.errorlist.32=$(font.errorfont),back:$(colors.logpanel) -selection.back=#000000 +selection.back=#9F9F9F # Editor: fast switching between files diff --git a/context/data/textadept/context/lexers/lexer.lua b/context/data/textadept/context/lexers/lexer.lua index 8451abc85..289697b72 100644 --- a/context/data/textadept/context/lexers/lexer.lua +++ b/context/data/textadept/context/lexers/lexer.lua @@ -12,6 +12,24 @@ local info = { -- before version 10 but I can't figure out what else to do. It looks like there -- is some loading of lexer.lua but I can't see where. +-- For a while it looked like we're stuck with scite 3 because there would be no +-- update of scintillua for the newer versions (c++ changes) but now it looks that +-- there will be updates (2021). There is a dll for scite >= 5 but it doesn't +-- work (yet). In version 5.20+ the scintillua dll makes scite crash (alsl when I +-- use the recommended import). In an early 5.02 loading the (shipped) lpeg lexer +-- does nothing at all. There have been changes in the lua interface too but I need +-- to compare the old and new lib. For now I gave up and got back to version 3+. It +-- would be nice if error messages would go to the log pane so that wget an idea +-- what happens. After all the code involved (below) is not that much and not that +-- complex either. +-- +-- Actually, scite 5.22 also crashed when a program was launched so better wait +-- for a while. (In the worst case, when it all stops working, we need to migrate +-- to visual code, which is out backup/fallback plan.) I didn't test if the latest +-- textadept still works with our lexer variant. In the meantime that editor has +-- grown to some 30 MB so it is no longer a lightweight option (scite with scintilla +-- is still quite small). + if lpeg.setmaxstack then lpeg.setmaxstack(1000) end local log = false @@ -71,7 +89,7 @@ local inspect = false -- can save some 15% (maybe easier on scintilla) -- big deal as we already had that in place (ConTeXt used lpeg from the day it -- showed up so we have several lexing options there too). -- --- Keep in mind that in ConTeXt (typesetting) lexing can follow several approached: +-- Keep in mind that in ConTeXt (typesetting) lexing can follow several approaches: -- line based (which is handy for verbatim mode), syntax mode (which is nice for -- tutorials), and tolerant mode (so that one can also show bad examples or errors). -- These demands can clash. diff --git a/context/data/textadept/context/lexers/scite-context-lexer.lua b/context/data/textadept/context/lexers/scite-context-lexer.lua index 8451abc85..289697b72 100644 --- a/context/data/textadept/context/lexers/scite-context-lexer.lua +++ b/context/data/textadept/context/lexers/scite-context-lexer.lua @@ -12,6 +12,24 @@ local info = { -- before version 10 but I can't figure out what else to do. It looks like there -- is some loading of lexer.lua but I can't see where. +-- For a while it looked like we're stuck with scite 3 because there would be no +-- update of scintillua for the newer versions (c++ changes) but now it looks that +-- there will be updates (2021). There is a dll for scite >= 5 but it doesn't +-- work (yet). In version 5.20+ the scintillua dll makes scite crash (alsl when I +-- use the recommended import). In an early 5.02 loading the (shipped) lpeg lexer +-- does nothing at all. There have been changes in the lua interface too but I need +-- to compare the old and new lib. For now I gave up and got back to version 3+. It +-- would be nice if error messages would go to the log pane so that wget an idea +-- what happens. After all the code involved (below) is not that much and not that +-- complex either. +-- +-- Actually, scite 5.22 also crashed when a program was launched so better wait +-- for a while. (In the worst case, when it all stops working, we need to migrate +-- to visual code, which is out backup/fallback plan.) I didn't test if the latest +-- textadept still works with our lexer variant. In the meantime that editor has +-- grown to some 30 MB so it is no longer a lightweight option (scite with scintilla +-- is still quite small). + if lpeg.setmaxstack then lpeg.setmaxstack(1000) end local log = false @@ -71,7 +89,7 @@ local inspect = false -- can save some 15% (maybe easier on scintilla) -- big deal as we already had that in place (ConTeXt used lpeg from the day it -- showed up so we have several lexing options there too). -- --- Keep in mind that in ConTeXt (typesetting) lexing can follow several approached: +-- Keep in mind that in ConTeXt (typesetting) lexing can follow several approaches: -- line based (which is handy for verbatim mode), syntax mode (which is nice for -- tutorials), and tolerant mode (so that one can also show bad examples or errors). -- These demands can clash. diff --git a/doc/context/documents/general/manuals/charts-mkiv.pdf b/doc/context/documents/general/manuals/charts-mkiv.pdf index 1a0289aa5..775cdab5d 100644 Binary files a/doc/context/documents/general/manuals/charts-mkiv.pdf and b/doc/context/documents/general/manuals/charts-mkiv.pdf differ diff --git a/doc/context/documents/general/manuals/followingup.pdf b/doc/context/documents/general/manuals/followingup.pdf index d7812d563..4d55b772d 100644 Binary files a/doc/context/documents/general/manuals/followingup.pdf and b/doc/context/documents/general/manuals/followingup.pdf differ diff --git a/doc/context/documents/general/manuals/notnow.pdf b/doc/context/documents/general/manuals/notnow.pdf index e597a48bd..f809c093a 100644 Binary files a/doc/context/documents/general/manuals/notnow.pdf and b/doc/context/documents/general/manuals/notnow.pdf differ diff --git a/doc/context/documents/general/manuals/ontarget.pdf b/doc/context/documents/general/manuals/ontarget.pdf new file mode 100644 index 000000000..c4d0077e4 Binary files /dev/null and b/doc/context/documents/general/manuals/ontarget.pdf differ diff --git a/doc/context/documents/general/manuals/primitives.pdf b/doc/context/documents/general/manuals/primitives.pdf index d42e72970..00ac18528 100644 Binary files a/doc/context/documents/general/manuals/primitives.pdf and b/doc/context/documents/general/manuals/primitives.pdf differ diff --git a/doc/context/documents/general/manuals/templates-mkiv.pdf b/doc/context/documents/general/manuals/templates-mkiv.pdf index 47a0d70a6..15aa4f900 100644 Binary files a/doc/context/documents/general/manuals/templates-mkiv.pdf and b/doc/context/documents/general/manuals/templates-mkiv.pdf differ diff --git a/doc/context/documents/general/manuals/xtables-mkiv.pdf b/doc/context/documents/general/manuals/xtables-mkiv.pdf index 255a730b8..d877a44fd 100644 Binary files a/doc/context/documents/general/manuals/xtables-mkiv.pdf and b/doc/context/documents/general/manuals/xtables-mkiv.pdf differ diff --git a/doc/context/sources/general/manuals/charts/charts-mkiv.tex b/doc/context/sources/general/manuals/charts/charts-mkiv.tex index 2081b872b..e3d05d7cb 100644 --- a/doc/context/sources/general/manuals/charts/charts-mkiv.tex +++ b/doc/context/sources/general/manuals/charts/charts-mkiv.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/charts + % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team % license : Creative Commons Attribution ShareAlike 4.0 International diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-bitwise.tex b/doc/context/sources/general/manuals/evenmore/evenmore-bitwise.tex new file mode 100644 index 000000000..030229413 --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-bitwise.tex @@ -0,0 +1,194 @@ +% language=us runpath=texruns:manuals/evenmore + +\environment evenmore-style + +\startcomponent evenmore-bitwise + +\startchapter[title={Bitwise operations}] + +Occasionally we use a bit set to pass (or store) options and one way of doing +that is to add up some constants. However, what you would like to do is to set a +specific bit. Of course one can write macros for that but performance wise one +can wonder if there are other ways. One solution is to extend the engine but that +has its own pitfalls. For instance, I played with additions to \type {\numexpr} +and although they worked okay and brought now performance degradation, I decided +to remove that experiment. One problem is that we have no real 32 bit cardinals +(unsigned integers) in \TEX\ and the engine makes sure that we never exceed the +minima and maxima. Another problem is that in expressions we then need either +verbose \type {and}, \type {or}, \type {xor}, \type {not} and other verbose +operators, if only because the usual symbols can have catcodes that are +unsuitable. + +So in the end I decided to come up with a set of primitive like commands that +do the job. It is no problem to have a set of dedicated verbose commands and we +can extend the repertoire if needed. So this is what we have now: + +\starttabulate[||i2c||] +\NC command \NC operator equivalent \NC optional \NC \NR +\NC \type {\bitwiseset a} \NC \type {a} \NC \NC \NR +\NC \type {\bitwisenot a} \NC \type {~a} \NC \NC \NR +\NC \type {\bitwisenil a b} \NC \type {a & (~ b)} \NC \type {with} \NC \NR +\NC \type {\bitwiseand a b} \NC \type {a & b} \NC \type {with} \NC \NR +\NC \type {\bitwiseor a b} \NC \type {a | b} \NC \type {with} \NC \NR +\NC \type {\bitwisexor a b} \NC \type {a ~ b} \NC \type {with} \NC \NR +\NC \type {\bitwiseshift a b} \NC \type {a >> b} \NC \type {by} \NC \NR +\NC \type {\bitwiseshift a -b} \NC \type {a << b} \NC \type {by} \NC \NR +\NC \type {\ifbitwiseand a b} \NC \type {(a & b) ~= 0} \NC \NC \NR +\stoptabulate + +Here a some examples: + +\startbuffer +\scratchcounter = \bitwiseand "01 "02 \uchexnumbers{\scratchcounter} \quad +\scratchcounter = \bitwiseand "01 with "02 \uchexnumbers{\scratchcounter} \quad +\scratchcounter = \bitwiseand "03 "02 \uchexnumbers{\scratchcounter} \qquad +\scratchcounter = \bitwiseor "01 "02 \uchexnumbers{\scratchcounter} \quad +\scratchcounter = \bitwiseor "01 with "02 \uchexnumbers{\scratchcounter} \quad +\scratchcounter = \bitwiseor "03 "02 \uchexnumbers{\scratchcounter} \qquad +\scratchcounter = \bitwisexor "01 "02 \uchexnumbers{\scratchcounter} \quad +\scratchcounter = \bitwisexor "01 with "02 \uchexnumbers{\scratchcounter} \quad +\scratchcounter = \bitwisexor "03 "02 \uchexnumbers{\scratchcounter} +\stopbuffer + +\typebuffer + +This gives the nine values: + +{\tt\getbuffer} + +Because they are numeric operations you can chain them, as in: + +\starttyping +\scratchcounter = \bitwisenil \bitwisenil "0F "02 "01 \relax +\scratchcounter = \bitwisenil \bitwisenil "0F with "02 with "01 \relax +\stoptyping + +We try as good as possible to support all bits in the range from zero upto \type +{0xFFFFFFFF}; + +\startbuffer +\scratchcounter \bitwiseset "FFFFFFFF + +\ifbitwiseand \scratchcounter "80000000 YES \else NOP \fi +\ifbitwiseand \scratchcounter "F0000000 YES \else NOP \fi +\ifbitwiseand \scratchcounter "10000000 YES \else NOP \fi + +\scratchcounter \bitwisenot \scratchcounter + +\ifbitwiseand \scratchcounter "80000000 YES \else NOP \fi +\ifbitwiseand \scratchcounter "F0000000 YES \else NOP \fi +\ifbitwiseand \scratchcounter "10000000 YES \else NOP \fi +\stopbuffer + +\typebuffer + +and we get: + +\startpacked \tt \getbuffer \stoppacked + +Of course you can just use normal counters and \TEX\ integers but using the bit +commands have the advantage that they do some checking and can do real \type {or} +etc operations. Here is some food for thought: + +\startbuffer +\scratchcounter \bitwiseand "01 "02 +\scratchcounter \numexpr "01+"02\relax + +\ifcase \bitwiseand \scratchcounterone \plusone \else \fi +\ifbitwiseand \scratchcounterone \plusone \else \fi +\ifnum \scratchcounterone=\plusone \else \fi +\stopbuffer + +\typebuffer + +and we get: + +\startpacked \tt \getbuffer \stoppacked + +You can also go real binary, but we only provide a combined setter|/|getter for +that, but you can mix that one with the other commands: + +\startbuffer +\scratchcounterone = \bitwise 1101 +\scratchcountertwo = \bitwise 11011101110111011101110111011101 +\scratchcounterthree = \bitwiseor \bitwise 0001 \bitwise 1100 + +{0x\uchexnumber{\scratchcounterone} \bitwise\scratchcounterone }\par +{0x\uchexnumber{\scratchcountertwo} \bitwise\scratchcountertwo }\par +{0x\uchexnumber{\scratchcounterthree} \bitwise\scratchcounterthree}\par +\stopbuffer + +\typebuffer + +We get bits back: + +\startpacked \tt \getbuffer \stoppacked + +The above commands are typical for the things we can do with \LUAMETATEX\ and +\LMTX\ and are unlikely to become available in \MKIV. + +There is a special command for (re)setting a bit in a register: + +\startbuffer + \scratchcounter 0 +\bitwiseflip \scratchcounter 1 [\the\scratchcounter] +\bitwiseflip \scratchcounter 4 [\the\scratchcounter] +\bitwiseflip \scratchcounter 8 [\the\scratchcounter] +\bitwiseflip \scratchcounter -5 [\the\scratchcounter] +\stopbuffer + +\typebuffer + +This gives: \inlinebuffer. Of course a global assignment works too: + +\startbuffer + \global \globalscratchcounter 0 +{\global \bitwiseflip \globalscratchcounter 2 } [\the\globalscratchcounter] +{\global \bitwiseflip \globalscratchcounter 4 } [\the\globalscratchcounter] +{\global \bitwiseflip \globalscratchcounter 8 } [\the\globalscratchcounter] +{\global \bitwiseflip \globalscratchcounter -6 } [\the\globalscratchcounter] +\stopbuffer + +\typebuffer + +Here we get: \inlinebuffer. A side effect of it being an number makes that +this is also valid: + +\starttyping +\scratchcounterone\bitwiseflip \scratchcountertwo -16 +\stoptyping + +\stopchapter + +\stopcomponent + +% (\scratchcounterone \bitwiseset "F \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FF \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FFF \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FFFF \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FFFFF \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FFFFFF \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FFFFFFF \uchexnumber{\scratchcounterone})\par +% (\scratchcounterone \bitwiseset "FFFFFFFF \uchexnumber{\scratchcounterone})\par +% +% (\scratchcounterone \bitwiseset "0000FFFF +% \scratchcounterone \bitwiseshift \scratchcounterone -16 +% \uchexnumber{\scratchcounterone})\par +% +% \scratchcounterone \bitwiseset "FFFFFFFF +% \scratchcountertwo \bitwiseset "FFFFFFFE +% +% \the\scratchcounterone : \uchexnumber{\scratchcounterone}\par +% \the\scratchcountertwo : \uchexnumber{\scratchcountertwo}\par + +% I need to check this on the garden run as it looks like that server is some 50% +% faster than my (in terms of computers) old laptop. + +% \testfeatureonce{10000}{\scratchcounter \bitwiseand "01 "02 } \elapsedtime\par +% \testfeatureonce{10000}{\scratchcounter \numexpr "01+"02\relax} \elapsedtime\par +% \testfeatureonce{10000}{\ifcase\bitwiseand \scratchcounterone \plusone \else \fi} \elapsedtime\par +% \testfeatureonce{10000}{\ifbitwiseand \scratchcounterone \plusone \else \fi} \elapsedtime\par +% %testfeatureonce{10000}{\ifnum \scratchcounterone=\plusone \else \fi} \elapsedtime\par +% +% \testfeatureonce{100000}{\scratchcounter = \bitwise 1101 } \elapsedtime\par +% \testfeatureonce{100000}{\scratchcounter = \bitwise 11011101110111011101110111011101 } \elapsedtime\par diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-contents.tex b/doc/context/sources/general/manuals/evenmore/evenmore-contents.tex index d20b45eee..6d617ad5f 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-contents.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-contents.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/evenmore + \startcomponent evenmore-contents \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-expansion.tex b/doc/context/sources/general/manuals/evenmore/evenmore-expansion.tex index 6c67f07d6..3ef5b3402 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-expansion.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-expansion.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-expressions.tex b/doc/context/sources/general/manuals/evenmore/evenmore-expressions.tex new file mode 100644 index 000000000..37bb5a5bd --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-expressions.tex @@ -0,0 +1,336 @@ +% language=us runpath=texruns:manuals/evenmore + +% This one accidentally ended up in the older history document followingup, +% btu it's now moved here. + +\startcomponent evenmore-expressions + +\environment evenmore-style + +\startchapter[title={Expressions}] + +\startsection[title={Introduction}] + +Do we need bitwise expressions? Actually the answer is \quotation {no, although +not until recently}. In \CONTEXT\ \MKII\ and \MKIV\ we just use integer addition +because we only need to enable things but in \LMTX\ we want to control de +detailed modes that some mechanisms in the engine provides and in order to not +have tons of parameters these use bit sets. We manipulate these with the bitwise +macros that actually are efficient \LUA\ function calls. But, as with some other +extensions in \LUAMETATEX, one way to prevent tracing clutter is to have a few +handy primitives. So let's see what we got. + +{\em I haven't checked all operators and combinations yet!} + +\stopsection + +\startsection[title={Exploration}] + +Already early in the \LUAMETATEX\ development (2019) the expression parser was +extended with an integer division operator \type {:} that we actually use in +\LMTX, and soon after that I added basic bitwise operators but these were never +activated but kept as comment because I didn't want to impact the scanner (even +if we can afford to loose some performance because the scanner has been +optimized). But in the process of cleaning up \quote {todo} comments in the +source code I eventually arrived at expressions again. + +The colon already makes the scanner incompatible because \type {\numexpr 1+2:} +expects a number (which means that we cannot port back) and more operators only +make that less likely. In \CONTEXT\ I nearly always use \type {\relax} as +terminator unless we're sure that lookahead is no issue. \footnote {In the \ETEX\ +expression parser, the normal \type {/} rounds the result. Both the \type {*} and +\type {/} operator have a dedicated code path that assures no loss of accuracy. +The \type {:} operator just divides like \LUA's \type {//} which is an integer +division operator. There are subtle differences between the division variants +which can be noticeable when you go round trip. That is actually the main reason +why this was one of the first things added to \LUAMETATEX\ as I wanted to get rid +of some few scaled point rounding issues. The \ETEX\ expression parser is +somewhat complicated because it can deal with a mix of integers, dimensions and +even glue, but always brings the result back to its main operating model. Because +we adopted some of these \ETEX\ rather early in \CONTEXT\ lookahead pitfalls are +taken care of already.} + +When going over the code in 2021, mostly because I wanted to get rid of some +commented experiments, I decided that the extension should not go into the +normal scanner but that a dedicated, simple and integer only scanner made more +sense, so during a rainy summer weekend I started playing with that. It eventually +became a bit more than initially intended, although the amount of code is rather +minimal. The performance was about twice that of the already available bitwise +macros but operator precedence was not provided (apart from the multiplication +and division operators). The final implementation was different, not that much +faster on simple bitwise operations but could do more complex things in one go. +Performance was not a real reason to provide this anyway because we're talking +microseconds, it's more about less code and better readability. + +The initial primitive command was \type {\bitexpr} and it supported nesting with +parenthesis as the other expressions do. Because there are many operators, also +verbose ones, the non|-|optional \type {\relax} token finishes parsing. But +soon we moved on to two dedicated primitives. + +\stopsection + +\startsection[title={Operators}] + +The set of operators that we have to support is the following. Most have +alternatives so that we can get around catcode issues. + +\starttabulate[||cT|cT|] +\BC add \NC + \NC \NC \NR +\BC subtract \NC - \NC \NC \NR +\BC multiply \NC * \NC \NC \NR +\BC divide \NC / : \NC \NC \NR +\BC mod \NC \letterpercent \NC mod \NC \NR +\BC band \NC & \NC band \NC \NR +\BC bxor \NC ^ \NC bxor \NC \NR +\BC bor \NC \letterbar \space v \NC bor \NC \NR +\BC and \NC && \NC and \NC \NR +\BC or \NC \letterbar\letterbar \NC or \NC \NR +\BC setbit \NC \NC bset \NC \NR +\BC resetbit \NC \NC breset \NC \NR +\BC left \NC << \NC \NC \NR +\BC right \NC >> \NC \NC \NR +\BC less \NC < \NC \NC \NR +\BC lessequal \NC <= \NC \NC \NR +\BC equal \NC = == \NC \NC \NR +\BC moreequal \NC >= \NC \NC \NR +\BC more \NC > \NC \NC \NR +\BC unequal \NC <> != \lettertilde = \NC \NC \NR +\BC not \NC ! \lettertilde \NC not \NC \NR +\stoptabulate + +I considered using \type {++} and type {--} as the \type {bset} and \type +{bunset} shortcuts but that leads to issues because in \TEX\ \type {-+-++--10} is +a valid number and one never knows what sequence (without spaces) gets fed into +an expression. + +Originally I'd added some \UNICODE\ characters but for some reason support of +logical operators is suboptimal so I removed that feature. Because these special +characters are multi|-|byte \UTF\ sequences they are not that much better than +verbose words anyway. + +% 0x00AC ! ¬ lua: not +% 0x00D7 * × +% 0x00F7 / ÷ +% 0x2227 && ∧ c: and lua: and +% 0x2228 || ∨ c: or lua: or +% 0x2229 & ∩ c: bitand lua: band +% 0x222A | ∪ c: bitor lua: bor +% ^ c: bitxor lua: bxor +% 0x2260 != ≠ +% 0x2261 == ≡ +% 0x2264 <= ≤ +% 0x2265 >= ≥ +% 0x22BB xor ⊻ +% 0x22BC nand ⊼ +% 0x22BD nor ⊽ +% 0x22C0 and ⋀ n-arry logical and +% 0x22C1 or ⋁ n-arry logical or +% 0x2AA1 << ⪡ +% 0x2AA2 >> ⪢ + +\stopsection + +\startsection[title={Integers and dimensions}] + +When I was playing a bit with this feature, I wondered if we could mix in some +dimensions. It was actually not that hard to add this: only explicit (verbose) +dimensions had to be intercepted because dimen registers and such are seen as +integers by the integer scanner. Once we're able do handle that, a next step was +to make sure that \typ {2 * 10pt} was permitted, something that the \ETEX\ \type +{\dimexpr} primitives can't handle. So, a variant of the dimen parser has to be +used that makes the unit optional: \type {\dimexpression} and \type +{\numexpression} were born. + +The resulting parsers worked quite well but were about twice as slow as the +normal expression scanners but that is no surprise because they do more. For +instance we are case insensitive and need to handle letter and other (and in a +few cases alignment and superscript) catcodes too. However, with a slightly tuned +integer parser, also possible because the sentinel \type {\relax} makes parsing +more predictable, and a dedicated unit scanner, in the end both the integer and +dimension parser were performing well. It's not like we run them millions of +times in a document. + +\startbuffer +\scratchcounter = \numexpression + "00000 bor "00001 bor "00020 bor "00400 bor "08000 bor "F0000 +\relax +\stopbuffer + +Here is an example that results in {0x\inlinebuffer\uchexnumber\scratchcounter}: + +\typebuffer + +\startbuffer +\scratchcounter = \numexpression + "FFFFF bxor "10101 +\relax +\stopbuffer + +And this gives {0x\inlinebuffer\uchexnumber\scratchcounter}: + +\typebuffer + +We can give numerous example but you get the picture. In the above table you can +see that some operators have equivalents. The reason for this is that a macro +package can change catcodes and some characters have special meanings. So, the +scanner is rather tolerant. + +\startbuffer +\scratchcounterone = 10 +\scratchcountertwo = 20 +\ifcase \numexpression + (\scratchcounterone > 5) && (\scratchcountertwo > 5) +\relax yes\else nop\fi +% +\space +% +\scratchcounterone = 2 +\scratchcountertwo = 4 +\ifcase \numexpression + (\scratchcounterone > 5) and (\scratchcountertwo > 5) +\relax nop\else yes\fi +\stopbuffer + +And this gives \quote {\tttf \inlinebuffer}: + +\typebuffer + +The normal expansion rules apply, so one can use macros and other symbolic +numbers. The only difference in handling dimensions is that we don't support +\type {true} units but these are obsolete in \LUAMETATEX\ anyway. + +In the end I decided to also add an extra conditional so that we can say: + +\starttyping +\ifexpression (\scratchcounterone > 5) and (\scratchcountertwo > 5)\relax + nop +\else + yes +\fi +\stoptyping + +which looks more natural. Actually, this is an nowadays alias because we have two +variants: + +\starttyping +\ifnumexpression ... \relax ... \else ... \fi +\ifdimexpression ... \relax ... \else ... \fi +\stoptyping + +where the later is equivalent to + +\starttyping +\ifboolean\dimexpression ... \relax ... \else ... \fi +\stoptyping + +\stopsection + +\startsection[title={Tracing}] + +When \type {\tracingexpressions} is set to one or higher the intermediate \quote +{reverse polish notation} stack that is used for the calculation is shown, for +instance: + +\starttyping +4:8: {numexpression rpn: 2 5 > 4 5 > and} +\stoptyping + +When you want the output on your console, you need to say: + +\starttyping +\tracingexpressions 1 +\tracingonline 1 +\stoptyping + +The fact that we process the expression in two phases makes it possible to provide this +kind of tracing. + +\stopsection + +\startsection[title={Performance}] + +The following table shows the results of 100.000 evaluations (per line) so you'll +notice that there is a difference. But keep in mind that the new variant can so +more, so it might pay off when we have cases that otherwise demand multiple +traditional expressions. + +\starttabulate[|l|c|] +\NC \type {\dimexpr 4pt*2 + 6pt\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchdimen \dimexpr 4pt*2 + 6pt\relax} \elapsedtime\fi \NC \NR +\NC \type {\dimexpression 4pt*2 + 6pt\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchdimen \dimexpression 4pt*2 + 6pt\relax} \elapsedtime\fi \NC \NR +\NC \type {\dimexpression 2*4pt + 6pt\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchdimen \dimexpression 4pt*2 + 6pt\relax} \elapsedtime\fi \NC \NR +\TB +\NC \type {\numexpr 4 * 2 + 6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr 4 * 2 + 6\relax} \elapsedtime\fi \NC \NR +\NC \type {\numexpression 2 * 4 + 6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression 2 * 4 + 6\relax} \elapsedtime\fi \NC \NR +\TB +\NC \type {\numexpr 4*2+6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr 4*2+6\relax} \elapsedtime\fi \NC \NR +\NC \type {\numexpression 2*4+6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression 2*4+6\relax} \elapsedtime\fi \NC \NR +\TB +\NC \type {\numexpr (1+2)*(3+4)\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr (1+2)*(3+4)\relax} \elapsedtime\fi \NC \NR +\NC \type {\numexpression (1+2)*(3+4)\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression (1+2)*(3+4)\relax} \elapsedtime\fi \NC \NR +\TB +\NC \type {\numexpr (1 + 2) * (3 + 4) \relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr (1 + 2) * (3 + 4) \relax} \elapsedtime\fi \NC \NR +\NC \type {\numexpression (1 + 2) * (3 + 4) \relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression (1 + 2) * (3 + 4) \relax} \elapsedtime\fi \NC \NR +\stoptabulate + +As usual I'll probably find some way to improve performance a bit but that might +than also concern the traditional one. When we compare them, the new numeric +scanner suffers from more options while the new dimension parser gain on the +units. Also, keep in mind than the \LUAMETATEX\ normal parsers are already +somewhat faster than the ones in \LUATEX. The numbers above are calculated when +this document is rendered, so they may change over time and per run. The two +engines compare as follows (mid 2021): + +\starttabulate[|l|c|c|] +\NC \BC \LUATEX \BC \LUAMETATEX \NC \NR +\NC \type {\dimexpr 4pt*2 + 6pt\relax} \NC 0.073 \NC 0.045 \NC \NR +\NC \type {\numexpr 4 * 2 + 6\relax} \NC 0.034 \NC 0.028 \NC \NR +\NC \type {\numexpr 4*2+6\relax} \NC 0.035 \NC 0.032 \NC \NR +\NC \type {\numexpr (1+2)*(3+4)\relax} \NC 0.050 \NC 0.047 \NC \NR +\NC \type {\numexpr (1 + 2) * (3 + 4) \relax} \NC 0.052 \NC 0.048 \NC \NR +\stoptabulate + +Of course tests like these are dubious because often \CPU\ cache will keep the +current code accessible, but who knows. + +It will probably take a while before I will use this in the source code because +first I need to make sure that all works as expected and while doing that I might +adapt some of this. But the basic framework is there. + +\stopsection + +% \start +% \nologbuffering +% \scratchdimen 100pt +% \scratchdimenone 65.536pt +% \scratchdimentwo 65.536bp + +% \tracingonline1 +% \tracingexpressions1 +% \scratchcounter\bitexpr \scratchdimen / 2 \relax\the\scratchcounter\par + +% \scratchcounter\numexpression \scratchdimen / 2sp \relax \the\scratchcounter\par +% \scratchcounter\numexpression \scratchdimen / 1pt \relax \the\scratchcounter\par +% \scratchcounter\numexpression \scratchdimenone / 65.536pt \relax \the\scratchcounter\par +% \scratchcounter\numexpression \scratchdimentwo / 2 \relax \the\scratchcounter\par + +% \scratchcounter\numexpression \scratchcounterone / 4 \relax \the\scratchcounter\par +% \scratchdimen \dimexpression \scratchcounterone / 4 \relax \the\scratchdimen\par + +% \scratchdimen \dimexpression 2 * 4pt \relax \the\scratchdimen\par + +% \tracingexpressions0 +% \tracingonline0 + +% \startTEXpage +% \tracingonline1 +% \tracingexpressions1 +% \the\dimexpr -10pt\relax\quad +% \the\dimexpr 10pt\relax\quad +% \the\dimexpr 10.12 pt\relax\quad +% \the\dimexpression -10pt\relax\quad +% \the\dimexpression 10pt\relax\quad +% \stopTEXpage + +\stopchapter + +\stopcomponent diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-fonts.tex b/doc/context/sources/general/manuals/evenmore/evenmore-fonts.tex index fea3f7ac5..c25f425f1 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-fonts.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-fonts.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-formats.tex b/doc/context/sources/general/manuals/evenmore/evenmore-formats.tex new file mode 100644 index 000000000..7ec95a7d4 --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-formats.tex @@ -0,0 +1,362 @@ +% language=us runpath=texruns:manuals/evenmore + +% This one accidentally ended up in the older history document followingup, +% btu it's now moved here. + +\environment evenmore-style + +\startcomponent evenmore-format + +\startchapter[title={The format file}] + +It is interesting when someone compares macro package and used parameters like +the size of a format file, the output of \type {\tracingall}, or startup time to +make some point. The point I want to make here is that unless you know exactly +what goes on in a run that involves a real document, which can itself involve +multiple runs, such a comparison is rather pointless. For sure I do benchmark but +I can only draw conclusions of what I (can) know (about). Yes, benchmarking your +own work makes sense but doing that in comparisons to what you consider +comparable variants assumes knowledge of more than your own work and objectives. + + +For instance, when you load few fonts, typeset one page and don't do anything +that demands any processing or multiple runs, you basically don't measure +anything. More interesting are the differences between 10 or 500 pages, a few +font calls or tens of thousands, no color of extensive usage of color and other +properties, interfacing, including inheritance of document constructs, etc. And +even then, when comparing macro packages, it is kind of tricky to deduce much +from what you observe. You really need to know what is going on inside and also +how that relates to for instance adaptive font scaling. You can have a fast start +up but if a users needs one tikz picture, loading that package alone will make +you forget the initial startup time. You always pay a price for advanced features +and integration! And we didn't even talk about the operating system caching +files, running on a network share, sharing processors among virtual machines, +etc. + +Pointless comparing is also true for looking at the log file when enabling \type +{\tracingall}. When a macro package loads stuff at startup you can be sure that +the log file is larger. When a font or language is loaded the first time, or +maybe when math is set up there can be plenty of lines dumped. Advanced analysis +of conditions and trial runs come at a price too. And eventually, when a box is +shown the configured depth and breadth really matter, and it might also be that +the engine provides much more (verbose) detail. So, a comparison is again +pointless. It can also backfire. Over the decades of developing \CONTEXT\ I have +heard people working on systems make claims like \quotation {We prefer not to +\unknown} or \quotation {It is better to it this way \unknown} or (often about +operating system) \quotation {It is bad that \unknown} just to see years later +the same being done in the presumed better alternative. I can have a good laugh +about that: do this and don't do that backfiring. + +That brings us to the format file. When you make a \CONTEXT\ format with the +English user interface, with interfacing being a feature that itself introduces +overhead, the \LUATEX\ engine will show this at the end: + +\starttyping +Beginning to dump on file cont-en.fmt + (format=cont-en 2021.6.9) +48605 strings using 784307 bytes +1050637 memory locations dumped; current usage is 414&523763 +44974 multiletter control sequences +\font\nullfont=nullfont +0 preloaded fonts +\stoptyping + +The file itself is quite large: 11,129,903 bytes. However, it is actually much +larger because the format file is compressed! The real size is 19.399.216. Not +taking that into account when comparing the size of format files is kind of bad +because compression directly relates to what resources a format use and how +usage is distributed over the available memory blobs. The \LUATEX\ engine does +some optimizations and saves the data sparse but the more holes you create, the +worse it gets. For instance, the large character vectors are compartmentalized in +order to handle \UNICODE\ efficiently so the used memory relates to what you +define: do you set up all catcodes or just a subset. Maybe you delay some +initialization to after the format is loaded, in which case a smaller format file +gets compensated by more memory usage and initializaton time afterwards. Maybe +your temporary macros create holes in the token array. The memory that is +configured in the configuration files also matter. Some memory blobs are saved at +their configured size, others dismiss the top part that is not used when saving +the format but allocate the lot when the format is loaded. That means that memory +usage in for instance \LUATEX\ can be much larger than a format file suggests. +Keep in mind that a format file is basically a memory dump. + +Now, how does \LUAMETATEX\ compare to \LUATEX. Again we will look at the size of +the format file, but you need to keep in mind that for various reasons the \LMTX\ +macros are somewhat more efficient than the \MKIV\ ones, in the meantime some new +mechanism were added, which adds more \TEX\ and \LUA\ code, but I still expect +(at least for now) a smaller format file. However when we create the format we +see this (reformatted): + +\starttyping +Dumping format 'cont-en.fmt 2021.6.9' in file 'cont-en.fmt': +tokenlist compacted from 489733 to 488204 entries, +1437 potentially aliased lua call/value entries, +max string length 69, 16 fingerprint ++ 16 engine + 28 preamble ++ 836326 stringpool ++ 10655 nodes + 3905660 tokens ++ 705300 equivalents ++ 23072 math codes + 493024 text codes ++ 38132 primitives + 497352 hashtable ++ 4 fonts + 10272 math + 1008 language + 180 insert ++ 10305643 bytecodes ++ 12 housekeeping = 16826700 total. +\stoptyping + +This looks quite different from the \LUATEX\ output. Here we report more detail: +for each blob we mention the number of bytes used. The final result is a file +that takes 16.826.700 bytes. That number should be compared with the 19.399.216 +for \LUATEX. So, we need less indeed. But, when we compress the \LUAMETATEX\ +format we get this: 5,913,932 which is much less than the 11,129,903 compressed +size that the \LUATEX\ engine makes of it. One reason for using level 3 zip +compression compression in \LUATEX\ is that (definitely when we started) it loads +faster. It adds to creating the dump but doesn't really influence loading, +although that depends a bit on the compiler used. It is not easy to see from +these numbers what goes on, but when you consider the fact that we mostly store +32 bit numbers it will also be clear that many can be zero or have two or three +zero bytes. There's a lot of repetition involved! + +So let's look at some of these numbers. The mentioning of token list compaction +relates to getting rid of holes in memory. Each token takes 8 bytes, 4 for the +token identifier, internally called a cmd and chr, and 4 for a value like an +integer or dimension value, or a glue pointer, or a pointer to a next token, etc. +In our case compaction doesn't save that much. + +The mentioning of potentially aliased \LUA\ call|/|value entries is more a warning. +Because the \LUA\ engine starts fresh each run, you cannot store its \quote +{pointers} and because hashes are randomized this means that you need to delay +initialization to startup time, definitely for function tokens. + +Strings in \TEX\ can be pretty long but in practice they aren't. In \CONTEXT\ the +maximum string length is 69. This makes it possible to use one byte for +registering the string length instead of four which saves quite a bit. Of course +one large string will spoil this game. + +The fingerprint, engine, preamble and later housekeeping bytes can be neglected +but the string pool not. These are the bytes that make up the strings. The bytes +are stored in format but when loaded become dynamically allocated. The \LUATEX\ +engine and its successor don't really have a pool. + +Now comes a confusing number. There are not tens of thousands of nodes allocated. +A node is just a pointer into a large array so actually node references are just +indices. Their size varies from 2 slots to 25; the largest are par nodes, while +shape nodes are allocated dynamically. So what gets reported are the number of +bytes that nodes take. each node slot takes 8 bytes, so a glyph node of 12 +bytes takes 96 bytes, while a glue spec node (think skip registers) takes 5 slots +or 40 bytes. These are amounts of memory that were not realistic when \TEX\ was +written. For the record: in \LUATEX\ glue spec nodes are not shared, so we have +many more. + +The majority of \TEX\ related dump data is for tokens, and here we need 3905660 +which means 488K tokens (each reported value also includes some overhead). The +memory used for the table of equivalents makes for some 88K of them. This table +relates to macros (their names and content). Keep in mind that (math) character +references are also macros. + +The next sections that get loaded are math and text codes. These are the +mentioned compartimized character properties. The number of math codes is not +that large (because we delay much of math) but the text codes are plenty, think +of lc, uc, sf, hj, catcodes, etc. Compared to \LUATEX\ we have more categories +but use less space because we have an more granular storage model. Optimizing +that bit really payed off, also because we have more vectors. + +The way primitives and macro names get resolved is pretty much the same in all +engines but by using the fact that we operate in 32 bit I could actually get rid +of some parallel tables that handle saving and restore. Some optimizations relate +to the fact that the register ranges are part of the game so basically we have +some holes in there when they are not used. I guess this is why \ETEX\ uses a +sparse model for the registers above 255. What also saved a lot is that we don't +need to store font names, because these are available in another way; even in +\LUATEX\ that takes a large, basically useless, chunk. The memory that a macro +without parameters consumes is 8 bytes smaller and in \CONTEXT\ we have lots of +these. +We don't really store fonts, so that section is small, but we do store the math +parameters, and there is not much we can save there. We also have more such +parameters in \LUAMETATEX\ so there we might actually use more storage. The +information related to languages is also minimal because patterns and exceptions +are loaded at runtime. A new category (compared to \LUATEX) is inserts because in +\LUAMETATEX\ we can use an alternative (not register based) variant. As you can +see from the 180 bytes used, indeed \CONTEXT\ is using that variant. + +That leaves a large block of more than 10 million bytes that relates to \LUA\ +byte code. A large part of that is the huge \LUA\ character table that \CONTEXT\ +uses. The implementation of font handling also takes quite a bit and we're not +even talking of all the auxiliary \LUA\ modules, \XML\ processing, etc. When +\CONTEXT\ would load that on demand, which is nearly always, the format file +would be much smaller but one would pay for it later. Loading the (some 600) +\LUA\ byte code chunks takes of course some time as does initialization but not +much. + +All that said, the reason why we have a large format file can be understood well +if one considers what goes in there. The \CONTEXT\ format files for \PDFTEX\ and +\XETEX\ are 3.3 and 4.7 MB each which is smaller but not that much when you +consider the fact that there is no \LUA\ code stored and that there are less +character tables and an \ETEX\ register model used. But a format file is not the +whole story. Runtime memory usage also comes at a price. + +The current memory settings of \CONTEXT\ are as follows; these values get +reported when a format has been generated and can be queried at runtime an any +moment: + +\starttabulate[|l|r|r|r|r|] +\BC \BC max \BC min \BC set \BC stp \BC \NR +\HL +\BC string \NC 2097152 \NC 150000 \NC 500000 \NC 100000 \NC \NR +\BC pool \NC 100000000 \NC 10000000 \NC 20000000 \NC 1000000 \NC \NR +\BC hash \NC 2097152 \NC 150000 \NC 250000 \NC 100000 \NC \NR +\BC lookup \NC 2097152 \NC 150000 \NC 250000 \NC 100000 \NC \NR +\BC node \NC 50000000 \NC 1000000 \NC 5000000 \NC 500000 \NC \NR +\BC token \NC 10000000 \NC 1000000 \NC 10000000 \NC 250000 \NC \NR +\BC buffer \NC 100000000 \NC 1000000 \NC 10000000 \NC 1000000 \NC \NR +\BC input \NC 100000 \NC 10000 \NC 100000 \NC 10000 \NC \NR +\BC file \NC 2000 \NC 500 \NC 2000 \NC 250 \NC \NR +\BC nest \NC 10000 \NC 1000 \NC 10000 \NC 1000 \NC \NR +\BC parameter \NC 100000 \NC 20000 \NC 100000 \NC 10000 \NC \NR +\BC save \NC 500000 \NC 100000 \NC 500000 \NC 10000 \NC \NR +\BC font \NC 100000 \NC 250 \NC 250 \NC 250 \NC \NR +\BC language \NC 10000 \NC 250 \NC 250 \NC 250 \NC \NR +\BC mark \NC 10000 \NC 50 \NC 50 \NC 50 \NC \NR +\BC insert \NC 500 \NC 10 \NC 10 \NC 10 \NC \NR +\stoptabulate + +The maxima is what can be used at most. Apart from the magic number 2097152 all +these maxima can be bumped at compile time but if you need more, you might wonder +of your approach to rendering makes sense. The minima are what always gets +allocated, and again these are hard coded defaults. The size can be configured +and is normally the same as the minima but we use larger values in \CONTEXT. The +step is how much an initial memory blob will grow when more is needed than is +currently available. The last four entries show that we don't start out with many +fonts (especially when we use the \CONTEXT\ compact font model not that many are +needed) and because \CONTEXT\ implements marks in a different way we actually +don't need them. We do use the new insert properties storage model and for now +the set sizes are enough for what we need. + +In practice a \LUAMETATEX\ run uses less memory than a \LUATEX\ one, not only +because memory allocation is more dynamic, but also because of other +optimizations. When the compact font model is used (something \CONTEXT) even less +memory is needed. Even this claim should be made with care. Whenever I discuss +the use of resources one needs to limit the conclusions to \CONTEXT. I can't +speak for other macro packages simply because I don't know the internals and the +design decisions made and their impact on the statistics. As a teaser I show the +impact of some definitions: + +\starttyping +\chardef \MyFooA1234 +\Umathchardef\MyFooB"1 "0 "1234 +\Umathcode 1 2 3 4 +\def \MyFooC{ABC} +\def \MyFooD#1{A#1C} +\def \MyFooE{\directlua{print("some lua")}} +\stoptyping + +The stringpool grows because we store the names (here they are of equal length). +Only symbolic definitions bump the hashtable and equivalents. And with +definitions that have text inside the number of bytes taken by tokens grows fast +because every character in that linked list takes 8 bytes, 4 for the character +with its catcode state and 4 for the link to the next token. + +\starttabulate[|l||||||] +\BC \BC stringpool \BC tokens \BC equivalents \BC hashtable \BC total \NC \NR +\HL +\NC \NC 836408 \NC 3906124 \NC 705316 \NC 497396 \NC 16828987 \NC \NR +\NC \type {\chardef} \NC 836415 \NC 3906116 \NC 705324 \NC 497408 \NC 16829006 \NC \NR +\NC \type {\Umathchardef} \NC 836422 \NC 3906116 \NC 705324 \NC 497420 \NC 16829025 \NC \NR +\NC \type {\Umathcode} \NC 836422 \NC 3906124 \NC 705324 \NC 497420 \NC 16829033 \NC \NR +\NC \type {\def} (no arg) \NC 836429 \NC 3906148 \NC 705332 \NC 497428 \NC 16829080 \NC \NR +\NC \type {\def} (arg) \NC 836436 \NC 3906196 \NC 705340 \NC 497440 \NC 16829155 \NC \NR +\NC \type {\def} (text) \NC 836443 \NC 3906372 \NC 705348 \NC 497452 \NC 16829358 \NC \NR +\stoptabulate + +So, every time a user wants some feature (some extra checking, a warning, color +or font support for some element) that results in a trivial extension to the +core, it can bump the size fo the format file more than you think. Of course when +it leads to some overhaul sharing code can actually make the format shrink too. I +hope it is clear now that there really is not much to deduce from the bare +numbers. Just try to imagine what: + +\starttyping +\definefilesynonym + [type-imp-newcomputermodern-book.mkiv] + [type-imp-newcomputermodern.mkiv] +\stoptyping + +adds to the format. Convenience has a price. + +\stopchapter + +\stopcomponent + +% Some bonus content: + +When processing thousand paragraphs \type {tufte.tex}, staying below 4 seconds +(just over 60 pages per second) all|-|in that looks ok. But it doesn't say that +much. Outputting 1000 pages in 2 seconds tells a bit about the overhead on a page +but again in practice things work out differently. So what do we need to +consider? + +\startitemize + +\startitem + Check what macros and resources are preloaded and what gets always loaded at + runtime. +\stopitem + +\startitem + After a first run it's likely that the operating system has resources in its + cache so start measuring after a few runs. +\stopitem + +\startitem + Best run a test many times and and take the average runtime. +\stopitem + +\startitem + Simple macro performance tests can be faster than in real usage because the + related bytes are in \CPU\ cache memory. So one can only use that to test a + specific improvement (or hit due to added functionality). +\stopitem + +\startitem + The size of the used \TEX\ tree can matter. The file databases need to be + loaded and consulted. +\stopitem + +\startitem + The binary matters: is it optimized, does it load libraries, is it 64 bit or not. +\stopitem + +\startitem + Local and|/|or global font definitions can hit performance and when a style + does many redundant switches it might hit performance. Of course that only is + the case when font switching is adaptive. +\stopitem + +\startitem + The granularity of subsystems impacts performance: advanced color support, + inheritance used in mechanisms, abstraction combined with extensive + support for features, it all matters. +\stopitem + +\startitem + The more features one enables the more it will impact performance as does + preprocessing the input (normalizing, bidi checking, etc). +\stopitem + +\startitem + It matters how the page (and layout) dimensions are defined. Although + language doesn't really play a role (apart from possible hyphenation) + specific scripts might. +\stopitem + +\stopitemize + +These are just a few points, but it might be clear that I don't take comparisons +too serious simply because it's real runs that matter. As long as we're in the +runtime comfort zone we're okay. You can run tests within the domain of a macro +package but comparing macro package makes not that much sense. It can even +backfire, especially when claims were made about what should be or not be in a +kernel (while later violating that) or relying on old stories (or rumors) about a +variant macro package being slow. (The same is true when comparing one's favorite +operating system.) Yes, the \CONTEXT\ format file is huge and performance less +than for instance plain \TEX. If that is a problem and not a virtue then make +sure your own alternative will never end up like that. And just don't come to +conclusions about a system that you don't really know. diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-hyphenation.tex b/doc/context/sources/general/manuals/evenmore/evenmore-hyphenation.tex new file mode 100644 index 000000000..50113ed27 --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-hyphenation.tex @@ -0,0 +1,426 @@ +% language=us runpath=texruns:manuals/evenmore + +\environment evenmore-style + +\startcomponent evenmore-hyphenation + +\usebodyfont[pagella] + +\startchapter[title=Hyphenation] + +\startsection[title={Introduction}] + +Hyphenation is driven by the character codes. In a traditional \TEX\ such a code +accesses a glyph in a font, which is why the font encoding mattered, but in +\LUATEX\ we use \UNICODE\ and when hyphenation is applied. \footnote {In +\CONTEXT\ \MKII\ we also use \UTF\ patterns, which made it possible to ship +patterns that didn't depend on a font encoding. Mojca and Arthur made \UTF\ the +default when the (upgraded) hyphenation pattern project started.} Later, the +character codes are adapted by the font handler where they become glyphs. There +are moments when you don't want to hyphenate and a cheap trick is to switch to a +language that has no hyphenation patterns. But, in a system like \CONTEXT\ that +doesn't work well because we have lots of language bound properties. Therefore in +\MKIV\ we set the left- and right hyphen minima to extreme values, something that +blocks hyphenation quite well. But this is not a pretty solution at all. Even +worse is that when we have situations where discretionaries (\type +{\discretionary}), automatic (\type{-}) or explicit (\type {\-}) are used these +still kick in. + +For that reason in \LMTX\ we have a mode variable that controls hyphenation. In +\LUATEX\ we have primitives like \type {\compoundhyphenmode}, \type +{\hyphenationbounds} and \type {\hyphenpenaltymode} that controlled how +hyphenation and discretionary injection is handled but when in \LUAMETATEX\ the +more generic \type {\hyphenationmode} parameter was introduced the precursors +were all merged into this one. One can argue that this is a form of regression +but there are good reasons, most noticeably the fact that we keep these +properties with glyph nodes so that we have better control over them in grouped +situations where as some operations happen when the paragraph as whole get +treated local overloads are lost. \footnote {Of course it also is a wink to those +who complain that we add primitives to an otherwise leaner variant of \LUATEX, +but let us not elaborate on that misunderstanding.} It anyway means that in +\LMTX\ we have to set different parameters but that is no big deal because users +are supposed to use the more high level interfaces; instead of setting parameters +to values one flips bits in \type {\hyphenationmode}, which in the end makes more +sense and also permits extensions later without adding much overhead. + +Currently this mode parameter controls the following options: + +\starttabulate[|Tr|||] +\NC \uchexnumber{\normalhyphenationcode} \NC \type{\normalhyphenationcode} \NC honour the (normal) \type{\discretionary} primitive \NC \NR +\NC \uchexnumber{\automatichyphenationcode} \NC \type{\automatichyphenationcode} \NC turn \type {-} into (automatic) discretionaries \NC \NR +\NC \uchexnumber{\explicithyphenationcode} \NC \type{\explicithyphenationcode} \NC turn \type {\-} into (explicit) discretionaries \NC \NR +\NC \uchexnumber{\syllablehyphenationcode} \NC \type{\syllablehyphenationcode} \NC hyphenate (syllable) according to language \NC \NR +\NC \uchexnumber{\uppercasehyphenationcode} \NC \type{\uppercasehyphenationcode} \NC hyphenate uppercase characters too \NC \NR +\NC \uchexnumber{\compoundhyphenationcode} \NC \type{\compoundhyphenationcode} \NC permit break at an explicit hyphen (border cases) \NC \NR +\NC \uchexnumber{\strictstarthyphenationcode} \NC \type{\strictstarthyphenationcode} \NC traditional \TEX\ compatibility wrt the start of a word \NC \NR +\NC \uchexnumber{\strictendhyphenationcode} \NC \type{\strictendhyphenationcode} \NC traditional \TEX\ compatibility wrt the end of a word \NC \NR +\NC \uchexnumber{\automaticpenaltyhyphenationcode} \NC \type{\automaticpenaltyhyphenationcode} \NC use \type {\automatichyphenpenalty} \NC \NR +\NC \uchexnumber{\explicitpenaltyhyphenationcode} \NC \type{\explicitpenaltyhyphenationcode} \NC use \type {\explicithyphenpenalty} \NC \NR +\NC \uchexnumber{\permitgluehyphenationcode} \NC \type{\permitgluehyphenationcode} \NC turn glue in discretionaries into kerns \NC \NR +\stoptabulate + +The default \CONTEXT\ setup is: + +\starttyping +\hyphenationmode \numexpr + \normalhyphenationcode + + \automatichyphenationcode + + \explicithyphenationcode + + \syllablehyphenationcode + + \uppercasehyphenationcode + + \compoundhyphenationcode + % \strictstarthyphenationcode + % \strictendhyphenationcode + + \automaticpenaltyhyphenationcode + + \explicitpenaltyhyphenationcode + + \permitgluehyphenationcode +\relax +\stoptyping + +When a discretionary node is created (triggered by \type {\discretionary}) the +current value is used. Injected glyph nodes on the other hand will store the +current value and use that when it is needed for hyphenating the list. + +\stopsection + +\startsection[title={Controlling hyphenation}] + +We start with an example that has some Dutch words: + +\startbuffer[sample] +NEDERLANDS\par Nederlands\par nederlands\par +\CONTEXT \par test\-test\par test-test \par +\stopbuffer + +\typebuffer[sample] + +\startbuffer[result] +\startlinecorrection +\dontleavehmode \dorecurse{\boxlines\scratchboxone} {% + \setbox\scratchbox\boxline\scratchboxone#1% + \ruledhpack{\strut\unhbox\scratchbox}% + \kern.25\emwidth +} +\stoplinecorrection +\stopbuffer + +When we typeset this with a \type {\hsize} of 2mm we get: + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} + +\getbuffer[result] + +But when we block hyphenation with \type {\nohyhens} we see: + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \nohyphens \getbuffer[sample]} + +\getbuffer[result] + +The \MKIV\ behavior can be emulated by setting the mode as follows + +\startbuffer[demo] +\bitwiseflip \hyphenationmode \syllablehyphenationcode +\stopbuffer + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[demo] \getbuffer[sample]} + +\getbuffer[result] + +This time the three non|-|syllable variants get hyphenated and that is not what +we want. In this case there is a \type {\discretionary} in the definition of the +macro that generates \CONTEXT\ and, apart from the fact that we might not even +want to hyphenate logos, we have to block it when we apply \type {\nohyphens}. + +This mode setting are directly applied to the three non|-|syllable variants but +delayed in the syllable discretionaries because hyphenation happens later so the +state becomes a property of glyph nodes. Doing the same for the other +discretionaries would demand an adaption of various pieces of the engine code and +plugged in user (\LUA) code also has to consider it which makes no sense. + +\startbuffer[sample] +\nohyphens nederlands {\dohyphens nederlands} nederlands\par +\stopbuffer + +\typebuffer[sample] + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} +\getbuffer[result] + +Compare this with: + +\startbuffer[sample] +nederlands {\nohyphens nederlands} nederlands\par +\stopbuffer + +\typebuffer[sample] + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} +\getbuffer[result] + +\stopsection + +\startsection[title={Compound hyphenation}] + +Yet another discretionary related issue is with compound words, that is: cases +where \type {\discretionary} commands sit between words. There are of course +tricks to deal with it like adding a huge penalty combined with a zero skip. This +is okay in a traditional \TEX\ engine but in an opened up one you might not want +this. Just to mention one aspect: when processing \OPENTYPE\ fonts you actually +need to look into discretionaries in order to deal with glyphs that interact. And +you don't want to deal with penalties and skips unless they have an explicit +meaning. We show the four possibilities: + +\startbuffer[sample] +nederlands\discretionary {!}{!}{!}nederlands\blank +\stopbuffer + +\typebuffer[sample] + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} +\getbuffer[result] + +\startbuffer[sample] +nederlands\discretionary options 1 {!}{!}{!}nederlands\blank +\stopbuffer + +\typebuffer[sample] + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} +\getbuffer[result] + +\startbuffer[sample] +nederlands\discretionary options 2 {!}{!}{!}nederlands\blank +\stopbuffer + +\typebuffer[sample] + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} +\getbuffer[result] + +\startbuffer[sample] +nederlands\discretionary options 3 {!}{!}{!}nederlands\blank +\stopbuffer + +\typebuffer[sample] + +\setbox\scratchboxone\vbox{\dontcomplain \nl \hsize 2mm \getbuffer[sample]} +\getbuffer[result] + +Here is an example of such an interference. Of course in practice this happens +seldom and certainly not with ligatures. Some fonts have kerning between certain +glyphs and for instance dashes and there it could matter. + +\startbuffer +ef% +\penalty \plustenthousand +\hskip \zeropoint +\discretionary{-}{f}{f}% +\penalty \plustenthousand +\hskip \zeropoint +e +ef\discretionary options 3 {-}{f}{f}e +\stopbuffer + +\typebuffer + +As you can see, we only get the ligature when we set the options. In the process +of processing \OPENTYPE\ features it can be that one actually looses a +discretionary, although we try to prevent this when possible. + +\startlinecorrection +\scale[height=2cm]{\setupbodyfont[pagella]\showglyphs\getbuffer} +\stoplinecorrection + +But, as said, the fact that we don't need the penalties and glue helps at the +\LUA\ end: the cleaner the node list, the better. + +\stopsection + +\startsection[title={Tracing}] + +The already present tracker command has been extended so handle the options: + +\startbuffer[sample0] +\enabletrackers[discretionaries] +\stopbuffer +\startbuffer[sample1] +test\discretionary {]} {[} {[]}test +\stopbuffer +\startbuffer[sample2] +testing\discretionary {]} {[} {[]}testing +\stopbuffer +\startbuffer[sample3] +testing\discretionary options 3 {]} {[} {[]}testing +\stopbuffer + +\typebuffer[sample0,sample1,sample2,sample3] + +\setbox\scratchboxone\vbox{\dontcomplain \getbuffer[sample0,sample1]} \getbuffer[result] +\setbox\scratchboxone\vbox{\dontcomplain \hsize 2mm \getbuffer[sample0,sample2]} \getbuffer[result] +\setbox\scratchboxone\vbox{\dontcomplain \hsize 2mm \getbuffer[sample0,sample3]} \getbuffer[result] + +\stopsection + +\startsection[title={Glue in discretionaries}] + +In the case you cannot predict what goes into a discretionary you can get run into +an error message with respect to unsupported glue in a disc node. The mode value +\number\permitgluehyphenationcode\space makes glue acceptable and turn into +kern, as demonstrated here; + +\startbuffer +{\hsize 1mm \darkblue \discretionary{potential conspiracy}{prophets}{influencers}\par} +\stopbuffer + +\typebuffer + +The line break occurs but the space in the pre part is of course frozen: + +{\getbuffer} + +As usual \TEX\ users will come up with applications. + +\stopsection + +\startsection[title={Penalties}] + +By default the par builder will use the value of \type {\hyphenpenalty} that gets +stored in the discretionary node. However, when the \type {\discretionary} is +followed by a \type {penalty} keyword and a number, that one will. + +\stopsection + +\startsection[title=Exceptions] + +At some point a user on the \CONTEXT\ mailing list wondered how to deal with a case +like this: + +\startbuffer[example] +\switchtobodyfont[pagella]\mainlanguage[de]auffasse +\stopbuffer + +\typebuffer[example] + +\startlinecorrection +\scale[height=2cm]{\inlinebuffer[example]} +\stoplinecorrection + +\startbuffer +\startexceptions[de] +au{f-}{-f}{ff}(f\zwnj f)asse +\stopexceptions +\stopbuffer + +In \LUAMETATEX\ you can block the unwanted ligature using this trick: + +\typebuffer \getbuffer + +\startlinecorrection +\scale[height=2cm]{\inlinebuffer[example]} +\stoplinecorrection + +The exception mechanism in \LUATEX\ and therefore \LUAMETATEX\ works as follows. +When we have this exception: + +\starttyping +au{f-}{-f}{ff}asse +\stoptyping + +the engine will register that exception under \type {auffasse}, that is: the +replacement part determines the word. When it runs into that word, it will create +a so called discretionary node with a pre, post and replace part. However, it +only uses the \type {ff} for a lookup and keeps the original two glyphs: these +become the replacement text. However, in \LUAMETATEX\ you can add an alternative +replacement: + +\startbuffer +\startexceptions[de] +au{f-}{-f}{ff}(st)asse +\stopexceptions +\stopbuffer + +\typebuffer \getbuffer + +This time the replacement text becomes \type {xx}. So we get \type {austasse} and +it is that sequence that is seen by the font handler when it applies its tricks. +On some fonts however + +\startbuffer[example] +\switchtobodyfont[pagella]\mainlanguage[de]auffasse +\stopbuffer + +\startlinecorrection +\scale[height=2cm]{\showglyphs\showfontkerns\inlinebuffer[example]} +\stoplinecorrection + +But in the Pagella font that we use here, a kern is added between the \type {s} and +the \type {t}. If you don't want that you can say this: + +\startbuffer +\startexceptions[de] +au{f-}{-f}{ff}(s\zwnj t)asse +\stopexceptions +\stopbuffer + +\typebuffer \getbuffer + +\startlinecorrection +\scale[height=2cm]{\showglyphs\showfontkerns\inlinebuffer[example]} +\stoplinecorrection + +A \type {zwj} will block a ligature (some fonts have an \type {st} ligature) and a +\type {zwnj} blocks a ligatures as well as kerns. + +You can actually abuse this mechanism for trickery like this: + +\startbuffer +\startexceptions[nl] +wis-kun-d{e-}{o}{eo}(e-o)n-der-wijs +\stopexceptions +\stopbuffer + +\typebuffer \getbuffer + +The Dutch word \type {wiskundeonderwijs} is found as exception and comes out like +this: + +\startbuffer[example] +\switchtobodyfont[pagella]\mainlanguage[nl]wiskundeonderwijs +\stopbuffer + +\startlinecorrection +\scale[height=1cm]{\showglyphs\showfontkerns\inlinebuffer[example]} +\stoplinecorrection + +Watch the hyphen that makes the compound word more visible! The other hyphens in +the exception are proper hyphenation points and when a break happens there a +hyphen is automatically added. The \type {\nokerning} and \type {\noligaturing} +macros can be used grouped: + +\startbuffer[example] +{every}\quad +{\nokerning every}\quad +{\noligaturing every}\quad +{e{\nokerning v}ery}\quad +{e{\glyphoptions\noleftkernglyphoptioncode v}ery}\quad +{e{\glyphoptions\norightkernglyphoptioncode v}ery}\quad +\stopbuffer + +\typebuffer[example] + +There are several low level control options. In addition to those shown here we +have a pair for ligatures: \typ {\noleftligatureglyphoptioncode} and \typ +{\norightligatureglyphoptioncode}. + +\startlinecorrection[blank] +\scale[width=\textwidth]{\showglyphs\showfontkerns\inlinebuffer[example]} +\stoplinecorrection + +There are alternative mechanism, like a blocker that implements a font feature +and a replacement mechanism, but these are not discussed here. + +\stopsection + +\stopchapter + +\stopcomponent diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-inserts.tex b/doc/context/sources/general/manuals/evenmore/evenmore-inserts.tex new file mode 100644 index 000000000..75c79e888 --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-inserts.tex @@ -0,0 +1,198 @@ +% language=us runpath=texruns:manuals/evenmore + +\environment evenmore-style + +\startcomponent evenmore-inserts + +\startchapter[title=Inserts] + +{\em As in other chapters, this is mostly a wrapup of some developments and nota +manual. What is described here might actually evolve. Normally \CONTEXT\ users +will not notice these details, but it might help them to get an idea of what +complications we're dealing with.} + +\startsection[title=Notes] + +Quite some \TEX\ users love footnotes. These typographical elements have two +properties: + +\startitemize[n] +\startitem + A blob of text, often placed at the bottom of a page or at the end of a + section, chapter or document. A note can be a short one|-|liner but also + a paragraph of text. +\stopitem +\startitem + A symbol in the running text that serves as reference to that blob. It can be + a number of just some unique symbol. +\stopitem +\stopitemize + +When such a note is placed at the bottom of a page, the page builder needs to +take its dimensions into account when calculating an optimal page break. It might +even have to split a note over pages. Preferably the reference and note are on +the same page. In order to achieve this there is a special mechanism that deals +with this: inserts. In this chapter we only discuss them in the perspective of +notes but they can also be used for, for instance, top or bottom figures, tables, +etc. + +\stopsection + +\startsection[title=Global and local] + +Normally notes are placed in the text flow. We call these global inserts. When +they are attached to a specific typographical element we tend to call them local +and there is a good reason for that. For instance, when used in a table, one +might want to keep them with that table when it is moved to a place where it +fits. When notes are packaged {\em with} the table they are flushed on demand and +not handled by the page builder. Actually they don't use inserts then. One +problem with global notes is that the \TEX\ engine has some limitations with +respect to inserts. In traditional \TEX, the following example will only result +in note \quote {a} being placed. The other two are buried too deeply. + +\startbuffer[insert] +test \footnote {a} \hbox {test \footnote{b} \hbox {test \footnote {c}} test} test +\stopbuffer + +\typebuffer[insert] + +There are a few places where \TEX\ will bubble up the notes: when it constructs +lines during a breaking a paragraph into lines, and when constructing alignment +cells. In all other cases they are simply discarded. + +In \CONTEXT\ \MKII\ there is a mechanism that can postpone such notes. Normally a +user will not use boxes directly so when some mechanism is used that does box +its content, \CONTEXT\ can collect the notes and flush them in a spot that +exposes them. This works reasonable well be it that we loose some of the +synchronization with the page builder: flushing can happen after a page break. + +Already early in the \CONTEXT\ \MKIV\ development a more advanced mechanism was +introduced, using \LUA\ callbacks: auto migration. There (optionally) inserts are +bubbled up to a level where they are visible but even that has its limitations. Users +explicitly need to enable automigration in \MKIV. + +In \CONTEXT\ \LMTX\ we do it a bit different because the \LUAMETATEX\ engine has +a feature that helps. More about that next. + +\stopsection + +\startsection[title=Pre and post migration] + +There is another mechanism that moves material from the spot: adjusts. Here is an +example (watch the interline spacing): + +\startbuffer[adjust] +\strut test \vadjust pre {\red \strut before} test \vadjust {\red \strut after} +\stopbuffer + +\typebuffer[adjust] + +This gives: + +\blank {\bf\showstruts \getbuffer[adjust]} \blank + +The \type {pre} variant was introduced in \PDFTEX\ at a time that I actually has +use for it (think of marginal notes) but in \MKIV\ we don't use this adjust +mechanism. + +Inserts on the other hand always end up \quote {post} where they are injected. +Some day I might think about what good there is in injecting them \quote {pre} +where they are injected. It might become an option. + +\blank {\bf \showboxes \getbuffer[insert]} \blank + +In this already shown example the footnotes (that are implemented using inserts) +result indeed in text showing up at the bottom of the page, even when they are +wrapped in boxes. The reason is that we have enabled a mechanism that +automatically bubble them up: + +\starttyping +\automigrationmode = 3 % \numexpr 1 + 2 \relax +\stoptyping + +Here bit~1 controls migration in lists and bit~2 controls migration in the main +vertical list (the page flow). In \CONTEXT\ \LMTX\ we enable both. + +\startbuffer +h: \setbox0\hbox{box \footnote{h: box}}\setbox2\hbox{\box 0}\box2\quad +h: \setbox0\hbox{copy \footnote{h: copy}}\setbox2\hbox{\copy 0}\box2\quad +h: \setbox0\hbox{unbox \footnote{h: unhbox}}\setbox2\hbox{\unhbox 0}\box2\quad +h: \setbox0\hbox{uncopy \footnote{h: unhcopy}}\setbox2\hbox{\unhcopy0}\box2\quad +v: \setbox0\hbox{box \footnote{v: box}}\setbox2\vbox{\box 0}\box2\quad +v: \setbox0\hbox{copy \footnote{v: copy}}\setbox2\vbox{\copy 0}\box2\quad +v: \setbox0\hbox{unbox \footnote{v: unhbox}}\setbox2\vbox{\unhbox 0}\box2\quad +v: \setbox0\hbox{uncopy \footnote{v: unhcopy}}\setbox2\vbox{\unhcopy0}\box2\quad +\stopbuffer + +\typebuffer + +This is a kind of torture test: + +\blank \hbox{\bf \getbuffer} \blank + +It might look kind of trivial to accomplish this but it actually took some +experimenting to get it right. It also must be noted that because in \LUATEX\ and +even more in \LUAMETATEX\ the nodes that store a box are much larger than in +traditional \TEX\ and that the price for adding some additional overhead for +dealing with pre- and post material had not that much impact on the performance +of \LUAMETATEX. + +On the one hand we need to make sure that the inserts are kept with the box but +on the other hand they should really migrate when the boxed material gets boxed +itself. This has to be combined with unboxing and copying. It has to work in +regular boxes but also in the main vertical list (the page flow). + +\stopsection + +\startsection[title=Playground] + +You can play a bit with this mechanism by getting and setting the pre and post +material than relates to a box. + +\startbuffer +\setbox0\ruledhbox{\strut content \footnote {footnote}} + +[flush pre\prelistbox 0] % \prelistcopy 0 +[flush post\postlistbox 0] % \postlistcopy 0 +[set pre\setprelistbox 0\ruledhbox{\strut before}] +[set post\setpostlistbox0\ruledhbox{\strut after}] + +\blank \box0 \blank +\stopbuffer + +\typebuffer + +\getbuffer + +Of course there can be side effects but so far this mechanism works reasonable +well, given what it has to deal with. + +\stopsection + +\startsection[title=Marks] + +The low level marks mechanism can use the same pre and post mechanisms but I +didn't test that yet. We have a different approach to marks but eventually might +revert to using the build in mechanism, be it in adapted form. + +\stopsection + +% \startsection[title=Maybe] +% +% Inserts are implemented in a bit special way. A class of inserts needs a box, +% dimen, count and skip register. So, the macro package has to allocate these +% en|-|block. This puts some constraints on the register allocation macros. On the +% agenda is to investigate if a dedicated compound register makes sense. However, +% this means that the interfaces change, but because a \CONTEXT\ user doesn't use +% the low level inserts directly this is not really a problem. Adapting the +% implementation can have the side effect of using a different insert chaining +% which can give a better interface to \LUA\ because the current interface is not +% really useable. +% +% \stopsection + +\stopchapter + +\stopcomponent + + diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-introduction.tex b/doc/context/sources/general/manuals/evenmore/evenmore-introduction.tex index d5bfbdad7..b66c8a5e4 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-introduction.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-introduction.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \startcomponent evenmore-introduction diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-keywords.tex b/doc/context/sources/general/manuals/evenmore/evenmore-keywords.tex index 1ffacf0ab..ddf3e0f0d 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-keywords.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-keywords.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore % Talking of keywords: Jacob Collier, Count The People is definitely an example % of showing keywords and no way that the fonts used there are done by tex: diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-libraries.tex b/doc/context/sources/general/manuals/evenmore/evenmore-libraries.tex index 5cca165bb..9e003ea2a 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-libraries.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-libraries.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \startcomponent evenmore-libraries diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex b/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex index 64ceaa353..80e065c48 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-normalization.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore % \enabletrackers[nodes.directions] @@ -102,8 +102,8 @@ The decision making is influenced by quite some factors, like: \starttexdefinition Sample #1#2 \OldNormalizeLineMode\normalizelinemode - \bitwiseflip \normalizelinemode \normalizelinemodecode - \bitwiseflip \normalizelinemode \indentskipmodecode + \bitwiseflip \normalizelinemode \normalizelinenormalizecode + \bitwiseflip \normalizelinemode \parindentskipnormalizecode \startsubsubject[title={#1}] \typebuffer[#2] \startlinecorrection diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-numbers.tex b/doc/context/sources/general/manuals/evenmore/evenmore-numbers.tex index 9b8e10a56..7bbdbdeec 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-numbers.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-numbers.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \startcomponent evenmore-numbers diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-offloading.tex b/doc/context/sources/general/manuals/evenmore/evenmore-offloading.tex new file mode 100644 index 000000000..6a7de5a06 --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-offloading.tex @@ -0,0 +1,127 @@ +% language=us runpath=texruns:manuals/evenmore + +% \showusage + +% This chapter was started when I updated some of the dynamic memory management +% code. The musical timestamp is the discovery (via Sputs & Ghostnote drumming +% sessions) of DOMI & JS BECK (also see: keyscape, nord) and after that Louis +% Cole (Live 2019). So much talent around ... + +\environment evenmore-style + +\startcomponent evenmore-offloading + +\startchapter[title={Offloading}] + +The \LUAMETATEX\ source code started from \LUATEX\ with the idea to make a lean +and mean variant. In the process the whole code base has been overhauled. Because +we develop and maintain \CONTEXT\ in parallel all kind of experiments can be +conducted. \footnote {To my best knowledge macro packages like \LATEX\ have +decided to use the (mostly stable) \LUATEX\ in combination with a built in font +renderer. The wish for stability from developers of other macro packages was one +of the reasons for starting the \LUAMETATEX\ project: \LUATEX\ could no longer +evolve without folks complaining. So, \LUAMETATEX, at least in the short run, is +no reasonable option for other macro packages, unless policies have changed.} + +Already early in the process the decision was made to remove the backend code. I +already had a working \LUATEX\ & \MKIV\ setup with an independent backend so that +was an easy step. Later I removed that code from \MKIV\ because a dualistic model +made for a messy \CONTEXT\ code base, and for \LUATEX\ I like to stick to the +reference implementation. Of course one can wonder if we still have \TEX\ when +there is no backend but you need to keep in mind that \TEX\ always needs some +kind of backend program. If \DVI\ was still a used format (with \CONTEXT) I could +write a \DVI\ backend without much effort, but \PDF\ is kind of the standard now. +The relevant primitives can easily be defined in \LUA. + +What more defines \TEX ? Of course the macro language, but in addition to that we +have the handling of fonts, hyphenation and building various lists, those that +eventually make it into paragraphs and pages. The \ETEX, \PDFTEX\ and \OMEGA\ +engines have added bits and pieces and of course \LUATEX\ added its share. + +In \LUAMETATEX\ the macro part has been extended. Very few things were dropped, +most noticeably \type {\long} and \type {\outer} are no longer effective but +that made for a better \type {\protected} and gave room for \type {\frozen}. + +The handling of fonts is mostly delegated to callbacks but we do have the +original transformation mechanism available. However, loading fonts is now up to +\LUA. If that is done right, there is no difference with \TEX. One can argue that +when a missing piece in the binary is complemented by a \LUA\ solution that gets +plugged in we just are like \TEX. After all, nowhere is said that the engine has +to be written in one language and in the \CWEB\ setup of \TEXLIVE\ we already mix +languages anyway. + +The language part is also upgraded and because handling hyphenation has been +extended we're, as with fonts, going beyond what traditional \TEX\ offers. The +code for hyphenating is slightly different because we permit runtime loading and +extension, compound and weighted patterns, etc. + +Another deviation is the handling of input and output. Although currently the log +file still happens at the engine end, all the reading and writing from files is +delegated to \LUA. This means that primitives like \type {\openin} and \type +{\write} have to be implemented in \LUA, which is not that hard. It makes a lot +of sense because everything already was driven by callbacks so delegating more to +\LUA\ in the end gave a simpler input handler. For the user (or macro package) it +makes no difference. + +So, assuming that the primitives not present in the engine are provided by \LUA\ +driven counterparts, we can speak of \TEX. So how about \ETEX ? We kept the macro +language extensions, but dropped some others, like the direction related code. +Also the querying of internal codes has been adapted to \LUATEX's internals. +Expressions have been extended a bit. The \type {\scantokens} primitive now uses +the same machinery as the \LUA|-|\TEX\ pipe, which made for less code and adds to +consistency. From the \PDFTEX\ engine we only kept a few things, like protrusion +and expansion. From \OMEGA\ only the concept of localpar and part of the +directional model was kept, but because it's the backend that deals with +directions there is not much there. We also dropped some \LUATEX\ features, for +instance first class image handling only stays as concept (a special kind of +rule) but again it is the backend that really needs to deal with it. + +So, in the end we end up, as intended, with a simpler code base indeed. Of course +there is also stuff that is not in a traditional \TEX\ or \LUATEX. In addition to +\LUA\ some libraries are present, but we avoid dependencies on large third party +bodies of code. The continuous updating in \LUATEX\ told me that this dependency +is a bad thing of we want the program to compile in decades from now (as +libraries come and go and often also politics are involved). There is a small +canonical set of what we provide and although one can use extra libraries it +takes some effort. The internals of \LUAMETATEX\ are hidden. + +Just for the record. Because I want to keep as working engine and adapt \CONTEXT\ +in parallel, the process is rather time consuming. Every optimization, removal of +unused code, addition of a feature, etc.\ takes multiple runs of the test suite, +checking with \CONTEXT, generating binaries, updating the distribution, and so +on. When we don't use something in \CONTEXT\ it goes on the todo stack, which +means that testing is delayed to when I wrap up in documentation, which only +happens when I think it's stable. For instance: when \type {\openin} and friends +were delegated to \LUA, the \type {\ifeof} primitive was kept around with a +temporary callback, so that \type {tikz} kept working. We don't use those read +channels in \CONTEXT\ so that was a compromise. A while later the if test was +also delegated and the temporary callback removed. There is no way I could do +this kind of stepwise development were it now within the \CONTEXT\ community +where users are willing to accept this. + +Another example of something that took some time is checking all memory +allocation code, adding safeguards, make it more dynamic, making sure we have +more statistics. It needs some experimenting and the \CONTEXT\ tracking code had +to be extended. The main motivation for this is that there are some users out +there (most noticeably Massimiliano who is involved in typesetting some scholar +encyclopedia work) who run really huge jobs and we can run out of memory or even +crash then. \footnote {I'm not that worried about an occasional bug, because the +number is small compared to what got added, changed and improved, but of course +there are always folks who ignore that fact and stress bug(let)s. But, as said, +the \CONTEXT\ users are a patient lot.} Tracing shows that although \TEX\ +allocates its share of memory, in these thousand plus page documents in small +print the regular few dozen megabytes can grow to hundreds, most noticeably taken +up by fonts. Tracing also gives some insight in how fast token and node memory +grows. Of course in this case \LUA\ takes way more memory, something between 1.5 +and 2~GB. Again this is due to the large amount of font instances and also is a +side effect of massive \XML\ processing (keep in mind that the whole tree is in +memory) and the fact that there are plenty of optimizations wrt typesetting +implemented and multiple large registers are added. It's this kind of +(regression) tests that help in stepwise improving \LUAMETATEX. \footnote {In the +process one sometimes find ways to safe memory. For instance, marks used +preallocated arrays that took 1280 KB memory while of course in practice no one +needs that many mark registers. By making that more dynamic we saved a lot.} + +\stopchapter + +\stopcomponent diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-paragraphs.tex b/doc/context/sources/general/manuals/evenmore/evenmore-paragraphs.tex new file mode 100644 index 000000000..c8958a487 --- /dev/null +++ b/doc/context/sources/general/manuals/evenmore/evenmore-paragraphs.tex @@ -0,0 +1,397 @@ +% language=us runpath=texruns:manuals/evenmore + +% End of July 2020 I decided to look into some of the backlog items and paragraphs +% are on them, as are inserts and so. The usual musical time stamp is buying the +% excellent Prince Live at the Aladdin Las Vegas DVD which I ran between the +% moments that I had enough of coding. Some tracks, like Family Name, fit perfectly +% in mid 2020. + +% https://www.youtube.com/watch?v=f8FAJXPBdOg + +\environment evenmore-style + +\startcomponent evenmore-paragraphs + +\enableexperiments[paragraphs.freeze] + +\startchapter[title=Paragraphs] + +{\em This is mostly a wrapup of some developments, and definitely not a tutorial. +What is described here is experimental and successive version of \CONTEXT\ \LMTX\ +will explore their potential. As long a users stay away from the low level +primitives we can try to guarantee consistent behavior (and catch side effect or +deal with known issues).} + +\startsection[title=Freezing] + +A well known property of paragraphs is that when the moment is there to split +into lines, the current state of variables drives it. There are a lot of quite +some variables involved. The most significant one is the \type {\hsize}. Take +this: + +\startbuffer +\bgroup + {\bf Ward:} + \hsize .25\textwidth + {\bf Ward:} \samplefile{ward} + \hsize .75\textwidth % last value +\egroup +\stopbuffer + +\typebuffer + +This gives: + +{\forgetparagraphfreezing \getbuffer} + +But wait, why do we get the full text width here? The reason is that by the time +the paragraph ends, after the \type {\egroup}, the \type {\hsize} is set back to +what it was before the group started. + +\startbuffer +\bgroup + \hsize .25\textwidth + {\bf Ward:} \samplefile{ward} + \hsize .75\textwidth % last value + \par +\egroup +\stopbuffer + +\typebuffer + +This gives: + +{\forgetparagraphfreezing \getbuffer} + +The last \type {\hsize} specified is used. That is not really a problem, but the +fact that we need to explicitly end a paragraph before the group ends actually +is, and in a moment we will see an example where that matters a lot. First a +general solution to this problem is discussed. In the next example, we do group, +but inside that group we take a snapshot of the \type {\hsize}: + +\startbuffer +\bgroup + \hsize .80\textwidth + \dontleavehmode + \snapshotpar "000001 + {\bf Ward:} \samplefile{ward} +\egroup +\stopbuffer + +\typebuffer + +This time we get: + +{\forgetparagraphfreezing \getbuffer} + +The magic number used in the snapshot relates to the \type {\hsize}. + +\startcolumns[n=2] +\starttabulate[|T||] +\NC 0x\uchexnumbers{\hsizefrozenparcode } \NC hsize \NC \NR +\NC 0x\uchexnumbers{\skipfrozenparcode } \NC leftskip rightskip \NC \NR +\NC 0x\uchexnumbers{\hangfrozenparcode } \NC hangindent hangafter \NC \NR +\NC 0x\uchexnumbers{\indentfrozenparcode } \NC parindent \NC \NR +\NC 0x\uchexnumbers{\parfillfrozenparcode } \NC parfillskip parfillleftskip \NC \NR +\NC 0x\uchexnumbers{\adjustfrozenparcode } \NC adjustspacing adjustspacingstep adjustspacingshrink adjustspacingstretch \NC \NR +\NC 0x\uchexnumbers{\protrudefrozenparcode } \NC protrudechars \NC \NR +\NC 0x\uchexnumbers{\tolerancefrozenparcode } \NC pretolerance tolerance \NC \NR +\NC 0x\uchexnumbers{\stretchfrozenparcode } \NC emergencystretch \NC \NR +\NC 0x\uchexnumbers{\loosenessfrozenparcode } \NC looseness \NC \NR +\NC 0x\uchexnumbers{\lastlinefrozenparcode } \NC lastlinefit \NC \NR +\NC 0x\uchexnumbers{\linepenaltyfrozenparcode } \NC linepenalty interlinepenalty interlinepenalties \NC \NR +\NC 0x\uchexnumbers{\clubpenaltyfrozenparcode } \NC clubpenalty clubpenalties \NC \NR +\NC 0x\uchexnumbers{\widowpenaltyfrozenparcode } \NC widowpenalty widowpenalties displaywidowpenalty displaywidowpenalties \NC \NR +\NC 0x\uchexnumbers{\brokenpenaltyfrozenparcode} \NC brokenpenalty \NC \NR +\NC 0x\uchexnumbers{\demeritsfrozenparcode } \NC adjdemerits doublehyphendemerits finalhyphendemerits \NC \NR +\NC 0x\uchexnumbers{\shapefrozenparcode } \NC parshape \NC \NR +\NC 0x\uchexnumbers{\linefrozenparcode } \NC baselineskip lineskip lineskiplimit \NC \NR +\NC \NC \NC \NR +\NC 0xFFFFFFF \BC all of them \NC \NR +\stoptabulate +\stopcolumns + +In practice you will set them all on one go, so: + +\starttyping +\snapshotpar "FFFFFFF +\stoptyping + +How often do we need such a feature? Actually more often than one thinks, +especially when we have an unpredictable situation. For instance, when you +typeset from an \XML\ source you often don't know what you get, and you can have +cases that end up like this: + +\startbuffer +\placefigure[left,none]{}{} {Ward: \bf \dorecurse{3}{\samplefile{ward}} } \par +\placefigure[left,none]{}{} {\bf Ward:} \dorecurse{3}{\samplefile{ward}} \par +\stopbuffer + +\typebuffer + +This might render as: + +{\forgetparagraphfreezing \getbuffer} \forgetsidefloats % needed due to interference + +The placement of such a figure is hooked into \type {\everypar} and uses hanging +indentation. Like \type {\hsize}, \type {\hangafter} and \type {\hangindent} can +be forgotten before the paragraph ends. In \MKII\ and \MKIV\ the recommended +solution is to always start a paragraph explicitly, with a strut, forced +indentation of preferably: + +\startbuffer +\dontleavehmode {Ward: \bf \dorecurse{3}{\samplefile{ward} } } \par +\dontleavehmode {\bf Ward:} \dorecurse{3}{\samplefile{ward} } \par +\stopbuffer + +\typebuffer + +In an \XML\ mapping we can hide it but in a regular \TEX\ source this is not +pretty. With little effort we can do the snapping automatically, so that we get: + +\placefigure[left,none]{}{} {Ward: \bf \dorecurse{3}{\samplefile{ward} } } \par +\placefigure[left,none]{}{} {\bf Ward:} \dorecurse{3}{\samplefile{ward} } \par + +and this is what \CONTEXT\ \LMTX\ will do once we're sure that the snapshot +feature behaves well and has no side effects. There is of course some overhead +involved in taking snapshots, keeping track of the values and accessing them +later, but it is rewarding. + +In addition to the numeric \type {\snapshotpar} primitive there is also another +way to take s snapshot. As with the numeric variant, it only takes a snapshot when +in horizontal mode: there has to be a so called local par node at the head of the +current list. The next code shows some how to play with some of what \CONTEXT\ +offers: + +\starttyping +\setuplayout[alternative=doublesided] + +\starttext + +\startbuffer + \dorecurse{8}{ + \interlinepenalties 1 \maxcard + CASE 1: \samplefile{tufte} \par + } \page + + \dorecurse{8}{ + CASE 2: \samplefile{tufte} + \interlinepenalties 1 \maxcard \par + } \page + + \dorecurse{8}{ + CASE 3: \samplefile{tufte} + \interlinepenalties 1 \maxcard \freezeparagraphproperties \par + } \page + + \dorecurse{8}{ + CASE 4: \samplefile{tufte} + \frozen \interlinepenalties 1 \maxcard \par + } \page + + \dorecurse{8}{ + CASE 5: \samplefile{tufte} + \frozen \interlinepenalty \maxcard \par + } \page +\stopbuffer + +\typebuffer \page \getbuffer + +\stoptext +\stoptyping + +When you process this you will notice that the \type {\frozen} prefix also +snapshots the parameter that gets set. Now, there is a pitfall here: some for +these settings are persistent, i.e. they are not reset after a paragraph has been +typeset. For instance, \type {\tolerance} is a general setting, but \type +{\hangindent} is a one shot setting: it's value gets reset after the paragraph +has been dealt with. + +Here is another test one can run to see what happens: + +\starttyping +\dontleavehmode +\defrostparagraphproperties +\writestatus{state}{after start \uchexnumbers{\the\snapshotpar}}% +\writestatus{state}{before set hangindent \uchexnumbers{\the\snapshotpar}}% +\frozen\hangindent10pt +\writestatus{state}{after set hangindent \uchexnumbers{\the\snapshotpar}}% +\writestatus{state}{before set looseness \uchexnumbers{\the\snapshotpar}}% +\frozen\looseness 1 +\writestatus{state}{after set looseness \uchexnumbers{\the\snapshotpar}}% +\writestatus{state}{before set hangafter \uchexnumbers{\the\snapshotpar}}% +\frozen\hangafter 2 +\writestatus{state}{after set hangafter \uchexnumbers{\the\snapshotpar}}% +\begingroup +\writestatus{state}{before set rightskip \uchexnumbers{\the\snapshotpar}}% +\frozen\rightskip2cm +\writestatus{state}{after set rightskip \uchexnumbers{\the\snapshotpar}}% +\endgroup +\writestatus{state}{before reset hangindent \uchexnumbers{\the\snapshotpar}}% +\snapshotpar-\frozenhangindentcode +\writestatus{state}{after reset hangindent \uchexnumbers{\the\snapshotpar}}% +\writestatus{state}{before reset hangafter \uchexnumbers{\the\snapshotpar}}% +\snapshotpar-\frozenhangaftercode +\writestatus{state}{after reset hangafter \uchexnumbers{\the\snapshotpar}}% +... content ... +\stoptyping + +You can group an assignment and then take a snapshot. That way the change doesn't +affect following paragraphs, unless of course to did a global assignment. In +\CONTEXT\ we have a bunch of constants that can be used instead of the hard to +remember bit positions. The \type {\frozen} prefix can also be used with for +instance a \type {\advance} operation. Of course it only has effect for those +(internal) parameters that relate to a paragraph. + +Keep in mind that what is show here will evolve: in \CONTEXT\ \LMTX\ we will +snapshot by default and the core macros are aware of this fact. Although the way +\CONTEXT\ is set up makes it relatively easy to make this paradigm shift users +should anyway be aware of this change when they do their own low level tweaking, +but in that case they probably already are aware of possible interferences. + +\stopsection + +\startsection[title=Wrapping up] + +Another new (low level) feature is wrapping up a paragraph. Traditional \TEX\ +comes with the powerful \type {\everypar} and in \LUAMETATEX\ we now have \type +{\wrapuppar}. This primitive collects tokens that will be expanded just before +the paragraph ends. Here is an example: + +\startbuffer +\dontleavehmode +\wrapuppar{\hfill {\bf ONE}}% +\wrapuppar{\crlf\strut\hfill {\bf TWO}\hfill\strut}% +\wrapuppar{\crlf\strut {\bf THREE}\hfill\strut {\bf FOUR}}% +\samplefile{ward} + +\samplefile{ward} +\stopbuffer + +\typebuffer + +We can only wrapup when we are in a paragraph although one can of course use the +\type {\wrapuppar} command inside an \type {\everypar} if needed. + +\getbuffer + +An more useful example is the following. We leave it to the reader to check it +out: + +\starttyping +\dorecurse{10}{ + \bgroup + \advance\hsize by -#1cm\relax + \dontleavehmode + \wrapuppar{\strut\nobreak\hfill\nobreak QED}% + \samplefile{ward} + \egroup + \par +} +\stoptyping + +\stopsection + +\startsection[title=Insertions] + +The concept of inserts is kind of complicated. They are nodes in a list that make +separate streams. An application of inserts are footnotes. In the text flow a +symbol is typeset (like a raised number) and the note itself becomes an insert. +When a paragraph is broken into lines, these inserts end up in to be boxed line, +but when the line is actually wrapped in a box, these inserts are collected and +injected after the line. The page builder will then take their dimensions into +account when it comes to breaking pages. Depending on how strict the rules are +the inserts will end up on the same page, move, of be broken into lines. + +This all works well as long as the inserts are not burried into boxes: they then +are invisible to the mechanism described before. Take the following example: + +\starttyping +\dontleavehmode +l\hbox{h\footnote{h1} test} +l\hbox{h\footnote{h2}} test +l\hbox{h\footnote{h3}} test +l\footnote{l4} test +l\footnote{l5} test +l\hbox{h\footnote{h6} test} +l\hbox{\hbox{\hbox{h\footnote{h7} test}}} +l\footnote{l8} test +\par +\stoptyping + +% \starttabulate +% \NC test \NC test \footnote{before} \samplefile{tufte} \footnote{after}\NC \NR +% ... +% \NC test \NC test \footnote{before} \samplefile{tufte} \footnote{after}\NC \NR +% \stoptabulate + +In the engines used with \MKII\ only a few footnotes actually show up. In \MKIV\ +the situation is slightly better because there we use some trickery to migrate +these notes to the outer level. But even there it's not perfect because the order +changes. We can actually fix that (and do so) but it comes at a performance penalty +so this is why in \MKIV\ dealing with this is optional. + +\start + \def\Hi#1{\high{\tx#1}} + \dontleavehmode + \hbox{lh\H1 test lh\H2 test lh\H3 test l\H4 test l\H5 test lh\H6 test lh\H7 test l\H8 test} + + \starttabulate[|||||||||||] + \NC \MKII \NC \PDFTEX\ & \XETEX \NC \Hi4 l4 \NC \Hi5 l5 \NC \Hi8 l8 \NC \NC \NC \NC \NC \NC \NR + \NC \MKIV \footnote{In older versions.} \NC \LUATEX \NC \Hi1 h1 \NC \Hi2 h2 \NC \Hi3 h3 \NC \Hi6 h6 \NC \Hi7 h7 \NC \Hi4 l4 \NC \Hi5 l5 \NC \Hi8 l8 \NC \NR + \NC \LMTX \NC \LUAMETATEX \NC \Hi1 h1 \NC \Hi2 h2 \NC \Hi3 h3 \NC \Hi4 l4 \NC \Hi5 l5 \NC \Hi6 h6 \NC \Hi7 h7 \NC \Hi8 l8 \NC \NR + \stoptabulate +\stop + +However, when you look at the results of \LMTX\ you will notice that the +situation is better. This is because we have some code to \LUAMETATEX\ that can +better deal with some cases. Combined with some \LUA\ magic (as in \MKIV) we get +the right order at hardly any runtime overhead. It must be noted that the +original \TEX\ engine for good reason works as it does because there the actual +typesetting (read: resolving glyphs, hyphenation, applying ligatures and kerns, +etc.) is interwoven with the main scanning|/|expanding loops in order to be +efficient on the machines of those times. In \LUATEX\ we have these stages +separated but the code dealing with inserts is the same, if only because we have +to be compatible. In \LUAMETATEX\ we have again a bit simpler code because we use +the fact that lists are double linked, which also makes it possible to add some +magic code dealing with nested inserts without obscuring the code. It comes of +course at a bit performance hit and the nodes related to lists also because +larger but they are already much larger than in other engines, so we don't care +too much about that. It is anyway a mechanism that need to be enabled explicitly. + +\stopsection + +\stopchapter + +\stopcomponent + +% \starttext + +% \def\TestA {\registerparwrapper {A} {[\ignorespaces}{\removeunwantedspaces]\showparwrapperstate{A}}} +% \def\TestB#1{\registerparwrapper {B#1}{(\ignorespaces}{\removeunwantedspaces)\showparwrapperstate{B#1}}} +% \def\TestC {\registerparwrapper {C} {<\ignorespaces}{\removeunwantedspaces>\showparwrapperstate{C}\forgetparwrapper}} +% \def\TestR {\registerparwrapperreverse{R} {<\ignorespaces}{\removeunwantedspaces>\showparwrapperstate{R}}} + +% \start +% \TestA +% \dorecurse{3}{1.#1 before \ruledvbox{\hsize2em\raggedcenter\TestB1 !\par} after\par} \blank +% \dorecurse{3}{2.#1 before \ruledvbox{\hsize3em\raggedcenter !\par} after\par} \blank +% \dorecurse{3}{3.#1 before \ruledvbox{\hsize4em\raggedcenter\TestB2 !} after\par} \blank +% \forgetparwrapper +% \dorecurse{3}{4.#1 before \ruledvbox{\hsize5em\raggedcenter\TestB3 !} after\par} \blank +% \TestC +% \dorecurse{3}{5.#1 before \ruledvbox{\hsize2em\raggedcenter\TestA !} after\par} \blank +% \stop + + +% \start +% \TestA +% \dorecurse{3}{6.#1 before after\par} \blank +% \TestB4 +% \dorecurse{3}{7.#1 before after\par} \blank +% \TestB5 +% \TestR +% \dorecurse{3}{8.#1 before after\par} \blank +% \stop + +% \stoptext diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-parameters.tex b/doc/context/sources/general/manuals/evenmore/evenmore-parameters.tex index b07378fae..c20b7bb2a 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-parameters.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-parameters.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore % This feature was done mid May 2020 with Alien Chatter (I really need to find the % original cd's) in the background. diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-parsing.tex b/doc/context/sources/general/manuals/evenmore/evenmore-parsing.tex index d0cc29db1..caed45846 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-parsing.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-parsing.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-pi.tex b/doc/context/sources/general/manuals/evenmore/evenmore-pi.tex index 197e86826..7a2dd9bf7 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-pi.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-pi.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-style.tex b/doc/context/sources/general/manuals/evenmore/evenmore-style.tex index 07168e57c..0a01679b1 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-style.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-style.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/evenmore + % \enablelmtx % \nopdfcompression diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-threesix.tex b/doc/context/sources/general/manuals/evenmore/evenmore-threesix.tex index d75f9b683..0c0c503ad 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-threesix.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-threesix.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-titlepage.tex b/doc/context/sources/general/manuals/evenmore/evenmore-titlepage.tex index 57418800c..44ca10512 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-titlepage.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-titlepage.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/evenmore + \startcomponent evenmore-titlepage \environment evenmore-style diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex b/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex index 8c1e3ccb0..c30959488 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore % TODO: copy_node_list : return tail % TODO: grabnested diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-whattex.tex b/doc/context/sources/general/manuals/evenmore/evenmore-whattex.tex index 10d3e49f2..0c208839d 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore-whattex.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore-whattex.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/evenmore \startcomponent evenmore-whattex diff --git a/doc/context/sources/general/manuals/evenmore/evenmore.tex b/doc/context/sources/general/manuals/evenmore/evenmore.tex index b351e8c68..f8037b682 100644 --- a/doc/context/sources/general/manuals/evenmore/evenmore.tex +++ b/doc/context/sources/general/manuals/evenmore/evenmore.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/evenmore + \environment evenmore-style \dontcomplain @@ -25,19 +27,13 @@ \component evenmore-parsing \component evenmore-tokens \component evenmore-keywords - - \startchapter[title=Paragraphs] {\em This chapter is not yet public.} \stopchapter - \startchapter[title=Hyphenation] {\em This chapter is not yet public.} \stopchapter - \startchapter[title=Bitwise] {\em This chapter is not yet public.} \stopchapter - \startchapter[title=Offloading] {\em This chapter is not yet public.} \stopchapter - \startchapter[title=Inserts] {\em This chapter is not yet public.} \stopchapter - - % \component evenmore-paragraphs - % \component evenmore-hyphenation - % \component evenmore-bitwise - % \component evenmore-offloading - % \component evenmore-inserts - + \component evenmore-paragraphs + \component evenmore-hyphenation + \component evenmore-bitwise + \component evenmore-offloading + \component evenmore-inserts + \component evenmore-formats + \component evenmore-expressions \stopbodymatter \stopdocument diff --git a/doc/context/sources/general/manuals/followingup/followingup-expressions.tex b/doc/context/sources/general/manuals/followingup/followingup-expressions.tex deleted file mode 100644 index 321496c97..000000000 --- a/doc/context/sources/general/manuals/followingup/followingup-expressions.tex +++ /dev/null @@ -1,333 +0,0 @@ -% language=us runpath=texruns:manuals/followingup - -\startcomponent followingup-expressions - -\environment followingup-style - -\startchapter[title={Expressions}] - -\startsection[title={Introduction}] - -Do we need bitwise expressions? Actually the answer is \quotation {no, although -not until recently}. In \CONTEXT\ \MKII\ and \MKIV\ we just use integer addition -because we only need to enable things but in \LMTX\ we want to control de -detailed modes that some mechanisms in the engine provides and in order to not -have tons of parameters these use bit sets. We manipulate these with the bitwise -macros that actually are efficient \LUA\ function calls. But, as with some other -extensions in \LUAMETATEX, one way to prevent tracing clutter is to have a few -handy primitives. So let's see what we got. - -{\em I haven't checked all operators and combinations yet!} - -\stopsection - -\startsection[title={Exploration}] - -Already early in the \LUAMETATEX\ development (2019) the expression parser was -extended with an integer division operator \type {:} that we actually use in -\LMTX, and soon after that I added basic bitwise operators but these were never -activated but kept as comment because I didn't want to impact the scanner (even -if we can afford to loose some performance because the scanner has been -optimized). But in the process of cleaning up \quote {todo} comments in the -source code I eventually arrived at expressions again. - -The colon already makes the scanner incompatible because \type {\numexpr 1+2:} -expects a number (which means that we cannot port back) and more operators only -make that less likely. In \CONTEXT\ I nearly always use \type {\relax} as -terminator unless we're sure that lookahead is no issue. \footnote {In the \ETEX\ -expression parser, the normal \type {/} rounds the result. Both the \type {*} and -\type {/} operator have a dedicated code path that assures no loss of accuracy. -The \type {:} operator just divides like \LUA's \type {//} which is an integer -division operator. There are subtle differences between the division variants -which can be noticeable when you go round trip. That is actually the main reason -why this was one of the first things added to \LUAMETATEX\ as I wanted to get rid -of some few scaled point rounding issues. The \ETEX\ expression parser is -somewhat complicated because it can deal with a mix of integers, dimensions and -even glue, but always brings the result back to its main operating model. Because -we adopted some of these \ETEX\ rather early in \CONTEXT\ lookahead pitfalls are -taken care of already.} - -When going over the code in 2021, mostly because I wanted to get rid of some -commented experiments, I decided that the extension should not go into the -normal scanner but that a dedicated, simple and integer only scanner made more -sense, so during a rainy summer weekend I started playing with that. It eventually -became a bit more than initially intended, although the amount of code is rather -minimal. The performance was about twice that of the already available bitwise -macros but operator precedence was not provided (apart from the multiplication -and division operators). The final implementation was different, not that much -faster on simple bitwise operations but could do more complex things in one go. -Performance was not a real reason to provide this anyway because we're talking -microseconds, it's more about less code and better readability. - -The initial primitive command was \type {\bitexpr} and it supported nesting with -parenthesis as the other expressions do. Because there are many operators, also -verbose ones, the non|-|optional \type {\relax} token finishes parsing. But -soon we moved on to two dedicated primitives. - -\stopsection - -\startsection[title={Operators}] - -The set of operators that we have to support is the following. Most have -alternatives so that we can get around catcode issues. - -\starttabulate[||cT|cT|] -\BC add \NC + \NC \NC \NR -\BC subtract \NC - \NC \NC \NR -\BC multiply \NC * \NC \NC \NR -\BC divide \NC / : \NC \NC \NR -\BC mod \NC \letterpercent \NC mod \NC \NR -\BC band \NC & \NC band \NC \NR -\BC bxor \NC ^ \NC bxor \NC \NR -\BC bor \NC \letterbar \space v \NC bor \NC \NR -\BC and \NC && \NC and \NC \NR -\BC or \NC \letterbar\letterbar \NC or \NC \NR -\BC setbit \NC \NC bset \NC \NR -\BC resetbit \NC \NC breset \NC \NR -\BC left \NC << \NC \NC \NR -\BC right \NC >> \NC \NC \NR -\BC less \NC < \NC \NC \NR -\BC lessequal \NC <= \NC \NC \NR -\BC equal \NC = == \NC \NC \NR -\BC moreequal \NC >= \NC \NC \NR -\BC more \NC > \NC \NC \NR -\BC unequal \NC <> != \lettertilde = \NC \NC \NR -\BC not \NC ! \lettertilde \NC not \NC \NR -\stoptabulate - -I considered using \type {++} and type {--} as the \type {bset} and \type -{bunset} shortcuts but that leads to issues because in \TEX\ \type {-+-++--10} is -a valid number and one never knows what sequence (without spaces) gets fed into -an expression. - -Originally I'd added some \UNICODE\ characters but for some reason support of -logical operators is suboptimal so I removed that feature. Because these special -characters are multi|-|byte \UTF\ sequences they are not that much better than -verbose words anyway. - -% 0x00AC ! ¬ lua: not -% 0x00D7 * × -% 0x00F7 / ÷ -% 0x2227 && ∧ c: and lua: and -% 0x2228 || ∨ c: or lua: or -% 0x2229 & ∩ c: bitand lua: band -% 0x222A | ∪ c: bitor lua: bor -% ^ c: bitxor lua: bxor -% 0x2260 != ≠ -% 0x2261 == ≡ -% 0x2264 <= ≤ -% 0x2265 >= ≥ -% 0x22BB xor ⊻ -% 0x22BC nand ⊼ -% 0x22BD nor ⊽ -% 0x22C0 and ⋀ n-arry logical and -% 0x22C1 or ⋁ n-arry logical or -% 0x2AA1 << ⪡ -% 0x2AA2 >> ⪢ - -\stopsection - -\startsection[title={Integers and dimensions}] - -When I was playing a bit with this feature, I wondered if we could mix in some -dimensions. It was actually not that hard to add this: only explicit (verbose) -dimensions had to be intercepted because dimen registers and such are seen as -integers by the integer scanner. Once we're able do handle that, a next step was -to make sure that \typ {2 * 10pt} was permitted, something that the \ETEX\ \type -{\dimexpr} primitives can't handle. So, a variant of the dimen parser has to be -used that makes the unit optional: \type {\dimexpression} and \type -{\numexpression} were born. - -The resulting parsers worked quite well but were about twice as slow as the -normal expression scanners but that is no surprise because they do more. For -instance we are case insensitive and need to handle letter and other (and in a -few cases alignment and superscript) catcodes too. However, with a slightly tuned -integer parser, also possible because the sentinel \type {\relax} makes parsing -more predictable, and a dedicated unit scanner, in the end both the integer and -dimension parser were performing well. It's not like we run them millions of -times in a document. - -\startbuffer -\scratchcounter = \numexpression - "00000 bor "00001 bor "00020 bor "00400 bor "08000 bor "F0000 -\relax -\stopbuffer - -Here is an example that results in {0x\inlinebuffer\uchexnumber\scratchcounter}: - -\typebuffer - -\startbuffer -\scratchcounter = \numexpression - "FFFFF bxor "10101 -\relax -\stopbuffer - -And this gives {0x\inlinebuffer\uchexnumber\scratchcounter}: - -\typebuffer - -We can give numerous example but you get the picture. In the above table you can -see that some operators have equivalents. The reason for this is that a macro -package can change catcodes and some characters have special meanings. So, the -scanner is rather tolerant. - -\startbuffer -\scratchcounterone = 10 -\scratchcountertwo = 20 -\ifcase \numexpression - (\scratchcounterone > 5) && (\scratchcountertwo > 5) -\relax yes\else nop\fi -% -\space -% -\scratchcounterone = 2 -\scratchcountertwo = 4 -\ifcase \numexpression - (\scratchcounterone > 5) and (\scratchcountertwo > 5) -\relax nop\else yes\fi -\stopbuffer - -And this gives \quote {\tttf \inlinebuffer}: - -\typebuffer - -The normal expansion rules apply, so one can use macros and other symbolic -numbers. The only difference in handling dimensions is that we don't support -\type {true} units but these are obsolete in \LUAMETATEX\ anyway. - -In the end I decided to also add an extra conditional so that we can say: - -\starttyping -\ifexpression (\scratchcounterone > 5) and (\scratchcountertwo > 5)\relax - nop -\else - yes -\fi -\stoptyping - -which looks more natural. Actually, this is an nowadays alias because we have two -variants: - -\starttyping -\ifnumexpression ... \relax ... \else ... \fi -\ifdimexpression ... \relax ... \else ... \fi -\stoptyping - -where the later is equivalent to - -\starttyping -\ifboolean\dimexpression ... \relax ... \else ... \fi -\stoptyping - -\stopsection - -\startsection[title={Tracing}] - -When \type {\tracingexpressions} is set to one or higher the intermediate \quote -{reverse polish notation} stack that is used for the calculation is shown, for -instance: - -\starttyping -4:8: {numexpression rpn: 2 5 > 4 5 > and} -\stoptyping - -When you want the output on your console, you need to say: - -\starttyping -\tracingexpressions 1 -\tracingonline 1 -\stoptyping - -The fact that we process the expression in two phases makes it possible to provide this -kind of tracing. - -\stopsection - -\startsection[title={Performance}] - -The following table shows the results of 100.000 evaluations (per line) so you'll -notice that there is a difference. But keep in mind that the new variant can so -more, so it might pay off when we have cases that otherwise demand multiple -traditional expressions. - -\starttabulate[|l|c|] -\NC \type {\dimexpr 4pt*2 + 6pt\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchdimen \dimexpr 4pt*2 + 6pt\relax} \elapsedtime\fi \NC \NR -\NC \type {\dimexpression 4pt*2 + 6pt\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchdimen \dimexpression 4pt*2 + 6pt\relax} \elapsedtime\fi \NC \NR -\NC \type {\dimexpression 2*4pt + 6pt\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchdimen \dimexpression 4pt*2 + 6pt\relax} \elapsedtime\fi \NC \NR -\TB -\NC \type {\numexpr 4 * 2 + 6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr 4 * 2 + 6\relax} \elapsedtime\fi \NC \NR -\NC \type {\numexpression 2 * 4 + 6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression 2 * 4 + 6\relax} \elapsedtime\fi \NC \NR -\TB -\NC \type {\numexpr 4*2+6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr 4*2+6\relax} \elapsedtime\fi \NC \NR -\NC \type {\numexpression 2*4+6\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression 2*4+6\relax} \elapsedtime\fi \NC \NR -\TB -\NC \type {\numexpr (1+2)*(3+4)\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr (1+2)*(3+4)\relax} \elapsedtime\fi \NC \NR -\NC \type {\numexpression (1+2)*(3+4)\relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression (1+2)*(3+4)\relax} \elapsedtime\fi \NC \NR -\TB -\NC \type {\numexpr (1 + 2) * (3 + 4) \relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpr (1 + 2) * (3 + 4) \relax} \elapsedtime\fi \NC \NR -\NC \type {\numexpression (1 + 2) * (3 + 4) \relax} \EQ \iftrialtypesetting\else\testfeatureonce{100000}{\scratchcounter\numexpression (1 + 2) * (3 + 4) \relax} \elapsedtime\fi \NC \NR -\stoptabulate - -As usual I'll probably find some way to improve performance a bit but that might -than also concern the traditional one. When we compare them, the new numeric -scanner suffers from more options while the new dimension parser gain on the -units. Also, keep in mind than the \LUAMETATEX\ normal parsers are already -somewhat faster than the ones in \LUATEX. The numbers above are calculated when -this document is rendered, so they may change over time and per run. The two -engines compare as follows (mid 2021): - -\starttabulate[|l|c|c|] -\NC \BC \LUATEX \BC \LUAMETATEX \NC \NR -\NC \type {\dimexpr 4pt*2 + 6pt\relax} \NC 0.073 \NC 0.045 \NC \NR -\NC \type {\numexpr 4 * 2 + 6\relax} \NC 0.034 \NC 0.028 \NC \NR -\NC \type {\numexpr 4*2+6\relax} \NC 0.035 \NC 0.032 \NC \NR -\NC \type {\numexpr (1+2)*(3+4)\relax} \NC 0.050 \NC 0.047 \NC \NR -\NC \type {\numexpr (1 + 2) * (3 + 4) \relax} \NC 0.052 \NC 0.048 \NC \NR -\stoptabulate - -Of course tests like these are dubious because often \CPU\ cache will keep the -current code accessible, but who knows. - -It will probably take a while before I will use this in the source code because -first I need to make sure that all works as expected and while doing that I might -adapt some of this. But the basic framework is there. - -\stopsection - -% \start -% \nologbuffering -% \scratchdimen 100pt -% \scratchdimenone 65.536pt -% \scratchdimentwo 65.536bp - -% \tracingonline1 -% \tracingexpressions1 -% \scratchcounter\bitexpr \scratchdimen / 2 \relax\the\scratchcounter\par - -% \scratchcounter\numexpression \scratchdimen / 2sp \relax \the\scratchcounter\par -% \scratchcounter\numexpression \scratchdimen / 1pt \relax \the\scratchcounter\par -% \scratchcounter\numexpression \scratchdimenone / 65.536pt \relax \the\scratchcounter\par -% \scratchcounter\numexpression \scratchdimentwo / 2 \relax \the\scratchcounter\par - -% \scratchcounter\numexpression \scratchcounterone / 4 \relax \the\scratchcounter\par -% \scratchdimen \dimexpression \scratchcounterone / 4 \relax \the\scratchdimen\par - -% \scratchdimen \dimexpression 2 * 4pt \relax \the\scratchdimen\par - -% \tracingexpressions0 -% \tracingonline0 - -% \startTEXpage -% \tracingonline1 -% \tracingexpressions1 -% \the\dimexpr -10pt\relax\quad -% \the\dimexpr 10pt\relax\quad -% \the\dimexpr 10.12 pt\relax\quad -% \the\dimexpression -10pt\relax\quad -% \the\dimexpression 10pt\relax\quad -% \stopTEXpage - -\stopchapter - -\stopcomponent diff --git a/doc/context/sources/general/manuals/followingup/followingup-formats.tex b/doc/context/sources/general/manuals/followingup/followingup-formats.tex deleted file mode 100644 index 9b55c472a..000000000 --- a/doc/context/sources/general/manuals/followingup/followingup-formats.tex +++ /dev/null @@ -1,359 +0,0 @@ -% language=us runpath=texruns:manuals/followingup - -\environment followingup-style - -\startcomponent followingup-format - -\startchapter[title={The format file}] - -It is interesting when someone compares macro package and used parameters like -the size of a format file, the output of \type {\tracingall}, or startup time to -make some point. The point I want to make here is that unless you know exactly -what goes on in a run that involves a real document, which can itself involve -multiple runs, such a comparison is rather pointless. For sure I do benchmark but -I can only draw conclusions of what I (can) know (about). Yes, benchmarking your -own work makes sense but doing that in comparisons to what you consider -comparable variants assumes knowledge of more than your own work and objectives. - - -For instance, when you load few fonts, typeset one page and don't do anything -that demands any processing or multiple runs, you basically don't measure -anything. More interesting are the differences between 10 or 500 pages, a few -font calls or tens of thousands, no color of extensive usage of color and other -properties, interfacing, including inheritance of document constructs, etc. And -even then, when comparing macro packages, it is kind of tricky to deduce much -from what you observe. You really need to know what is going on inside and also -how that relates to for instance adaptive font scaling. You can have a fast start -up but if a users needs one tikz picture, loading that package alone will make -you forget the initial startup time. You always pay a price for advanced features -and integration! And we didn't even talk about the operating system caching -files, running on a network share, sharing processors among virtual machines, -etc. - -Pointless comparing is also true for looking at the log file when enabling \type -{\tracingall}. When a macro package loads stuff at startup you can be sure that -the log file is larger. When a font or language is loaded the first time, or -maybe when math is set up there can be plenty of lines dumped. Advanced analysis -of conditions and trial runs come at a price too. And eventually, when a box is -shown the configured depth and breadth really matter, and it might also be that -the engine provides much more (verbose) detail. So, a comparison is again -pointless. It can also backfire. Over the decades of developing \CONTEXT\ I have -heard people working on systems make claims like \quotation {We prefer not to -\unknown} or \quotation {It is better to it this way \unknown} or (often about -operating system) \quotation {It is bad that \unknown} just to see years later -the same being done in the presumed better alternative. I can have a good laugh -about that: do this and don't do that backfiring. - -That brings us to the format file. When you make a \CONTEXT\ format with the -English user interface, with interfacing being a feature that itself introduces -overhead, the \LUATEX\ engine will show this at the end: - -\starttyping -Beginning to dump on file cont-en.fmt - (format=cont-en 2021.6.9) -48605 strings using 784307 bytes -1050637 memory locations dumped; current usage is 414&523763 -44974 multiletter control sequences -\font\nullfont=nullfont -0 preloaded fonts -\stoptyping - -The file itself is quite large: 11,129,903 bytes. However, it is actually much -larger because the format file is compressed! The real size is 19.399.216. Not -taking that into account when comparing the size of format files is kind of bad -because compression directly relates to what resources a format use and how -usage is distributed over the available memory blobs. The \LUATEX\ engine does -some optimizations and saves the data sparse but the more holes you create, the -worse it gets. For instance, the large character vectors are compartmentalized in -order to handle \UNICODE\ efficiently so the used memory relates to what you -define: do you set up all catcodes or just a subset. Maybe you delay some -initialization to after the format is loaded, in which case a smaller format file -gets compensated by more memory usage and initializaton time afterwards. Maybe -your temporary macros create holes in the token array. The memory that is -configured in the configuration files also matter. Some memory blobs are saved at -their configured size, others dismiss the top part that is not used when saving -the format but allocate the lot when the format is loaded. That means that memory -usage in for instance \LUATEX\ can be much larger than a format file suggests. -Keep in mind that a format file is basically a memory dump. - -Now, how does \LUAMETATEX\ compare to \LUATEX. Again we will look at the size of -the format file, but you need to keep in mind that for various reasons the \LMTX\ -macros are somewhat more efficient than the \MKIV\ ones, in the meantime some new -mechanism were added, which adds more \TEX\ and \LUA\ code, but I still expect -(at least for now) a smaller format file. However when we create the format we -see this (reformatted): - -\starttyping -Dumping format 'cont-en.fmt 2021.6.9' in file 'cont-en.fmt': -tokenlist compacted from 489733 to 488204 entries, -1437 potentially aliased lua call/value entries, -max string length 69, 16 fingerprint -+ 16 engine + 28 preamble -+ 836326 stringpool -+ 10655 nodes + 3905660 tokens -+ 705300 equivalents -+ 23072 math codes + 493024 text codes -+ 38132 primitives + 497352 hashtable -+ 4 fonts + 10272 math + 1008 language + 180 insert -+ 10305643 bytecodes -+ 12 housekeeping = 16826700 total. -\stoptyping - -This looks quite different from the \LUATEX\ output. Here we report more detail: -for each blob we mention the number of bytes used. The final result is a file -that takes 16.826.700 bytes. That number should be compared with the 19.399.216 -for \LUATEX. So, we need less indeed. But, when we compress the \LUAMETATEX\ -format we get this: 5,913,932 which is much less than the 11,129,903 compressed -size that the \LUATEX\ engine makes of it. One reason for using level 3 zip -compression compression in \LUATEX\ is that (definitely when we started) it loads -faster. It adds to creating the dump but doesn't really influence loading, -although that depends a bit on the compiler used. It is not easy to see from -these numbers what goes on, but when you consider the fact that we mostly store -32 bit numbers it will also be clear that many can be zero or have two or three -zero bytes. There's a lot of repetition involved! - -So let's look at some of these numbers. The mentioning of token list compaction -relates to getting rid of holes in memory. Each token takes 8 bytes, 4 for the -token identifier, internally called a cmd and chr, and 4 for a value like an -integer or dimension value, or a glue pointer, or a pointer to a next token, etc. -In our case compaction doesn't save that much. - -The mentioning of potentially aliased \LUA\ call|/|value entries is more a warning. -Because the \LUA\ engine starts fresh each run, you cannot store its \quote -{pointers} and because hashes are randomized this means that you need to delay -initialization to startup time, definitely for function tokens. - -Strings in \TEX\ can be pretty long but in practice they aren't. In \CONTEXT\ the -maximum string length is 69. This makes it possible to use one byte for -registering the string length instead of four which saves quite a bit. Of course -one large string will spoil this game. - -The fingerprint, engine, preamble and later housekeeping bytes can be neglected -but the string pool not. These are the bytes that make up the strings. The bytes -are stored in format but when loaded become dynamically allocated. The \LUATEX\ -engine and its successor don't really have a pool. - -Now comes a confusing number. There are not tens of thousands of nodes allocated. -A node is just a pointer into a large array so actually node references are just -indices. Their size varies from 2 slots to 25; the largest are par nodes, while -shape nodes are allocated dynamically. So what gets reported are the number of -bytes that nodes take. each node slot takes 8 bytes, so a glyph node of 12 -bytes takes 96 bytes, while a glue spec node (think skip registers) takes 5 slots -or 40 bytes. These are amounts of memory that were not realistic when \TEX\ was -written. For the record: in \LUATEX\ glue spec nodes are not shared, so we have -many more. - -The majority of \TEX\ related dump data is for tokens, and here we need 3905660 -which means 488K tokens (each reported value also includes some overhead). The -memory used for the table of equivalents makes for some 88K of them. This table -relates to macros (their names and content). Keep in mind that (math) character -references are also macros. - -The next sections that get loaded are math and text codes. These are the -mentioned compartimized character properties. The number of math codes is not -that large (because we delay much of math) but the text codes are plenty, think -of lc, uc, sf, hj, catcodes, etc. Compared to \LUATEX\ we have more categories -but use less space because we have an more granular storage model. Optimizing -that bit really payed off, also because we have more vectors. - -The way primitives and macro names get resolved is pretty much the same in all -engines but by using the fact that we operate in 32 bit I could actually get rid -of some parallel tables that handle saving and restore. Some optimizations relate -to the fact that the register ranges are part of the game so basically we have -some holes in there when they are not used. I guess this is why \ETEX\ uses a -sparse model for the registers above 255. What also saved a lot is that we don't -need to store font names, because these are available in another way; even in -\LUATEX\ that takes a large, basically useless, chunk. The memory that a macro -without parameters consumes is 8 bytes smaller and in \CONTEXT\ we have lots of -these. -We don't really store fonts, so that section is small, but we do store the math -parameters, and there is not much we can save there. We also have more such -parameters in \LUAMETATEX\ so there we might actually use more storage. The -information related to languages is also minimal because patterns and exceptions -are loaded at runtime. A new category (compared to \LUATEX) is inserts because in -\LUAMETATEX\ we can use an alternative (not register based) variant. As you can -see from the 180 bytes used, indeed \CONTEXT\ is using that variant. - -That leaves a large block of more than 10 million bytes that relates to \LUA\ -byte code. A large part of that is the huge \LUA\ character table that \CONTEXT\ -uses. The implementation of font handling also takes quite a bit and we're not -even talking of all the auxiliary \LUA\ modules, \XML\ processing, etc. When -\CONTEXT\ would load that on demand, which is nearly always, the format file -would be much smaller but one would pay for it later. Loading the (some 600) -\LUA\ byte code chunks takes of course some time as does initialization but not -much. - -All that said, the reason why we have a large format file can be understood well -if one considers what goes in there. The \CONTEXT\ format files for \PDFTEX\ and -\XETEX\ are 3.3 and 4.7 MB each which is smaller but not that much when you -consider the fact that there is no \LUA\ code stored and that there are less -character tables and an \ETEX\ register model used. But a format file is not the -whole story. Runtime memory usage also comes at a price. - -The current memory settings of \CONTEXT\ are as follows; these values get -reported when a format has been generated and can be queried at runtime an any -moment: - -\starttabulate[|l|r|r|r|r|] -\BC \BC max \BC min \BC set \BC stp \BC \NR -\HL -\BC string \NC 2097152 \NC 150000 \NC 500000 \NC 100000 \NC \NR -\BC pool \NC 100000000 \NC 10000000 \NC 20000000 \NC 1000000 \NC \NR -\BC hash \NC 2097152 \NC 150000 \NC 250000 \NC 100000 \NC \NR -\BC lookup \NC 2097152 \NC 150000 \NC 250000 \NC 100000 \NC \NR -\BC node \NC 50000000 \NC 1000000 \NC 5000000 \NC 500000 \NC \NR -\BC token \NC 10000000 \NC 1000000 \NC 10000000 \NC 250000 \NC \NR -\BC buffer \NC 100000000 \NC 1000000 \NC 10000000 \NC 1000000 \NC \NR -\BC input \NC 100000 \NC 10000 \NC 100000 \NC 10000 \NC \NR -\BC file \NC 2000 \NC 500 \NC 2000 \NC 250 \NC \NR -\BC nest \NC 10000 \NC 1000 \NC 10000 \NC 1000 \NC \NR -\BC parameter \NC 100000 \NC 20000 \NC 100000 \NC 10000 \NC \NR -\BC save \NC 500000 \NC 100000 \NC 500000 \NC 10000 \NC \NR -\BC font \NC 100000 \NC 250 \NC 250 \NC 250 \NC \NR -\BC language \NC 10000 \NC 250 \NC 250 \NC 250 \NC \NR -\BC mark \NC 10000 \NC 50 \NC 50 \NC 50 \NC \NR -\BC insert \NC 500 \NC 10 \NC 10 \NC 10 \NC \NR -\stoptabulate - -The maxima is what can be used at most. Apart from the magic number 2097152 all -these maxima can be bumped at compile time but if you need more, you might wonder -of your approach to rendering makes sense. The minima are what always gets -allocated, and again these are hard coded defaults. The size can be configured -and is normally the same as the minima but we use larger values in \CONTEXT. The -step is how much an initial memory blob will grow when more is needed than is -currently available. The last four entries show that we don't start out with many -fonts (especially when we use the \CONTEXT\ compact font model not that many are -needed) and because \CONTEXT\ implements marks in a different way we actually -don't need them. We do use the new insert properties storage model and for now -the set sizes are enough for what we need. - -In practice a \LUAMETATEX\ run uses less memory than a \LUATEX\ one, not only -because memory allocation is more dynamic, but also because of other -optimizations. When the compact font model is used (something \CONTEXT) even less -memory is needed. Even this claim should be made with care. Whenever I discuss -the use of resources one needs to limit the conclusions to \CONTEXT. I can't -speak for other macro packages simply because I don't know the internals and the -design decisions made and their impact on the statistics. As a teaser I show the -impact of some definitions: - -\starttyping -\chardef \MyFooA1234 -\Umathchardef\MyFooB"1 "0 "1234 -\Umathcode 1 2 3 4 -\def \MyFooC{ABC} -\def \MyFooD#1{A#1C} -\def \MyFooE{\directlua{print("some lua")}} -\stoptyping - -The stringpool grows because we store the names (here they are of equal length). -Only symbolic definitions bump the hashtable and equivalents. And with -definitions that have text inside the number of bytes taken by tokens grows fast -because every character in that linked list takes 8 bytes, 4 for the character -with its catcode state and 4 for the link to the next token. - -\starttabulate[|l||||||] -\BC \BC stringpool \BC tokens \BC equivalents \BC hashtable \BC total \NC \NR -\HL -\NC \NC 836408 \NC 3906124 \NC 705316 \NC 497396 \NC 16828987 \NC \NR -\NC \type {\chardef} \NC 836415 \NC 3906116 \NC 705324 \NC 497408 \NC 16829006 \NC \NR -\NC \type {\Umathchardef} \NC 836422 \NC 3906116 \NC 705324 \NC 497420 \NC 16829025 \NC \NR -\NC \type {\Umathcode} \NC 836422 \NC 3906124 \NC 705324 \NC 497420 \NC 16829033 \NC \NR -\NC \type {\def} (no arg) \NC 836429 \NC 3906148 \NC 705332 \NC 497428 \NC 16829080 \NC \NR -\NC \type {\def} (arg) \NC 836436 \NC 3906196 \NC 705340 \NC 497440 \NC 16829155 \NC \NR -\NC \type {\def} (text) \NC 836443 \NC 3906372 \NC 705348 \NC 497452 \NC 16829358 \NC \NR -\stoptabulate - -So, every time a user wants some feature (some extra checking, a warning, color -or font support for some element) that results in a trivial extension to the -core, it can bump the size fo the format file more than you think. Of course when -it leads to some overhaul sharing code can actually make the format shrink too. I -hope it is clear now that there really is not much to deduce from the bare -numbers. Just try to imagine what: - -\starttyping -\definefilesynonym - [type-imp-newcomputermodern-book.mkiv] - [type-imp-newcomputermodern.mkiv] -\stoptyping - -adds to the format. Convenience has a price. - -\stopchapter - -\stopcomponent - -% Some bonus content: - -When processing thousand paragraphs \type {tufte.tex}, staying below 4 seconds -(just over 60 pages per second) all|-|in that looks ok. But it doesn't say that -much. Outputting 1000 pages in 2 seconds tells a bit about the overhead on a page -but again in practice things work out differently. So what do we need to -consider? - -\startitemize - -\startitem - Check what macros and resources are preloaded and what gets always loaded at - runtime. -\stopitem - -\startitem - After a first run it's likely that the operating system has resources in its - cache so start measuring after a few runs. -\stopitem - -\startitem - Best run a test many times and and take the average runtime. -\stopitem - -\startitem - Simple macro performance tests can be faster than in real usage because the - related bytes are in \CPU\ cache memory. So one can only use that to test a - specific improvement (or hit due to added functionality). -\stopitem - -\startitem - The size of the used \TEX\ tree can matter. The file databases need to be - loaded and consulted. -\stopitem - -\startitem - The binary matters: is it optimized, does it load libraries, is it 64 bit or not. -\stopitem - -\startitem - Local and|/|or global font definitions can hit performance and when a style - does many redundant switches it might hit performance. Of course that only is - the case when font switching is adaptive. -\stopitem - -\startitem - The granularity of subsystems impacts performance: advanced color support, - inheritance used in mechanisms, abstraction combined with extensive - support for features, it all matters. -\stopitem - -\startitem - The more features one enables the more it will impact performance as does - preprocessing the input (normalizing, bidi checking, etc). -\stopitem - -\startitem - It matters how the page (and layout) dimensions are defined. Although - language doesn't really play a role (apart from possible hyphenation) - specific scripts might. -\stopitem - -\stopitemize - -These are just a few points, but it might be clear that I don't take comparisons -too serious simply because it's real runs that matter. As long as we're in the -runtime comfort zone we're okay. You can run tests within the domain of a macro -package but comparing macro package makes not that much sense. It can even -backfire, especially when claims were made about what should be or not be in a -kernel (while later violating that) or relying on old stories (or rumors) about a -variant macro package being slow. (The same is true when comparing one's favorite -operating system.) Yes, the \CONTEXT\ format file is huge and performance less -than for instance plain \TEX. If that is a problem and not a virtue then make -sure your own alternative will never end up like that. And just don't come to -conclusions about a system that you don't really know. diff --git a/doc/context/sources/general/manuals/followingup/followingup.tex b/doc/context/sources/general/manuals/followingup/followingup.tex index 353b3ddd8..069cfa4eb 100644 --- a/doc/context/sources/general/manuals/followingup/followingup.tex +++ b/doc/context/sources/general/manuals/followingup/followingup.tex @@ -34,8 +34,6 @@ \component followingup-retrospect \component followingup-fonts \component followingup-memory - \component followingup-expressions - \component followingup-formats \stopbodymatter \stopdocument diff --git a/doc/context/sources/general/manuals/graphics/graphics.tex b/doc/context/sources/general/manuals/graphics/graphics.tex index 7c1000c4e..97785c170 100644 --- a/doc/context/sources/general/manuals/graphics/graphics.tex +++ b/doc/context/sources/general/manuals/graphics/graphics.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/graphics \usemodule[article-basic] \usemodule[abbreviations-smallcaps] diff --git a/doc/context/sources/general/manuals/lowlevel/lowlevel-boxes.tex b/doc/context/sources/general/manuals/lowlevel/lowlevel-boxes.tex index cf8b1a51f..8fbd796b4 100644 --- a/doc/context/sources/general/manuals/lowlevel/lowlevel-boxes.tex +++ b/doc/context/sources/general/manuals/lowlevel/lowlevel-boxes.tex @@ -694,3 +694,17 @@ test\quad \stopsection \stopdocument + +% todo: + +% \setbox0\hbox{\strut this is just a\footnote{oeps} test} +% \setbox2\hbox{\strut this is just a\footnote{oeps} test} +% +% \setprelistbox 0\hbox{\strut \quad before: \prelistbox0} +% \setpostlistbox0\hbox{\strut \quad after: \postlistbox0} +% +% \setprelistbox 2\hbox{\strut \quad before} +% \setpostlistbox2\hbox{\strut \quad after} +% +% test \par \box0 \par \box2 \par test + diff --git a/doc/context/sources/general/manuals/lowlevel/lowlevel-paragraphs.tex b/doc/context/sources/general/manuals/lowlevel/lowlevel-paragraphs.tex index 14c2e22a4..1c0264630 100644 --- a/doc/context/sources/general/manuals/lowlevel/lowlevel-paragraphs.tex +++ b/doc/context/sources/general/manuals/lowlevel/lowlevel-paragraphs.tex @@ -992,12 +992,46 @@ examples are visualized in \in {figure} [fig:flow]. \stopsection +\startsection[title=Dirty tricks] + +{\em todo: explain example for combining paragraphs} + +% test\wrapuppar{\crlf\ignorepars} +% +% test +% +% test\wrapuppar{\ignorepars} +% +% test +% +% test\wrapuppar{\removeunwantedspaces\ignorepars} +% +% test + +\stopsection + \stopdocument -\everyhbox \everyvbox : useless unless one resets -\parattr -\snapshotpar -\wrapuppar +% I rewarded myself after writing a section by watching the video "Final Thing On +% My Mind", The Pineapple This, Live, 2020, the usual perfect GH performance, +% wondering if live would turn to normal so that we could go to such concerts once +% again given successive covids. Writing manuals can do with a distraction. +% +% Gavin Harrison: Soundcheck, Drummerworld Jan 27, 2021 ... I wish I could make +% something called a check into pefect solo. Okay, another section and I'll check +% out the latest Simon Phillips and other favourite dummer uploads. + +% todo: + +% \everyhbox \everyvbox : useless unless one resets +% \parattr +% \snapshotpar +% \wrapuppar + +% \normalizelinemode0 +% x\ruledhbox{\parindent 1cm\indent x \indent x} +% \normalizelinemode \parindentskipnormalizecode +% x\ruledhbox{\parindent 1cm\indent x \indent x} % \starttext % \tracingoutput1 \tracingonline1 @@ -1012,14 +1046,3 @@ examples are visualized in \in {figure} [fig:flow]. % \par[newgraf][16=1,17=1], .... pretolerance 6000, .... % \parfillleftskip - -% I rewarded myself after writing a section by watching the video "Final Thing On -% My Mind", The Pineapple This, Live, 2020, the usual perfect GH performance, -% wondering if live would turn to normal so that we could go to such concerts once -% again given successive covids. Writing manuals can do with a distraction. -% -% Gavin Harrison: Soundcheck, Drummerworld Jan 27, 2021 ... I wish I could make -% something called a check into pefect solo. Okay, another section and I'll check -% out the latest Simon Phillips and other favourite dummer uploads. - - diff --git a/doc/context/sources/general/manuals/lua/lua-mkiv.tex b/doc/context/sources/general/manuals/lua/lua-mkiv.tex index 61d7dddad..2fd7e1573 100644 --- a/doc/context/sources/general/manuals/lua/lua-mkiv.tex +++ b/doc/context/sources/general/manuals/lua/lua-mkiv.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/lua % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex b/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex index cdea1c69b..d5b491e81 100644 --- a/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex +++ b/doc/context/sources/general/manuals/luametatex/luametatex-enhancements.tex @@ -1929,6 +1929,15 @@ kerns, should be handled in the input or macro package because there is no way w can predict the expected behaviour. In fact, the \lpr {linedir} is just a convenience extra which could also have been implemented using node list parsing. +Directions are complicated by the fact that they often need to work over groups +so a separate grouping related stack is used. A side effect is that there can be +paragraphs with only a local par node followed by direction synchronization +nodes. Paragraphs like that are seen as empty paragraphs and therefore ignored. +Because \type {\noindent} doesn't inject anything but a \type {\indent} injects +an box, paragraphs with only an indent and directions are handles and paragraphs +with content. When indentation is normalized a paragraph with an indentation +skip is seen as content. + \stopsubsection \startsubsection[title={Normalizing lines}] diff --git a/doc/context/sources/general/manuals/luatex/luatex-modifications.tex b/doc/context/sources/general/manuals/luatex/luatex-modifications.tex index 7cb14a18f..98f441c41 100644 --- a/doc/context/sources/general/manuals/luatex/luatex-modifications.tex +++ b/doc/context/sources/general/manuals/luatex/luatex-modifications.tex @@ -509,7 +509,7 @@ opened it up. \startsubsection[title=Changes from anywhere] -The \tyep {\partokenname} and \type {\partokencontext} primitives are taken from +The \type {\partokenname} and \type {\partokencontext} primitives are taken from the \PDFTEX\ change file posted on the implementers list. They are explained in the \PDFTEX\ manual and are classified as \ETEX\ extensions. @@ -1137,6 +1137,14 @@ kerns, should be handled in the input or macro package because there is no way w can predict the expected behaviour. In fact, the \lpr {linedir} is just a convenience extra which could also have been implemented using node list parsing. +Directions are complicated by the fact that they often need to work over groups +so a separate grouping related stack is used. A side effect is that there can be +paragraphs with only a local par node followed by direction synchronization +nodes. Paragraphs like that are seen as empty paragraphs and therefore ignored. +Because \type {\noindent} doesn't inject anything but a \type {\indent} injects +an box, paragraphs with only an indent and directions are handles ad paragraphs +with content. + \stopsubsection \startsubsection[title={Controlling glue with \lpr {breakafterdirmode}}] diff --git a/doc/context/sources/general/manuals/metafun/metafun-backgrounds.tex b/doc/context/sources/general/manuals/metafun/metafun-backgrounds.tex index 200620cae..02d1642ef 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-backgrounds.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-backgrounds.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-backpage.tex b/doc/context/sources/general/manuals/metafun/metafun-backpage.tex index 82931d756..5870bbf45 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-backpage.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-backpage.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-basics.tex b/doc/context/sources/general/manuals/metafun/metafun-basics.tex index 479f39602..5208d3ca0 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-basics.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-basics.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-colofon-paper.tex b/doc/context/sources/general/manuals/metafun/metafun-colofon-paper.tex index 1a611a325..d57e6a203 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-colofon-paper.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-colofon-paper.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-colofon-screen.tex b/doc/context/sources/general/manuals/metafun/metafun-colofon-screen.tex index 590c7725f..5df538d32 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-colofon-screen.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-colofon-screen.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-colofon.tex b/doc/context/sources/general/manuals/metafun/metafun-colofon.tex index b4c162a89..4396508b9 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-colofon.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-colofon.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-contents.tex b/doc/context/sources/general/manuals/metafun/metafun-contents.tex index 0767a0248..6a85aa364 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-contents.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-contents.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-conventions.tex b/doc/context/sources/general/manuals/metafun/metafun-conventions.tex index 6a6072c56..f951537c3 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-conventions.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-conventions.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-debugging.tex b/doc/context/sources/general/manuals/metafun/metafun-debugging.tex index f8957b5d8..383b01a77 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-debugging.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-debugging.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-document.tex b/doc/context/sources/general/manuals/metafun/metafun-document.tex index bb5540850..dc6ccb3aa 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-document.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-document.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-effects.tex b/doc/context/sources/general/manuals/metafun/metafun-effects.tex index 16fb4ef6a..f5b772ec2 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-effects.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-effects.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-embedding.tex b/doc/context/sources/general/manuals/metafun/metafun-embedding.tex index 10383fa3a..017dec2c9 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-embedding.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-embedding.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-environment-layout.tex b/doc/context/sources/general/manuals/metafun/metafun-environment-layout.tex index 8a30aad7c..b41cdf771 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-environment-layout.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-environment-layout.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/metafun + \startenvironment metafun-environment-layout \setupsystem diff --git a/doc/context/sources/general/manuals/metafun/metafun-environment-samples.tex b/doc/context/sources/general/manuals/metafun/metafun-environment-samples.tex index 5991a61d5..e9addbc5d 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-environment-samples.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-environment-samples.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/metafun + \startenvironment metafun-environment-sample \startuseMPgraphic{sample setup} diff --git a/doc/context/sources/general/manuals/metafun/metafun-environment-screen.tex b/doc/context/sources/general/manuals/metafun/metafun-environment-screen.tex index afeea7008..58ea5ccdb 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-environment-screen.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-environment-screen.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-environment.tex b/doc/context/sources/general/manuals/metafun/metafun-environment.tex index 57875008d..ebafc6182 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-environment.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-environment.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-examples.tex b/doc/context/sources/general/manuals/metafun/metafun-examples.tex index 5549a3bdd..3c16f96dd 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-examples.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-examples.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-functions.tex b/doc/context/sources/general/manuals/metafun/metafun-functions.tex index 780127408..4d6c47dde 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-functions.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-functions.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-gadgets.tex b/doc/context/sources/general/manuals/metafun/metafun-gadgets.tex index 014bd66e8..f4e887d0e 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-gadgets.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-gadgets.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-graphics.tex b/doc/context/sources/general/manuals/metafun/metafun-graphics.tex index 79c5df0df..ae2e66071 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-graphics.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-graphics.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-index.tex b/doc/context/sources/general/manuals/metafun/metafun-index.tex index 8be313d85..aab028d05 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-index.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-index.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-introduction.tex b/doc/context/sources/general/manuals/metafun/metafun-introduction.tex index e2c33f6f3..59aaf5fdc 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-introduction.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-introduction.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-layout.tex b/doc/context/sources/general/manuals/metafun/metafun-layout.tex index e81800bb4..91a4a3d71 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-layout.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-layout.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-lua.tex b/doc/context/sources/general/manuals/metafun/metafun-lua.tex index e445cef53..1fa399a74 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-lua.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-lua.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-macros.tex b/doc/context/sources/general/manuals/metafun/metafun-macros.tex index 969e7d1b1..cc9d2f05f 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-macros.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-macros.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-positioning.tex b/doc/context/sources/general/manuals/metafun/metafun-positioning.tex index 5a3b1ece0..81fcd171f 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-positioning.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-positioning.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-reference.tex b/doc/context/sources/general/manuals/metafun/metafun-reference.tex index bc59218b0..5e6b7eb77 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-reference.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-reference.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-sneaky.tex b/doc/context/sources/general/manuals/metafun/metafun-sneaky.tex index 56bf5d4d6..b02ce2dab 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-sneaky.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-sneaky.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-styles.tex b/doc/context/sources/general/manuals/metafun/metafun-styles.tex index 6de9a54da..21fa6009e 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-styles.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-styles.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-syntax.tex b/doc/context/sources/general/manuals/metafun/metafun-syntax.tex index 9407b1dd0..89f474418 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-syntax.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-syntax.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-text-lmtx.tex b/doc/context/sources/general/manuals/metafun/metafun-text-lmtx.tex index abdca3c3a..99f46a45b 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-text-lmtx.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-text-lmtx.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-text.tex b/doc/context/sources/general/manuals/metafun/metafun-text.tex index 8229e35db..2870fbb3b 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-text.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-text.tex @@ -1,4 +1,4 @@ -% engine=luatex language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun-titlepage-paper.tex b/doc/context/sources/general/manuals/metafun/metafun-titlepage-paper.tex index 558e3b798..8b5af6054 100644 --- a/doc/context/sources/general/manuals/metafun/metafun-titlepage-paper.tex +++ b/doc/context/sources/general/manuals/metafun/metafun-titlepage-paper.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/metafun.tex b/doc/context/sources/general/manuals/metafun/metafun.tex index 6739b1ed5..cbbc7e3ed 100644 --- a/doc/context/sources/general/manuals/metafun/metafun.tex +++ b/doc/context/sources/general/manuals/metafun/metafun.tex @@ -1,4 +1,4 @@ -% language=uk macros=mkvi +% language=us runpath=texruns:manuals/metafun macros=mkvi % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team diff --git a/doc/context/sources/general/manuals/metafun/mfun-700.tex b/doc/context/sources/general/manuals/metafun/mfun-700.tex index e393a34f3..02b9dadfe 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-700.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-700.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-771.tex b/doc/context/sources/general/manuals/metafun/mfun-771.tex index 460c16f2a..484c5949e 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-771.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-771.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-772.tex b/doc/context/sources/general/manuals/metafun/mfun-772.tex index 99dec1551..66ebaff91 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-772.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-772.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-773.tex b/doc/context/sources/general/manuals/metafun/mfun-773.tex index daefcfe66..f70213184 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-773.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-773.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-774.tex b/doc/context/sources/general/manuals/metafun/mfun-774.tex index 725d1199b..1b52069a8 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-774.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-774.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-775.tex b/doc/context/sources/general/manuals/metafun/mfun-775.tex index e9d2b85a7..23cb3257a 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-775.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-775.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-776.tex b/doc/context/sources/general/manuals/metafun/mfun-776.tex index 82b5c18dd..2591e2117 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-776.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-776.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-800.tex b/doc/context/sources/general/manuals/metafun/mfun-800.tex index aa0c1260f..0678fea70 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-800.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-800.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-900.tex b/doc/context/sources/general/manuals/metafun/mfun-900.tex index c295ec908..6a2e65ffb 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-900.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-900.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-901.tex b/doc/context/sources/general/manuals/metafun/mfun-901.tex index 23adf77b4..635f8c3e5 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-901.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-901.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/metafun/mfun-902.tex b/doc/context/sources/general/manuals/metafun/mfun-902.tex index 7c4fed73e..18d87f4ee 100644 --- a/doc/context/sources/general/manuals/metafun/mfun-902.tex +++ b/doc/context/sources/general/manuals/metafun/mfun-902.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/metafun % % copyright=pragma-ade readme=readme.pdf licence=cc-by-nc-sa diff --git a/doc/context/sources/general/manuals/notnow/notnow-columns-and-notes.tex b/doc/context/sources/general/manuals/notnow/notnow-columns-and-notes.tex index 336c39480..24290fc20 100644 --- a/doc/context/sources/general/manuals/notnow/notnow-columns-and-notes.tex +++ b/doc/context/sources/general/manuals/notnow/notnow-columns-and-notes.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/notnow + \usemodule[art-01] \starttext diff --git a/doc/context/sources/general/manuals/notnow/notnow-sidefloats.tex b/doc/context/sources/general/manuals/notnow/notnow-sidefloats.tex index c7de6c2c5..3c454d367 100644 --- a/doc/context/sources/general/manuals/notnow/notnow-sidefloats.tex +++ b/doc/context/sources/general/manuals/notnow/notnow-sidefloats.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/notnow + \usemodule[art-01] \definefloat diff --git a/doc/context/sources/general/manuals/notnow/notnow.tex b/doc/context/sources/general/manuals/notnow/notnow.tex index e1768db41..d50c10f10 100644 --- a/doc/context/sources/general/manuals/notnow/notnow.tex +++ b/doc/context/sources/general/manuals/notnow/notnow.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/notnow \usemodule[typesetting] @@ -128,6 +128,26 @@ already complex code more than needed. Also, it will never be perfect anyway. \stopchapter +\startchapter[title=Sidefloats] + +Support for side floats is non|-|trivial and no solution will serve all intended +usage. Over the years we have improved on border cases but it is still not +perfect. For that reason the implementation is (apart from solving bugs, mostly) frozen. +Here is an example of a use case that we ran into. We manipulate the spacing with +an offset parameter. + +\typefile{notnow-sidefloats.tex} + +You can best play with these parameters and see what they do. If you use this +mechanism in a long term project, use a frozen instance of \CONTEXT ! + +\TwoPages{notnow-sidefloats} + +The second pages has preceding and trailing whitespace outside the sidefloat +flow. + +\stopchapter + \startpagemakeup[pagestate=stop,page=left,doublesided=no] \scale[width=\paperwidth,height=\paperheight] \bgroup \bTABLE[strut=no,height=29.7pt,width=21pt,align={lohi,middle},foregroundcolor=white,framecolor=white,background=color] @@ -152,24 +172,4 @@ already complex code more than needed. Also, it will never be perfect anyway. \egroup \stoppagemakeup -\startchapter[title=Sidefloats] - -Support for side floats is non|-|trivial and no solution will serve all intended -usage. Over the years we have improved on border cases but it is still not -perfect. For that reason the implementation is (apart from solving bugs, mostly) frozen. -Here is an example of a use case that we ran into. We manipulate the spacing with -an offset parameter. - -\typefile{notnow-sidefloats.tex} - -You can best play with these parameters and see what they do. If you use this -mechanism in a long term project, use a frozen instance of \CONTEXT ! - -\TwoPages{notnow-sidefloats} - -The second pages has preceding and trailing whitespace outside the sidefloat -flow. - -\stopchapter - \stopdocument diff --git a/doc/context/sources/general/manuals/ontarget/ontarget-contents.tex b/doc/context/sources/general/manuals/ontarget/ontarget-contents.tex new file mode 100644 index 000000000..f0a733b01 --- /dev/null +++ b/doc/context/sources/general/manuals/ontarget/ontarget-contents.tex @@ -0,0 +1,13 @@ +% language=us runpath=texruns:manuals/ontarget + +\startcomponent ontarget-contents + +\environment ontarget-style + +\starttitle[title={Table of contents}] + + \placelist[chapter] + +\stoptitle + +\stopcomponent diff --git a/doc/context/sources/general/manuals/ontarget/ontarget-introduction.tex b/doc/context/sources/general/manuals/ontarget/ontarget-introduction.tex new file mode 100644 index 000000000..f7ac0a19d --- /dev/null +++ b/doc/context/sources/general/manuals/ontarget/ontarget-introduction.tex @@ -0,0 +1,104 @@ +% language=us runpath=texruns:manuals/ontarget + +\startcomponent ontarget-introduction + +\environment ontarget-style + +\startchapter[title={Introduction}] + +This is the seventh wrapup of the \LUATEX\ and \LUAMETATEX\ development cycle. It +is dedicated to all those users who kept up with developments and are always +willing to test the new features. Without them a project like this would not be +possible. + +At the time this introduction is written the \LUAMETATEX\ code base is rather +stable and quite a bit of the \MKIV\ code base has been adapted to new situation. +But, as usual, there are always new possibilities to explore, so I expect that +this document will grow over time as did the others. I'm not going to repeat all +that has been done because that's what the previous episodes are about. + +As the title suggest, we're still on target. When the \LUAMETATEX\ project +started there actually was no deadline formulated so in fact we're always on +target. The core components \TEX, \METAPOST, and \LUA\ are all long term efforts +so we're in no hurry at all. However, this is the year that a fast pace will +become a slow pace with respect to the \LUAMETATEX\ code base. There are still +some things on the agenda but in principle the goals are reached. One problem in +today's code development is that useability and quality seems to relate to the +amount of changes in code. No update can mean old, unusable and uninteresting. +It's probably why some sources get this silly yearly copyright year update. +However, the update cycle of good old \TEX\ has an decade interval by now while +it is still a pretty useable program. It would be nice to end up in such a long +term cycle with \LUAMETATEX: bug fixes only. + +Although \CONTEXT\ has always adapted early to new developments (color, graphics, +\PDF, \METAPOST, \ETEX, \PDFTEX, \LUATEX, \UTF, fonts) the effects on the +\CONTEXT\ code base are mostly hidden for users. There have been some changes +between \MKII\ and \MKIV, simply because there has been a shift from specific +eight bit encodings to \UTF\ and \TYPEONE\ to \OPENTYPE\ fonts. Both had an +impact on important subsystems: input encodings, font definitions and features, +language and script support. On other subsystems the impact was hardly +noticeable, like for instance backend related features (these have always been +kind of abstract). That doesn't mean that these haven't changed deep down, they +definitely have. Some mechanisms became better in \MKIV, simply because less +hackery was needed. My experience is that when users see that it gets better or +easier, they are also willing to adapt the few lines in their document source +that benefit from it. Of course the impact on the \METAPOST\ integration in +\CONTEXT\ had a real large impact, especially in terms of performance. + +The upgrade to \LMTX, the version of \CONTEXT\ for \LUAMETATEX, is even less +visible although already some new mechanisms showed up. This time a couple of +engine specific features have been improved and made more flexible. In fact, the +whole code base of the engine has been overhauled. This happened stepwise because +we had to make sure all things kept working. As a first step code was made +independent of the compilation infrastructure and the dependencies, other than a +very few small ones, have been removed. The result is a rather lean and mean +setup, even when we consider what has been added at the primitive level and +traditional subsystems. A benefit is that in the meantime the \LUAMETATEX\ \LMTX\ +combination outperforms \LUATEX\ with \MKIV, something that was not ensured when +the built|-|in \PDF\ backend was removed and delegated to \LUA. By binding +development closely to \CONTEXT\ we also hope that the code base stays clean of +arbitrary extensions. + +Because in the end, \TEX\ is also a programming language, there have been +extensions that make programming easier. There is already a stable middle layer +of auxiliary macros in \CONTEXT\ that help the user who likes to program but +doesn't like real low level primitives and dirty tricks, but by extending the +primitive repertoire a bit users can now stay closer to the original \TEX\ +concepts. Adding more and more layers of indirectness makes no sense if we can +improve the bottom programming layer. It also makes coding a bit more natural +(the \TEX\ look), apart from offering performance benefits. This is where you can +see differences between the \MKIV\ and \LMTX\ code base which for that reason is +now nearly split completely. The \METAPOST\ subsystem has been extended with +proper scanners so that we can enhance the interfaces in a natural way and as a +result we also have an upgraded code base there. We also moved to \LUA 5.4 and +will keep up as long as compatibility is no issue. Some \LUA\ code is likely to +remain common between \MKIV\ and \LMTX, for instance font handling and helpers +but we'll see where that ends. + +The \LUAMETATEX\ engine provides control over most internals and there are all +kind of new interesting features. Decades of \CONTEXT\ development are behind +that. Also, in the days that there were discussions about extending \TEX, +\CONTEXT\ was not that much of influence and on the road to and from user group +meetings, Taco and I often discussed what we'd like to see added (and some was +actually implemented in \type {eetex} but that only lived on our machines. One +can consider \LUATEX\ to be a follow up on that, and \LUAMETATEX\ in turn follows +up on that project, which we both liked doing a lot. In some way \LUATEX\ lowered +the boundary for implementing some of the more intrusive extensions in +\LUAMETATEX\ and the follow up on \MPLIB. And once you start along that road +small steps become large steps and one can as well be try to be as complete as +possible. We've come a long way but eventually arrived at the destination. +Personally I think we got there by not being in a hurry. + +But even targets that are reached can eventually move, + +\blank[2*big] + +\startlines +Hans Hagen +Hasselt NL +August 2021\high{++} +\stoplines + +\stopchapter + +\stopcomponent diff --git a/doc/context/sources/general/manuals/ontarget/ontarget-style.tex b/doc/context/sources/general/manuals/ontarget/ontarget-style.tex new file mode 100644 index 000000000..eac3ffe31 --- /dev/null +++ b/doc/context/sources/general/manuals/ontarget/ontarget-style.tex @@ -0,0 +1,68 @@ +% language=us runpath=texruns:manuals/ontarget + +\startenvironment ontarget-style + +\usemodule[abbreviations-smallcaps] +\usemodule[scite] + +\enableexperiments[fonts.compact] + +\logo [LUAMETATEX] {LuaMeta\TeXsuffix} + +\setupbodyfont[plex,10pt] % not that ok for titling + +\setuplayout + [width=middle, + height=middle, + header=0pt, + footer=1cm, + footerdistance=5mm, + backspace=2cm, + cutspace=15mm, + topspace=2cm, + bottomspace=1cm, + style=bold, + color=maincolor] + +\setuppagenumbering + [alternative=doublesided] + +\setupwhitespace + [big] + +\setupfootertexts + [][{\getmarking[chapter]\quad\pagenumber}] + [{\pagenumber\quad\getmarking[chapter]}][] + +\definecolor + [maincolor] + [darkred] + +\setuphead + [chapter] + [style=\bfc, + color=maincolor] + +\setuphead + [section] + [style=\bfa, + color=maincolor] + +\setuphead + [subsection] + [style=\bf, + color=maincolor] + +\setupalign + [tolerant,stretch] + +\setuptyping + [color=maincolor] + +\setuptype + [color=maincolor] + +\setupitemize + [color=maincolor] + +\stopenvironment diff --git a/doc/context/sources/general/manuals/ontarget/ontarget-titlepage.tex b/doc/context/sources/general/manuals/ontarget/ontarget-titlepage.tex new file mode 100644 index 000000000..516955f7e --- /dev/null +++ b/doc/context/sources/general/manuals/ontarget/ontarget-titlepage.tex @@ -0,0 +1,73 @@ +% language=us runpath=texruns:manuals/ontarget + +\startcomponent ontarget-titlepage + +\environment ontarget-style + +\startMPpage[pagestate=stop] + +StartPage ; + + picture p ; p := image ( + + numeric d ; d := 6.5 ; + + path c ; c := fullcircle scaled 40 shifted (2,7) ; + + path e ; e := + (0,0) -- (3,0) -- + (3,1) -- (1,1) -- + (1,2) -- (3,2) -- + (3,3) -- (1,3) -- + (1,4) -- (3,4) -- + (3,4) -- (3,4) -- + (3,5) -- (0,5) -- cycle ; + + path r ; r := ((0,0) -- (3,0) -- (3,5) -- (0,5)) -- reverse e -- cycle; + + path q ; q := fullsquare xyscaled (3,60) shifted (3/2,0) ; + + fill c withcolor "lightgray" ; + + fill q withpen pencircle scaled 1/5 withcolor "white" ; + + for i=-25 step 5 until 35 : + fill (if (i mod 10) = 0 : e else : r fi) shifted (0,i - d) withcolor "darkred" ; + endfor ; + + draw q withpen pencircle scaled 1/5 withcolor "darkgray" ; + + for i=-25 step 10 until 30 : + draw textext.lft("\tttf " & decimal ((i/10)+21.5)) ysized 3 + shifted (9,19 + i-6.5 - d) withcolor "darkgray" ; + endfor ; + + draw ((-25, 0) -- (25, 0)) shifted center c withpen pencircle scaled 2/10 withcolor "yellow" ; + draw (( 0,-25) -- ( 0,25)) shifted center c withpen pencircle scaled 2/10 withcolor "yellow" ; + + clip currentpicture to c ; + draw c withpen pencircle scaled 4/10 withcolor "yellow" ; + ) ; + + fill Page withcolor "darkred" ; + + p := p shifted - center p ; + + draw p scaled 6mm + shifted center Page + shifted (0mm,50mm) ; + + draw image (draw textext ("\tt\bf on\thinspace target") xsized (16cm) withcolor "yellow") + shifted center bottomboundary Page + shifted (0mm,45mm) ; + + draw image (draw textext ("\tttf luametatex & context lmtx") xsized (13cm) withcolor "yellow") + shifted center bottomboundary Page + shifted (0mm,20mm) ; + +StopPage ; + +\stopMPpage + +\stopcomponent + diff --git a/doc/context/sources/general/manuals/ontarget/ontarget.tex b/doc/context/sources/general/manuals/ontarget/ontarget.tex new file mode 100644 index 000000000..c66318030 --- /dev/null +++ b/doc/context/sources/general/manuals/ontarget/ontarget.tex @@ -0,0 +1,19 @@ +% language=us runpath=texruns:manuals/ontarget + +\environment ontarget-style + +\dontcomplain + +\startdocument + + \component ontarget-titlepage + + \startfrontmatter + \component ontarget-contents + \stopfrontmatter + + \startbodymatter + \component ontarget-introduction + \stopbodymatter + +\stopdocument diff --git a/doc/context/sources/general/manuals/primitives/primitives.tex b/doc/context/sources/general/manuals/primitives/primitives.tex index 30d3815bb..f75736be6 100644 --- a/doc/context/sources/general/manuals/primitives/primitives.tex +++ b/doc/context/sources/general/manuals/primitives/primitives.tex @@ -2583,7 +2583,7 @@ usual (in some cases). Because we mention some \type {def} and \type {let} primitives here, it makes sense to also mention a primitive that will swap two values (meanings). This one has to be used with care. Of course that what gets swapped has to be of the same -type (or at least similar enough to to cause issues). Registers for instance +type (or at least similar enough not to cause issues). Registers for instance store their values in the token, but as soon as we are dealing with token lists we also need to keep an eye on reference counting. So, to some extend this is an experimental feature. diff --git a/doc/context/sources/general/manuals/templates/templates-mkiv.tex b/doc/context/sources/general/manuals/templates/templates-mkiv.tex index e46b1de8a..822954316 100644 --- a/doc/context/sources/general/manuals/templates/templates-mkiv.tex +++ b/doc/context/sources/general/manuals/templates/templates-mkiv.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/templates % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team @@ -66,8 +66,8 @@ for j=1 upto m : fill unitsquare - xysized (w,h) - shifted ((i-1)*w,(j-1)*h) + xysized (w+1/5,h+1/5) % a little overlap to get rid of viewer inaccuracies + shifted ((i-1)*w-1/10,(j-1)*h-1/10) withcolor (.5[red,blue] randomized(.75,.75,.75)) ; endfor ; diff --git a/doc/context/sources/general/manuals/units/units-mkiv.tex b/doc/context/sources/general/manuals/units/units-mkiv.tex index 66338497a..34da0e123 100644 --- a/doc/context/sources/general/manuals/units/units-mkiv.tex +++ b/doc/context/sources/general/manuals/units/units-mkiv.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/units % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team diff --git a/doc/context/sources/general/manuals/workflows/workflows-contents.tex b/doc/context/sources/general/manuals/workflows/workflows-contents.tex index a32f4737d..0e3c18a38 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-contents.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-contents.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-graphics.tex b/doc/context/sources/general/manuals/workflows/workflows-graphics.tex index 2cff143f4..a24d293df 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-graphics.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-graphics.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-injectors.tex b/doc/context/sources/general/manuals/workflows/workflows-injectors.tex index 5bd9bf057..4b784f7cf 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-injectors.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-injectors.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-introduction.tex b/doc/context/sources/general/manuals/workflows/workflows-introduction.tex index a88640b27..a2d7c0cc0 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-introduction.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-introduction.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-mkiv.tex b/doc/context/sources/general/manuals/workflows/workflows-mkiv.tex index dffc7d65c..6b8b172b0 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-mkiv.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-mkiv.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/workflows + % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team % license : Creative Commons Attribution ShareAlike 4.0 International diff --git a/doc/context/sources/general/manuals/workflows/workflows-parallel.tex b/doc/context/sources/general/manuals/workflows/workflows-parallel.tex index 632d2e3e6..a82028de6 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-parallel.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-parallel.tex @@ -1,4 +1,4 @@ -% language=us +% language=us runpath=texruns:manuals/workflows \startcomponent workflows-parallel diff --git a/doc/context/sources/general/manuals/workflows/workflows-resources.tex b/doc/context/sources/general/manuals/workflows/workflows-resources.tex index cbed64864..41de6dc35 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-resources.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-resources.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-running.tex b/doc/context/sources/general/manuals/workflows/workflows-running.tex index 6d3e9f5c4..710714920 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-running.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-running.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-setups.tex b/doc/context/sources/general/manuals/workflows/workflows-setups.tex index e9d120f7b..9ecc46354 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-setups.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-setups.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-style.tex b/doc/context/sources/general/manuals/workflows/workflows-style.tex index f29129fcd..364755443 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-style.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-style.tex @@ -1,3 +1,5 @@ +% language=us runpath=texruns:manuals/workflows + \startenvironment workflows-style \usemodule diff --git a/doc/context/sources/general/manuals/workflows/workflows-suspects.tex b/doc/context/sources/general/manuals/workflows/workflows-suspects.tex index 621fd6f59..d2054bd1f 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-suspects.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-suspects.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-synctex.tex b/doc/context/sources/general/manuals/workflows/workflows-synctex.tex index 8f1fac632..bb2128da4 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-synctex.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-synctex.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-titlepage.tex b/doc/context/sources/general/manuals/workflows/workflows-titlepage.tex index f184152b3..00071a5eb 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-titlepage.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-titlepage.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/workflows/workflows-xml.tex b/doc/context/sources/general/manuals/workflows/workflows-xml.tex index 0f29f5f55..93b03bb7b 100644 --- a/doc/context/sources/general/manuals/workflows/workflows-xml.tex +++ b/doc/context/sources/general/manuals/workflows/workflows-xml.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/workflows \environment workflows-style diff --git a/doc/context/sources/general/manuals/xtables/xtables-mkiv.tex b/doc/context/sources/general/manuals/xtables/xtables-mkiv.tex index d79c14ace..f5a477372 100644 --- a/doc/context/sources/general/manuals/xtables/xtables-mkiv.tex +++ b/doc/context/sources/general/manuals/xtables/xtables-mkiv.tex @@ -1,4 +1,4 @@ -% language=uk +% language=us runpath=texruns:manuals/xtables % author : Hans Hagen % copyright : PRAGMA ADE & ConTeXt Development Team @@ -131,7 +131,7 @@ pages. \startxcell[settings] ... \stopxcell \stopxcellgroup \stopxrow - \startxrowgroup + \stopxrowgroup \stopxtablehead|next|body|foot \stopxtable \stoptyping @@ -954,7 +954,7 @@ The setups themselves are rather simple as we don't capture any attributes. We now process the example. Of course it will also work for files. \startbuffer - \xmlprocessbuffer{main}{test}{} +\xmlprocessbuffer{main}{test}{} \stopbuffer \typebuffer @@ -990,7 +990,6 @@ After that: \eTR \eTABLE \stopbuffer -\stopbuffer \typebuffer @@ -1614,6 +1613,8 @@ in the distribution. \typebuffer \getbuffer +\stopsection + \page % \ruledhbox{\getbuffer[normal,demo]} diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index 24ca54092..4ff589018 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2021.07.27 19:24} +\newcontextversion{2021.07.30 00:41} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index d4b21f8d2..ecb8d8069 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2021.07.27 19:24} +\edef\contextversion{2021.07.30 00:41} %D For those who want to use this: diff --git a/tex/context/base/mkii/mult-en.mkii b/tex/context/base/mkii/mult-en.mkii index a21723db8..5af7737d1 100644 --- a/tex/context/base/mkii/mult-en.mkii +++ b/tex/context/base/mkii/mult-en.mkii @@ -207,6 +207,8 @@ \setinterfacevariable{fixed}{fixed} \setinterfacevariable{flexible}{flexible} \setinterfacevariable{float}{float} +\setinterfacevariable{flushbackward}{flushbackward} +\setinterfacevariable{flushforward}{flushforward} \setinterfacevariable{flushinner}{flushinner} \setinterfacevariable{flushleft}{flushleft} \setinterfacevariable{flushouter}{flushouter} @@ -1236,6 +1238,7 @@ \setinterfaceconstant{splitoffset}{splitoffset} \setinterfaceconstant{spot}{spot} \setinterfaceconstant{stack}{stack} +\setinterfaceconstant{stacking}{stacking} \setinterfaceconstant{stackname}{stackname} \setinterfaceconstant{start}{start} \setinterfaceconstant{starter}{starter} @@ -1265,6 +1268,7 @@ \setinterfaceconstant{symbolset}{symbolset} \setinterfaceconstant{symcolor}{symcolor} \setinterfaceconstant{symstyle}{symstyle} +\setinterfaceconstant{synchronize}{synchronize} \setinterfaceconstant{synonym}{synonym} \setinterfaceconstant{synonymcolor}{synonymcolor} \setinterfaceconstant{synonymcommand}{synonymcommand} @@ -1363,6 +1367,7 @@ \setinterfaceelement{load}{load} \setinterfaceelement{local}{local} \setinterfaceelement{makeup}{makeup} +\setinterfaceelement{namedtyping}{namedtyping} \setinterfaceelement{next}{next} \setinterfaceelement{place}{place} \setinterfaceelement{previous}{previous} diff --git a/tex/context/base/mkii/mult-it.mkii b/tex/context/base/mkii/mult-it.mkii index 4ba837a13..d3b670a0a 100644 --- a/tex/context/base/mkii/mult-it.mkii +++ b/tex/context/base/mkii/mult-it.mkii @@ -1367,6 +1367,7 @@ \setinterfaceelement{load}{carica} \setinterfaceelement{local}{locale} \setinterfaceelement{makeup}{makeup} +\setinterfaceelement{namedtyping}{namedtyping} \setinterfaceelement{next}{successivo} \setinterfaceelement{place}{metti} \setinterfaceelement{previous}{precedente} diff --git a/tex/context/base/mkii/mult-nl.mkii b/tex/context/base/mkii/mult-nl.mkii index 78445f695..6996a3730 100644 --- a/tex/context/base/mkii/mult-nl.mkii +++ b/tex/context/base/mkii/mult-nl.mkii @@ -207,6 +207,8 @@ \setinterfacevariable{fixed}{vast} \setinterfacevariable{flexible}{flexibel} \setinterfacevariable{float}{blok} +\setinterfacevariable{flushbackward}{flushbackward} +\setinterfacevariable{flushforward}{flushforward} \setinterfacevariable{flushinner}{lijnbinnen} \setinterfacevariable{flushleft}{lijnlinks} \setinterfacevariable{flushouter}{lijnbuiten} @@ -1236,6 +1238,7 @@ \setinterfaceconstant{splitoffset}{splitsoffset} \setinterfaceconstant{spot}{spot} \setinterfaceconstant{stack}{stapel} +\setinterfaceconstant{stacking}{stapels} \setinterfaceconstant{stackname}{stapelnaam} \setinterfaceconstant{start}{start} \setinterfaceconstant{starter}{opener} @@ -1364,6 +1367,7 @@ \setinterfaceelement{load}{laad} \setinterfaceelement{local}{lokale} \setinterfaceelement{makeup}{opmaak} +\setinterfaceelement{namedtyping}{namedtyping} \setinterfaceelement{next}{volgende} \setinterfaceelement{place}{plaats} \setinterfaceelement{previous}{vorige} diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 48164c9fc..d454c8d15 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2021.07.27 19:24} +\newcontextversion{2021.07.30 00:41} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index b75a70285..32af5b99b 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -45,7 +45,7 @@ %D {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2021.07.27 19:24} +\edef\contextversion{2021.07.30 00:41} %D Kind of special: diff --git a/tex/context/base/mkiv/file-res.lua b/tex/context/base/mkiv/file-res.lua index 8b16ff7da..3ed7abc8d 100644 --- a/tex/context/base/mkiv/file-res.lua +++ b/tex/context/base/mkiv/file-res.lua @@ -64,7 +64,7 @@ local function readfilename(specification,backtrack,treetoo) local fname = names[i] for i=1,backtrack,1 do fname = "../" .. fname - local pname = path and (path .. "/" .. fname) or fname + local pname = path and (path ~= "") and (path .. "/" .. fname) or fname if isfile(pname) then if trace_files then report_files("found by backtracking: %s",pname) diff --git a/tex/context/base/mkiv/grph-fil.lua b/tex/context/base/mkiv/grph-fil.lua index e40420125..005657947 100644 --- a/tex/context/base/mkiv/grph-fil.lua +++ b/tex/context/base/mkiv/grph-fil.lua @@ -97,6 +97,7 @@ function jobfiles.run(name,action) -- can be anything but we assume it gets checked by the sandbox os.execute(action) elseif ta == "table" then + -- these paths will be ignored when they are not set local path = action.path local runpath = action.runpath action.path = environment.arguments.path diff --git a/tex/context/base/mkiv/lpdf-ano.lua b/tex/context/base/mkiv/lpdf-ano.lua index a120ff2f4..ebed9aec6 100644 --- a/tex/context/base/mkiv/lpdf-ano.lua +++ b/tex/context/base/mkiv/lpdf-ano.lua @@ -1215,24 +1215,24 @@ local pdf_stop = pdfconstant("Stop") local pdf_resume = pdfconstant("Resume") local pdf_pause = pdfconstant("Pause") -local function movie_or_sound(operation,arguments) +local function movie_or_sound(operation,what,arguments) arguments = (type(arguments) == "table" and arguments) or settings_to_array(arguments) return pdfdictionary { S = pdf_movie, - T = format("movie %s",arguments[1] or "noname"), + T = format("%s %s",what,arguments[1] or "noname"), Operation = operation, } end -function executers.startmovie (arguments) return movie_or_sound(pdf_start ,arguments) end -function executers.stopmovie (arguments) return movie_or_sound(pdf_stop ,arguments) end -function executers.resumemovie(arguments) return movie_or_sound(pdf_resume,arguments) end -function executers.pausemovie (arguments) return movie_or_sound(pdf_pause ,arguments) end +function executers.startmovie (arguments) return movie_or_sound(pdf_start ,"movie",arguments) end +function executers.stopmovie (arguments) return movie_or_sound(pdf_stop ,"movie",arguments) end +function executers.resumemovie(arguments) return movie_or_sound(pdf_resume,"movie",arguments) end +function executers.pausemovie (arguments) return movie_or_sound(pdf_pause ,"movie",arguments) end -function executers.startsound (arguments) return movie_or_sound(pdf_start ,arguments) end -function executers.stopsound (arguments) return movie_or_sound(pdf_stop ,arguments) end -function executers.resumesound(arguments) return movie_or_sound(pdf_resume,arguments) end -function executers.pausesound (arguments) return movie_or_sound(pdf_pause ,arguments) end +function executers.startsound (arguments) return movie_or_sound(pdf_start ,"sound",arguments) end +function executers.stopsound (arguments) return movie_or_sound(pdf_stop ,"sound",arguments) end +function executers.resumesound(arguments) return movie_or_sound(pdf_resume,"sound",arguments) end +function executers.pausesound (arguments) return movie_or_sound(pdf_pause ,"sound",arguments) end function specials.action(var) local operation = var.operation diff --git a/tex/context/base/mkiv/lpdf-mov.lua b/tex/context/base/mkiv/lpdf-mov.lua index a0f82b25b..0f0eefd2f 100644 --- a/tex/context/base/mkiv/lpdf-mov.lua +++ b/tex/context/base/mkiv/lpdf-mov.lua @@ -17,13 +17,16 @@ local pdfdictionary = lpdf.dictionary local pdfarray = lpdf.array local pdfborder = lpdf.border +-- We should actually make sure that inclusion only happens once. But this mechanism +-- is dropped in pdf anyway so it will go away (read: mapped onto the newer mechanisms). + function nodeinjections.insertmovie(specification) -- managed in figure inclusion: width, height, factor, repeat, controls, preview, label, foundname local width = specification.width local height = specification.height local factor = specification.factor or number.dimenfactors.bp local moviedict = pdfdictionary { - F = specification.foundname, + F = specification.foundname or specification.file, Aspect = pdfarray { factor * width, factor * height }, Poster = (specification.preview and true) or false, } @@ -36,7 +39,7 @@ function nodeinjections.insertmovie(specification) Subtype = pdfconstant("Movie"), Border = bs, C = bc, - T = format("movie %s",specification.label), + T = format("movie %s",specification.tag or specification.label), Movie = moviedict, A = controldict, } @@ -44,24 +47,23 @@ function nodeinjections.insertmovie(specification) end function nodeinjections.insertsound(specification) - -- managed in interaction: repeat, label, foundname - local soundclip = interactions.soundclips.soundclip(specification.label) - if soundclip then - local controldict = pdfdictionary { - Mode = (specification["repeat"] and pdfconstant("Repeat")) or nil - } - local sounddict = pdfdictionary { - F = soundclip.filename - } - local bs, bc = pdfborder() - local action = pdfdictionary { - Subtype = pdfconstant("Movie"), - Border = bs, - C = bc, - T = format("sound %s",specification.label), - Movie = sounddict, - A = controldict, + local controldict = nil + if specification["repeat"] then + controldict = pdfdictionary { + Mode = pdfconstant("Repeat") } - context(nodeinjections.annotation(0,0,0,action())) -- test: context(...) end + local sounddict = pdfdictionary { + F = specification.foundname or specification.file + } + local bs, bc = pdfborder() + local action = pdfdictionary { + Subtype = pdfconstant("Movie"), + Border = bs, + C = bc, + T = format("sound %s",specification.tag or specification.label), + Movie = sounddict, + A = controldict, + } + context(nodeinjections.annotation(0,0,0,action())) -- test: context(...) end diff --git a/tex/context/base/mkiv/mult-def.lua b/tex/context/base/mkiv/mult-def.lua index d720a5b21..d65c7bcf6 100644 --- a/tex/context/base/mkiv/mult-def.lua +++ b/tex/context/base/mkiv/mult-def.lua @@ -12884,6 +12884,9 @@ return { ["pe"]="آرایش", ["ro"]="makeup", }, + ["namedtyping"]={ + ["en"]="namedtyping", + }, ["next"]={ ["cs"]="dalsi", ["de"]="folgende", diff --git a/tex/context/base/mkiv/scrn-wid.lua b/tex/context/base/mkiv/scrn-wid.lua index 17c9ebddc..e8688e4b2 100644 --- a/tex/context/base/mkiv/scrn-wid.lua +++ b/tex/context/base/mkiv/scrn-wid.lua @@ -231,13 +231,15 @@ function soundclips.register(specification) end end -function soundclips.insert(tag) - local sc = soundclips[tag] - if not sc then - -- todo: message - return soundclips.register { tag = tag } - else - return sc +function soundclips.insert(specification) + local tag = specification.tag + if tag and tag ~= "" then + local sc = soundclips[tag] + if not sc then + -- todo: message + sc = soundclips.register { tag = tag } + end + nodeinjections.insertsound(sc) end end diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf index f258bcfba..4edea249a 100644 Binary files a/tex/context/base/mkiv/status-files.pdf and b/tex/context/base/mkiv/status-files.pdf differ diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf index 11b45b798..a3561d70c 100644 Binary files a/tex/context/base/mkiv/status-lua.pdf and b/tex/context/base/mkiv/status-lua.pdf differ diff --git a/tex/context/base/mkxl/buff-ver.mkxl b/tex/context/base/mkxl/buff-ver.mkxl index 65c919556..3cde23d5e 100644 --- a/tex/context/base/mkxl/buff-ver.mkxl +++ b/tex/context/base/mkxl/buff-ver.mkxl @@ -177,8 +177,8 @@ \installcommandhandler \??typing {typing} \??typing \appendtoks - \frozen\instance\setuevalue{\e!start\currenttyping}{\buff_verbatim_typing_start{\currenttyping}}% - \frozen\instance\setuevalue{\e!stop \currenttyping}{\buff_verbatim_typing_stop {\currenttyping}}% + \frozen\protected\instance\edefcsname\e!start\currenttyping\endcsname{\buff_verbatim_typing_start{\currenttyping}{\currenttyping}}% + \frozen\protected\instance\edefcsname\e!stop \currenttyping\endcsname{\buff_verbatim_typing_stop}% \ifempty\currenttypingparent \normalexpanded{\definelinenumbering[\currenttyping]}% \else @@ -452,34 +452,33 @@ \mutable\let\beginofverbatimlines\relax % hooks \mutable\let\endofverbatimlines \relax % hooks -\protected\def\buff_verbatim_typing_start#1% tricky non standard lookahead +\protected\def\buff_verbatim_typing_start#1#2% tricky non standard lookahead {\begingroup - \edef\currenttyping{#1}% + \edef\currenttyping {#1}% + \edef\currenttypingwrapper{#2}% \obeylines \futureexpandis[\buff_verbatim_typing_start_yes\buff_verbatim_typing_start_nop} -\def\buff_verbatim_typing_start_nop - {\typingparameter\c!before\relax - \startpacked[\v!blank]% - \buff_verbatim_setup_line_numbering - \buff_verbatim_initialize_typing_one - \buff_verbatim_setup_keep_together - \normalexpanded{\buff_verbatim_type_block{\e!start\currenttyping}{\e!stop\currenttyping}}} - \def\buff_verbatim_typing_start_yes[#1]% - {\typingparameter\c!before\relax - \startpacked[\v!blank]% - \ifcondition\validassignment{#1}% + {\ifcondition\validassignment{#1}% \setupcurrenttyping[#1]% \else \doif\v!continue{#1}{\lettypingparameter\c!continue\v!yes}% \fi + \typingparameter\c!before\relax % moved down + \startpacked[\v!blank]% % moved down \buff_verbatim_setup_line_numbering \buff_verbatim_initialize_typing_one \buff_verbatim_setup_keep_together - \normalexpanded{\buff_verbatim_type_block{\e!start\currenttyping}{\e!stop\currenttyping}}} + \normalexpanded{\buff_verbatim_type_block{\e!start\currenttypingwrapper}{\e!stop\currenttypingwrapper}}} -% \startnamedtyping[#1] +\def\buff_verbatim_typing_start_nop + {\typingparameter\c!before\relax + \startpacked[\v!blank]% + \buff_verbatim_setup_line_numbering + \buff_verbatim_initialize_typing_one + \buff_verbatim_setup_keep_together + \normalexpanded{\buff_verbatim_type_block{\e!start\currenttypingwrapper}{\e!stop \currenttypingwrapper}}} \protected\def\buff_verbatim_setup_keep_together {\edef\p_keeptogether{\typingparameter\c!keeptogether}% @@ -518,13 +517,18 @@ \dostoptagged \begincsname#2\endcsname} -\protected\def\buff_verbatim_typing_stop#1% hm, currenttyping +\protected\def\buff_verbatim_typing_stop {\stoppacked \typingparameter\c!after\relax \useindentnextparameter\typingparameter \endgroup \dorechecknextindentation} +\permanent\protected\def\startnamedtyping[#1]% + {\buff_verbatim_typing_start{#1}\e!namedtyping} + +\permanent\let\stopnamedtyping\buff_verbatim_typing_stop + %D Line numbering for files is combined with filtering, while display verbatim has %D the ability to continue. %D @@ -581,7 +585,7 @@ \mutable\let\currenttypingfile\empty \appendtoks - \frozen\instance\setuevalue{\e!type\currenttyping\v!file}{\typefile[\currenttyping]}% + \frozen\protected\instance\edefcsname\e!type\currenttyping\v!file\endcsname{\typefile[\currenttyping]}% \to \everydefinetyping \permanent\tolerant\protected\def\typefile[#1]#*[#2]#:#3% diff --git a/tex/context/base/mkxl/colo-ini.mkxl b/tex/context/base/mkxl/colo-ini.mkxl index cccbb218e..21f1e1174 100644 --- a/tex/context/base/mkxl/colo-ini.mkxl +++ b/tex/context/base/mkxl/colo-ini.mkxl @@ -331,6 +331,13 @@ \let\colo_helpers_show_message\showmessage \to \everyjob +% handled elsewhere but kind of a reference here: +% +% \def\colo_force_colormodel#1#2% +% {\setcolorsparameter\c!rgb {#1}% +% \setcolorsparameter\c!cmyk{#2}% +% \the\everysetupcolors} + %D We provide stacking independent of grouping. % the old one diff --git a/tex/context/base/mkxl/cont-new.mkxl b/tex/context/base/mkxl/cont-new.mkxl index f8a7700b1..3d35f23e0 100644 --- a/tex/context/base/mkxl/cont-new.mkxl +++ b/tex/context/base/mkxl/cont-new.mkxl @@ -13,7 +13,7 @@ % \normalend % uncomment this to get the real base runtime -\newcontextversion{2021.07.27 19:24} +\newcontextversion{2021.07.30 00:41} %D This file is loaded at runtime, thereby providing an excellent place for hacks, %D patches, extensions and new features. There can be local overloads in cont-loc diff --git a/tex/context/base/mkxl/context.mkxl b/tex/context/base/mkxl/context.mkxl index 01dc31fab..3a28a1a84 100644 --- a/tex/context/base/mkxl/context.mkxl +++ b/tex/context/base/mkxl/context.mkxl @@ -29,7 +29,7 @@ %D {YYYY.MM.DD HH:MM} format. \immutable\edef\contextformat {\jobname} -\immutable\edef\contextversion{2021.07.27 19:24} +\immutable\edef\contextversion{2021.07.30 00:41} %overloadmode 1 % check frozen / warning %overloadmode 2 % check frozen / error diff --git a/tex/context/base/mkxl/lang-ini.mkxl b/tex/context/base/mkxl/lang-ini.mkxl index 42d365ff8..336dd5d69 100644 --- a/tex/context/base/mkxl/lang-ini.mkxl +++ b/tex/context/base/mkxl/lang-ini.mkxl @@ -811,10 +811,14 @@ \protected\def\lang_basics_traced_discretionary#1#2#3#4% {\normaldiscretionary#1{\darkred#2}{\darkgreen#3}{\darkblue#4}} +\def\lang_basics_trace_discretionary_yes{\enforced\let\discretionary\traceddiscretionary} % indirect because of overload +\def\lang_basics_trace_discretionary_nop{\enforced\let\discretionary\normaldiscretionary} % indirect because of overload + + \installtextracker {discretionaries} - {\enforced\let\discretionary\traceddiscretionary} - {\enforced\let\discretionary\normaldiscretionary} + {\lang_basics_trace_discretionary_yes} + {\lang_basics_trace_discretionary_nop} \permanent\protected\def\samplediscretionary {\traceddiscretionary diff --git a/tex/context/base/mkxl/lang-rep.lmt b/tex/context/base/mkxl/lang-rep.lmt new file mode 100644 index 000000000..6139a03f7 --- /dev/null +++ b/tex/context/base/mkxl/lang-rep.lmt @@ -0,0 +1,467 @@ +if not modules then modules = { } end modules ['lang-rep'] = { + version = 1.001, + comment = "companion to lang-rep.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- A BachoTeX 2013 experiment, probably not that useful. Eventually I used a simpler +-- more generic example. I'm sure no one ever notices of even needs this code. +-- +-- As a follow up on a question by Alan about special treatment of dropped caps I wonder +-- if I can make this one more clever (probably in a few more dev steps). For instance +-- injecting nodes or replacing nodes. It's a prelude to a kind of lpeg for nodes, +-- although (given experiences so far) we don't really need that. After all, each problem +-- is somewhat unique. + +local type, tonumber, next = type, tonumber, next +local gmatch, gsub = string.gmatch, string.gsub +local utfbyte, utfsplit = utf.byte, utf.split +local P, C, U, Cc, Ct, Cs, lpegmatch = lpeg.P, lpeg.C, lpeg.patterns.utf8character, lpeg.Cc, lpeg.Ct, lpeg.Cs, lpeg.match +local find = string.find + +local zwnj = 0x200C +local grouped = P("{") * ( Ct((U/utfbyte-P("}"))^1) + Cc(false) ) * P("}")-- grouped +local splitter = Ct(( + #P("{") * ( + P("{}") / function() return zwnj end + + Ct(Cc("discretionary") * grouped * grouped * grouped) + + Ct(Cc("noligature") * grouped) + ) + + U/utfbyte + )^1) + +local stripper = P("{") * Cs((1-P(-2))^0) * P("}") * P(-1) + +local trace_replacements = false trackers.register("languages.replacements", function(v) trace_replacements = v end) +local trace_details = false trackers.register("languages.replacements.details", function(v) trace_details = v end) + +local report_replacement = logs.reporter("languages","replacements") + +local glyph_code = nodes.nodecodes.glyph +local glue_code = nodes.nodecodes.glue + +local spaceskip_code = nodes.gluecodes.spaceskip +local xspaceskip_code = nodes.gluecodes.xspaceskip + +local nuts = nodes.nuts + +local getnext = nuts.getnext +local getprev = nuts.getprev +local getattr = nuts.getattr +local getid = nuts.getid +local getsubtype = nuts.getsubtype +local getchar = nuts.getchar +local isglyph = nuts.isglyph + +local setlink = nuts.setlink +local setnext = nuts.setnext +local setprev = nuts.setprev +local setchar = nuts.setchar +local setattrlist = nuts.setattrlist +local setoptions = nuts.setoptions + +local glyphoptioncodes = tex.glyphoptioncodes +local norightligature_option = glyphoptioncodes.norightligature +local noleftligature_option = glyphoptioncodes.noleftligature + +local insertbefore = nuts.insertbefore +local insertafter = nuts.insertafter +local remove_node = nuts.remove +local copy_node = nuts.copy +local flushlist = nuts.flushlist + +local nodepool = nuts.pool +local new_disc = nodepool.disc + +local texsetattribute = tex.setattribute +local unsetvalue = attributes.unsetvalue + +local enableaction = nodes.tasks.enableaction + +local v_reset = interfaces.variables.reset + +local implement = interfaces.implement + +local processors = typesetters.processors +local splitprocessor = processors.split + +local replacements = languages.replacements or { } +languages.replacements = replacements + +local a_replacements = attributes.private("replacements") + +local lists = { } +local last = 0 +local trees = { } + +table.setmetatableindex(lists,function(lists,name) + last = last + 1 + local list = { } + local data = { name = name, list = list, attribute = last } + lists[last] = data + lists[name] = data + trees[last] = list + return data +end) + +lists[v_reset].attribute = unsetvalue -- so we discard 0 + +-- todo: glue kern attr + +local function add(root,word,replacement) + local processor, replacement = splitprocessor(replacement,true) -- no check + replacement = lpegmatch(stripper,replacement) or replacement + local list = utfsplit(word) -- ,true) + local size = #list + for i=1,size do + local l = utfbyte(list[i]) + if not root[l] then + root[l] = { } + end + if i == size then + local special = find(replacement,"{",1,true) + local newlist = lpegmatch(splitter,replacement) + root[l].final = { + word = word, + replacement = replacement, + processor = processor, + oldlength = size, + newcodes = newlist, + special = special, + } + end + root = root[l] + end +end + +function replacements.add(category,word,replacement) + local root = lists[category].list + if type(word) == "table" then + for word, replacement in next, word do + add(root,word,replacement) + end + else + add(root,word,replacement or "") + end +end + +-- local strip = lpeg.stripper("{}") + +function languages.replacements.addlist(category,list) + local root = lists[category].list + if type(list) == "string" then + for new in gmatch(list,"%S+") do + local old = gsub(new,"[{}]","") + -- local old = lpegmatch(strip,new) + add(root,old,new) + end + else + for i=1,#list do + local new = list[i] + local old = gsub(new,"[{}]","") + -- local old = lpegmatch(strip,new) + add(root,old,new) + end + end +end + +local function tonodes(list,template) + local head, current + for i=1,#list do + local new = copy_node(template) + setchar(new,list[i]) + if head then + head, current = insertafter(head,current,new) + else + head, current = new, new + end + end + return head +end + +local is_punctuation = characters.is_punctuation + +-- We can try to be clever and use the fact that there is no match to skip +-- over to the next word but it is gives fuzzy code so for now I removed +-- that optimization (when I really need a high performance version myself +-- I will look into it (but so far I never used this mechanism myself). +-- +-- We used to have the hit checker as function but is got messy when checks +-- for punctuation was added. + +local function replace(head,first,last,final,hasspace,overload) + local current = first + local prefirst = getprev(first) or head + local postlast = getnext(last) + local oldlength = final.oldlength + local newcodes = final.newcodes + local newlength = newcodes and #newcodes or 0 + if trace_replacements then + report_replacement("replacing word %a by %a",final.word,final.replacement) + end + if hasspace or final.special then + -- It's easier to delete and insert so we do just that. On the todo list is + -- turn injected spaces into glue but easier might be to let the char break + -- handler do that ... + local prev = getprev(current) + local next = getnext(last) + local list = current + setnext(last) + setlink(prev,next) + current = prev + if not current then + head = nil + end + local i = 1 + while i <= newlength do + local codes = newcodes[i] + if type(codes) == "table" then + local method = codes[1] + if method == "discretionary" then + local pre, post, replace = codes[2], codes[3], codes[4] + if pre then + pre = tonodes(pre,first) + end + if post then + post = tonodes(post,first) + end + if replace then + replace = tonodes(replace,first) + end + -- todo: also set attr + local new = new_disc(pre,post,replace) + setattrlist(new,first) + head, current = insertafter(head,current,new) + elseif method == "noligature" then + -- not that efficient to copy but ok for testing + local list = codes[2] + if list then + local n = #list + for i=1,n do + local new = copy_node(first) + setchar(new,list[i]) + if i == 1 then + setoptions(new,norightligature_option) + elseif i == n then + setoptions(new,glyphoptioncodes.noleftligature | norightligature_option) + else + setoptions(new,glyphoptioncodes.noleftligature) + end + head, current = insertafter(head,current,new) + end + else + -- local new = copy_node(first) + -- setchar(new,zwnj) + -- head, current = insertafter(head,current,new) + setoptions(current,norightligature_option) + end + else + report_replacement("unknown method %a",method or "?") + end + else + local new = copy_node(first) + setchar(new,codes) + head, current = insertafter(head,current,new) + end + i = i + 1 + end + flushlist(list) + elseif newlength == 0 then + -- we overload + elseif oldlength == newlength then + if final.word ~= final.replacement then + for i=1,newlength do + setchar(current,newcodes[i]) + current = getnext(current) + end + end + current = getnext(final) + elseif oldlength < newlength then + for i=1,newlength-oldlength do + local n = copy_node(current) + setchar(n,newcodes[i]) + head, current = insertbefore(head,current,n) + current = getnext(current) + end + for i=newlength-oldlength+1,newlength do + setchar(current,newcodes[i]) + current = getnext(current) + end + else + for i=1,oldlength-newlength do + head, current = remove_node(head,current,true) + end + for i=1,newlength do + setchar(current,newcodes[i]) + current = getnext(current) + end + end + if overload then + overload(final,getnext(prefirst),getprev(postlast)) + end + return head, postlast +end + +-- we handle just one space + +function replacements.handler(head) + local current = head + local overload = attributes.applyoverloads + local mode = false -- we're in word or punctuation mode + local wordstart = false + local wordend = false + local prevend = false + local prevfinal = false + local tree = false + local root = false + local hasspace = false + while current do + local id = getid(current) -- or use the char getter + if id == glyph_code then + local a = getattr(current,a_replacements) + if a then + -- we have a run + tree = trees[a] + if tree then + local char = getchar(current) + local punc = is_punctuation[char] + if mode == "punc" then + if not punc then + if root then + local final = root.final + if final then + head = replace(head,wordstart,wordend,final,hasspace,overload) + elseif prevfinal then + head = replace(head,wordstart,prevend,prevfinal,hasspace,overload) + end + prevfinal = false + root = false + end + mode = "word" + end + elseif mode == "word" then + if punc then + if root then + local final = root.final + if final then + head = replace(head,wordstart,wordend,final,hasspace,overload) + elseif prevfinal then + head = replace(head,wordstart,prevend,prevfinal,hasspace,overload) + end + prevfinal = false + root = false + end + mode = "punc" + end + else + mode = punc and "punc" or "word" + end + if root then + root = root[char] + if root then + wordend = current + end + else + if prevfinal then + head = replace(head,wordstart,prevend,prevfinal,hasspace,overload) + prevfinal = false + end + root = tree[char] + if root then + wordstart = current + wordend = current + prevend = false + hasspace = false + end + end + else + root= false + end + else + tree = false + end + current = getnext(current) + elseif root then + local final = root.final + if mode == "word" and id == glue_code then + local s = getsubtype(current) + if s == spaceskip_code or s == xspaceskip_code then + local r = root[32] -- maybe more types + if r then + if not prevend then + local f = root.final + if f then + prevend = wordend + prevfinal = f + end + end + wordend = current + root = r + hasspace = true + goto moveon + end + end + end + if final then + head, current = replace(head,wordstart,wordend,final,hasspace,overload) + elseif prevfinal then + head, current = replace(head,wordstart,prevend,prevfinal,hasspace,overload) + end + prevfinal = false + root = false + ::moveon:: + current = getnext(current) + else + current = getnext(current) + end + end + if root then + local final = root.final + if final then + head = replace(head,wordstart,wordend,final,hasspace,overload) + elseif prevfinal then + head = replace(head,wordstart,prevend,prevfinal,hasspace,overload) + end + end + return head +end + +local enabled = false + +function replacements.set(n) + if n == v_reset then + n = unsetvalue + else + n = lists[n].attribute + if not enabled then + enableaction("processors","languages.replacements.handler") + if trace_replacements then + report_replacement("enabling replacement handler") + end + enabled = true + end + end + texsetattribute(a_replacements,n) +end + +-- interface + +implement { + name = "setreplacements", + actions = replacements.set, + arguments = "string" +} + +implement { + name = "addreplacements", + actions = replacements.add, + arguments = "3 strings", +} + +implement { + name = "addreplacementslist", + actions = replacements.addlist, + arguments = "2 strings", +} diff --git a/tex/context/base/mkxl/lang-rep.mkxl b/tex/context/base/mkxl/lang-rep.mkxl index 36e484242..03b761bb7 100644 --- a/tex/context/base/mkxl/lang-rep.mkxl +++ b/tex/context/base/mkxl/lang-rep.mkxl @@ -19,7 +19,7 @@ \unprotect -\registerctxluafile{lang-rep}{} +\registerctxluafile{lang-rep}{autosuffix} \definesystemattribute[replacements][public,global] diff --git a/tex/context/base/mkxl/lpdf-ano.lmt b/tex/context/base/mkxl/lpdf-ano.lmt index 5498627b8..485e143a7 100644 --- a/tex/context/base/mkxl/lpdf-ano.lmt +++ b/tex/context/base/mkxl/lpdf-ano.lmt @@ -754,6 +754,8 @@ local nofused = 0 local nofspecial = 0 local share = true +local refobjects = { } + local f_annot = formatters["<< /Type /Annot %s /Rect [ %.6N %.6N %.6N %.6N ] >>"] local f_quadp = formatters["<< /Type /Annot %s /QuadPoints [ %s ] /Rect [ %.6N %.6N %.6N %.6N ] >>"] @@ -803,8 +805,18 @@ local function finishreference(specification) else specifier = f_annot(prerolled,llx,lly,urx,ury) end + local objref = hashed[specifier] + local refatt = specification.reference + specification.objref = objref nofused = nofused + 1 - return pdfregisterannotation(hashed[specifier]) + if refatt then + refobjects[refatt] = objref + end + return pdfregisterannotation(objref) +end + +function codeinjections.getrefobj(refatt) -- bad name but experiment anyway + return refobjects[refatt] end local function finishannotation(specification) @@ -818,18 +830,20 @@ local function finishannotation(specification) pdfdelayedobject(annot,objref) else objref = pdfdelayedobject(annot) + specification.objref = objref end nofspecial = nofspecial + 1 return pdfregisterannotation(objref) end -function nodeinjections.reference(width,height,depth,prerolled,mesh) +function nodeinjections.reference(reference,width,height,depth,prerolled,mesh) if prerolled then if trace_references then report_references("link: width %p, height %p, depth %p, prerolled %a",width,height,depth,prerolled) end return new_latelua { action = finishreference, + reference = reference, width = width, height = height, depth = depth, @@ -868,6 +882,7 @@ pdfregisterannotation = function(n) else annotations = pdfarray { pdfreference(n) } -- no need to use lpdf.array cum suis end + return n end lpdf.registerannotation = pdfregisterannotation @@ -1219,24 +1234,24 @@ local pdf_stop = pdfconstant("Stop") local pdf_resume = pdfconstant("Resume") local pdf_pause = pdfconstant("Pause") -local function movie_or_sound(operation,arguments) +local function movie_or_sound(operation,what,arguments) arguments = (type(arguments) == "table" and arguments) or settings_to_array(arguments) return pdfdictionary { S = pdf_movie, - T = format("movie %s",arguments[1] or "noname"), + T = format("%s %s",what,arguments[1] or "noname"), Operation = operation, } end -function executers.startmovie (arguments) return movie_or_sound(pdf_start ,arguments) end -function executers.stopmovie (arguments) return movie_or_sound(pdf_stop ,arguments) end -function executers.resumemovie(arguments) return movie_or_sound(pdf_resume,arguments) end -function executers.pausemovie (arguments) return movie_or_sound(pdf_pause ,arguments) end +function executers.startmovie (arguments) return movie_or_sound(pdf_start ,"movie",arguments) end +function executers.stopmovie (arguments) return movie_or_sound(pdf_stop ,"movie",arguments) end +function executers.resumemovie(arguments) return movie_or_sound(pdf_resume,"movie",arguments) end +function executers.pausemovie (arguments) return movie_or_sound(pdf_pause ,"movie",arguments) end -function executers.startsound (arguments) return movie_or_sound(pdf_start ,arguments) end -function executers.stopsound (arguments) return movie_or_sound(pdf_stop ,arguments) end -function executers.resumesound(arguments) return movie_or_sound(pdf_resume,arguments) end -function executers.pausesound (arguments) return movie_or_sound(pdf_pause ,arguments) end +function executers.startsound (arguments) return movie_or_sound(pdf_start ,"sound",arguments) end +function executers.stopsound (arguments) return movie_or_sound(pdf_stop ,"sound",arguments) end +function executers.resumesound(arguments) return movie_or_sound(pdf_resume,"sound",arguments) end +function executers.pausesound (arguments) return movie_or_sound(pdf_pause ,"sound",arguments) end function specials.action(var) local operation = var.operation diff --git a/tex/context/base/mkxl/lpdf-emb.lmt b/tex/context/base/mkxl/lpdf-emb.lmt index f9fb7d3c2..daa7e41a1 100644 --- a/tex/context/base/mkxl/lpdf-emb.lmt +++ b/tex/context/base/mkxl/lpdf-emb.lmt @@ -279,6 +279,12 @@ end includecidset = false forcecidset = false -- for private testing only + function lpdf.setincludecidset(v) + -- incredible ... it's obselete, no viewer needs it so why still this crap + -- so in the end it had to be introduced again + includecidset = v + end + directives.register("backend.pdf.forcecidset",function(v) forcecidset = v end) @@ -873,7 +879,7 @@ do StemV = scale(stemv), XHeight = scale(xheight), FontFile2 = pdfreference(pdfflushstreamobject(fontdata)), --- CIDSet = tocidset, + CIDSet = tocidset, -- no longer needed but verifyers want it Metadata = fontmeta and pdfreference(pdfflushstreamobject(fontmeta)) or nil, } local parent = pdfdictionary { @@ -1827,13 +1833,17 @@ do elseif kind == "user" then return setmetatableindex(function(t,k) local list = data[k] - local kind = list.kind local v - if kind == "values" then - local d = list.data - v = direct(d.r or 0,d.g or 0,d.b or 0) - elseif kind == "attributes" then - v = indirect(list.color,list.transparency) + if list then + local kind = list.kind + if kind == "values" then + local d = list.data + v = direct(d.r or 0,d.g or 0,d.b or 0) + elseif kind == "attributes" then + v = indirect(list.color,list.transparency) + else + v = false -- textcolor + end else v = false -- textcolor end diff --git a/tex/context/base/mkxl/lpdf-fmt.lmt b/tex/context/base/mkxl/lpdf-fmt.lmt index 23534c518..f83432d48 100644 --- a/tex/context/base/mkxl/lpdf-fmt.lmt +++ b/tex/context/base/mkxl/lpdf-fmt.lmt @@ -912,6 +912,11 @@ function codeinjections.setformat(s) -- cmyk = spec.cmyk_colors and variables.yes or variables.no, -- rgb = spec.rgb_colors and variables.yes or variables.no, -- } + -- + local rgb = spec.rgb_colors and variables.yes or variables.no + local cmy = spec.cmyk_colors and variables.yes or variables.no + report_backend("permitted colorspaces: rgb %a, cmyk %a",rgb,cmy) + -- token.expandmacro ("colo_force_colormodel",true,rgb,true,cmy) -- colors.forcesupport( spec.gray_scale or false, @@ -952,6 +957,8 @@ function codeinjections.setformat(s) report_backend("error, format is already set to %a, ignoring %a",formatname,noname.format) end end + -- we could just pass the spec to the backend + lpdf.setincludecidset(spec.include_cidsets) else report_backend("error, format %a is not supported",format) end diff --git a/tex/context/base/mkxl/lpdf-lmt.lmt b/tex/context/base/mkxl/lpdf-lmt.lmt index f3eea58b4..f722a61e0 100644 --- a/tex/context/base/mkxl/lpdf-lmt.lmt +++ b/tex/context/base/mkxl/lpdf-lmt.lmt @@ -1196,7 +1196,7 @@ local flushimage do BBox = pdfarray(bbox), Resources = lpdf.collectedresources { serialize = false }, } - local objnum = pdfflushstreamobject(content,wrapper,false) + local objnum = pdfflushstreamobject(content,wrapper,false) -- why not compressed ? groups = groups + 1 usedxgroups[groups] = objnum return f_gr(groups) @@ -1770,7 +1770,7 @@ local finalize do boundingbox[4] * bpfactor, } - local contentsobj = pdfflushstreamobject(content,false,false) + local contentsobj = pdfflushstreamobject(content,false,true) pageattributes.Type = pdf_page pageattributes.Contents = pdfreference(contentsobj) @@ -1861,7 +1861,7 @@ local finalize do wrapper.Resources = next(boxresources) and boxresources or nil wrapper.ProcSet = lpdf.procset() - pdfflushstreamobject(content,wrapper,false,specification.objnum) + pdfflushstreamobject(content,wrapper,true,specification.objnum) end @@ -2164,7 +2164,7 @@ do local kind = compressed == "raw" and "raw" or "stream" local nolength = nil if compressed == "raw" then - compressed = nil + compressed = false nolength = true -- data = string.formatters["<< %s >>stream\n%s\nendstream"](attr,data) end @@ -2172,7 +2172,8 @@ do objnum = objnum, immediate = true, nolength = nolength, - compresslevel = compressed == false and 0 or nil, +-- compresslevel = compressed == false and 0 or nil, + compresslevel = compressed, type = "stream", string = data, attr = (dtype == "string" and dict) or (dtype == "table" and dict()) or nil, @@ -2187,7 +2188,8 @@ do return pdfdeferredobject { objnum = objnum, immediate = true, - compresslevel = compressed == false and 0 or nil, +-- compresslevel = compressed == false and 0 or nil, + compresslevel = compressed, type = "stream", file = filename, attr = (dtype == "string" and dict) or (dtype == "table" and dict()) or nil, @@ -2279,25 +2281,30 @@ local function flushstreamobj(data,n,dict,comp,nolength) n = nofobjects end local size = #data + if comp ~= false then + comp = compress and size > threshold + end if level == 0 then local b = nil local e = s_stream_e if nolength then - b = f_stream_b_d_r(n,dict) + b = f_stream_b_d_r(n,dict) -- raw object, already treated else - local done = false - if comp ~= false and compress and size > threshold then + if comp then local compdata = compressdata(data,size) if compdata then local compsize = #compdata if compsize <= size - threshold then data = compdata size = compsize - done = true + else + comp = false end + else + comp = false end end - if done then + if comp then b = dict and f_stream_b_d_c(n,dict,size) or f_stream_b_n_c(n,size) else b = dict and f_stream_b_d_u(n,dict,size) or f_stream_b_n_u(n,size) @@ -2310,21 +2317,23 @@ local function flushstreamobj(data,n,dict,comp,nolength) offset = offset + #b + size + #e else if nolength then - data = f_stream_d_r(n,dict,data) + data = f_stream_d_r(n,dict,data) -- raw object, already treated else - local done = false - if comp ~= false and compress and size > threshold then + if comp then local compdata = compressdata(data,size) if compdata then local compsize = #compdata if compsize <= size - threshold then data = compdata size = compsize - done = true + else + comp = false end + else + comp = false end end - if done then + if comp then data = dict and f_stream_d_c(n,dict,size,data) or f_stream_n_c(n,size,data) else data = dict and f_stream_d_u(n,dict,size,data) or f_stream_n_u(n,size,data) @@ -2417,7 +2426,8 @@ pdfimmediateobject = function(a,b,c,d) end -- todo: immediate if kind == "stream" then - flushstreamobj(data,objnum,attr,compresslevel and compresslevel > 0 or nil,nolength) +-- flushstreamobj(data,objnum,attr,compresslevel and compresslevel > 0 or nil,nolength) -- nil == auto + flushstreamobj(data,objnum,attr,compresslevel,nolength) -- nil == auto elseif objectstream and objcompression ~= false then addtocache(objnum,data) else @@ -2815,8 +2825,8 @@ do specification.type = kind specification.kind = kind end - local compress = compresslevel and compresslevel > 0 or nil - flushstreamobj(stream,objnum,dict,compress,nolength) +-- flushstreamobj(stream,objnum,dict,compresslevel and compresslevel > 0 or nil,nolength) + flushstreamobj(stream,objnum,dict,compresslevel,nolength) specification.objnum = objnum specification.rotation = specification.rotation or 0 specification.orientation = specification.orientation or 0 diff --git a/tex/context/base/mkxl/lpdf-mov.lmt b/tex/context/base/mkxl/lpdf-mov.lmt index a0f82b25b..0f0eefd2f 100644 --- a/tex/context/base/mkxl/lpdf-mov.lmt +++ b/tex/context/base/mkxl/lpdf-mov.lmt @@ -17,13 +17,16 @@ local pdfdictionary = lpdf.dictionary local pdfarray = lpdf.array local pdfborder = lpdf.border +-- We should actually make sure that inclusion only happens once. But this mechanism +-- is dropped in pdf anyway so it will go away (read: mapped onto the newer mechanisms). + function nodeinjections.insertmovie(specification) -- managed in figure inclusion: width, height, factor, repeat, controls, preview, label, foundname local width = specification.width local height = specification.height local factor = specification.factor or number.dimenfactors.bp local moviedict = pdfdictionary { - F = specification.foundname, + F = specification.foundname or specification.file, Aspect = pdfarray { factor * width, factor * height }, Poster = (specification.preview and true) or false, } @@ -36,7 +39,7 @@ function nodeinjections.insertmovie(specification) Subtype = pdfconstant("Movie"), Border = bs, C = bc, - T = format("movie %s",specification.label), + T = format("movie %s",specification.tag or specification.label), Movie = moviedict, A = controldict, } @@ -44,24 +47,23 @@ function nodeinjections.insertmovie(specification) end function nodeinjections.insertsound(specification) - -- managed in interaction: repeat, label, foundname - local soundclip = interactions.soundclips.soundclip(specification.label) - if soundclip then - local controldict = pdfdictionary { - Mode = (specification["repeat"] and pdfconstant("Repeat")) or nil - } - local sounddict = pdfdictionary { - F = soundclip.filename - } - local bs, bc = pdfborder() - local action = pdfdictionary { - Subtype = pdfconstant("Movie"), - Border = bs, - C = bc, - T = format("sound %s",specification.label), - Movie = sounddict, - A = controldict, + local controldict = nil + if specification["repeat"] then + controldict = pdfdictionary { + Mode = pdfconstant("Repeat") } - context(nodeinjections.annotation(0,0,0,action())) -- test: context(...) end + local sounddict = pdfdictionary { + F = specification.foundname or specification.file + } + local bs, bc = pdfborder() + local action = pdfdictionary { + Subtype = pdfconstant("Movie"), + Border = bs, + C = bc, + T = format("sound %s",specification.tag or specification.label), + Movie = sounddict, + A = controldict, + } + context(nodeinjections.annotation(0,0,0,action())) -- test: context(...) end diff --git a/tex/context/base/mkxl/lpdf-tag.lmt b/tex/context/base/mkxl/lpdf-tag.lmt index 8c76ca694..3ff058a4e 100644 --- a/tex/context/base/mkxl/lpdf-tag.lmt +++ b/tex/context/base/mkxl/lpdf-tag.lmt @@ -180,7 +180,20 @@ local function finishstructure() end -- for fulltag, element in sortedhash(elements) do -- sorting is easier on comparing pdf - pdfflushobject(element.knum,element.kids) + local kids = element.kids + -- if element.tag == "link" then + -- local d = kids[2] + -- if type(d) == "table" then + -- local refatt = element.refatt + -- if refatt then + -- local refobj = codeinjections.getrefobj(refatt) + -- if refobj then + -- d.Obj = pdfreference(refobj) + -- end + -- end + -- end + -- end + pdfflushobject(element.knum,kids) end end end @@ -192,6 +205,7 @@ local index, pageref, pagenum, list = 0, nil, 0, nil local pdf_mcr = pdfconstant("MCR") local pdf_struct_element = pdfconstant("StructElem") local pdf_s = pdfconstant("S") +local pdf_objr = pdfconstant("OBJR") local function initializepage() index = 0 @@ -251,7 +265,7 @@ local function makeelement(fulltag,parent) local attributes = nil if tagname == "ignore" then return false - elseif tagname == "mstackertop" or tagname == "mstackerbot" or tagname == "mstackermid"then + elseif tagname == "mstackertop" or tagname == "mstackerbot" or tagname == "mstackermid" then -- TODO return true elseif tagname == "tabulatecell" then @@ -338,7 +352,10 @@ end local f_BDC = formatters["/%s <> BDC"] -local function makecontent(parent,id,specification) +local a_destination = attributes.private('destination') +local a_reference = attributes.private('reference') + +local function makecontent(start,parent,id,specification) local tag = parent.tag local kids = parent.kids local last = index @@ -355,6 +372,16 @@ local function makecontent(parent,id,specification) kids[#kids+1] = d elseif pagenum == parent.pnum then kids[#kids+1] = last + -- if tag == "link" then + -- local ra = getattr(start,a_reference) + -- if ra then + -- parent.refatt = ra + -- kids[#kids+1] = pdfdictionary { + -- Type = pdf_objr, + -- Obj = pdfreference(0), + -- } + -- end + -- end else local d = pdfdictionary { Type = pdf_mcr, @@ -530,7 +557,7 @@ function nodeinjections.addtags(head) end end if prev then - literal = setstate(makecontent(prev,id,specification)) + literal = setstate(makecontent(start,prev,id,specification)) elseif ignore then literal = setstate(makeignore(specification)) else @@ -676,7 +703,7 @@ end -- -- else -- prev = prv -- r = r + 1 --- result[r] = makecontent(prev,id) +-- result[r] = makecontent(start,prev,id) -- -- end -- end -- diff --git a/tex/context/base/mkxl/node-ref.lmt b/tex/context/base/mkxl/node-ref.lmt index a7b624591..2a2bf267f 100644 --- a/tex/context/base/mkxl/node-ref.lmt +++ b/tex/context/base/mkxl/node-ref.lmt @@ -566,7 +566,7 @@ do if depth < dp then depth = dp end end -- logs.report("temp","used: ht=%p dp=%p",height,depth) - local annot = nodeinjections.reference(width,height,depth,set,resolved.mesh) + local annot = nodeinjections.reference(reference,width,height,depth,set,resolved.mesh) if annot then annot = tonut(annot) -- todo nofreferences = nofreferences + 1 diff --git a/tex/context/base/mkxl/scrn-wid.mklx b/tex/context/base/mkxl/scrn-wid.mklx index d760e3e2e..724bdb807 100644 --- a/tex/context/base/mkxl/scrn-wid.mklx +++ b/tex/context/base/mkxl/scrn-wid.mklx @@ -171,7 +171,7 @@ \setupattachment[\currentattachment][#settings]% \expandnamespaceparameter\??attachmentmethod\attachmentparameter\c!method\v!normal} -\setvalue{\??attachmentmethod\v!normal}% +\defcsname\??attachmentmethod\v!normal\endcsname {\edef\currentattachmentsymbol{\attachmentparameter\c!symbol}% \edef\currentattachmentwidth {\attachmentparameter\c!width }% \edef\currentattachmentheight{\attachmentparameter\c!height}% @@ -214,7 +214,7 @@ \dp\b_scrn_attachment_link\currentattachmentdepth \expandnamespaceparameter\??attachmentlocation\attachmentparameter\c!location\s!unknown} -\setvalue{\??attachmentmethod\v!hidden}% +\defcsname\??attachmentmethod\v!hidden\endcsname {\clf_insertattachment tag {\currentattachment}% registered {\currentattachmentregistered}% @@ -249,14 +249,14 @@ \let\scrn_attachment_flush\scrn_attachment_flush_normal -\setvalue{\??attachmentlocation\v!inmargin }{\inmargin {\scrn_attachment_flush}} -\setvalue{\??attachmentlocation\v!leftedge }{\inleftedge {\scrn_attachment_flush}} -\setvalue{\??attachmentlocation\v!rightedge }{\inrightedge {\scrn_attachment_flush}} -\setvalue{\??attachmentlocation\v!leftmargin }{\inleftmargin {\scrn_attachment_flush}} -\setvalue{\??attachmentlocation\v!rightmargin}{\inrightmargin{\scrn_attachment_flush}} -\setvalue{\??attachmentlocation\v!high }{\high {\scrn_attachment_flush}} +\defcsname\??attachmentlocation\v!inmargin \endcsname{\inmargin {\scrn_attachment_flush}} +\defcsname\??attachmentlocation\v!leftedge \endcsname{\inleftedge {\scrn_attachment_flush}} +\defcsname\??attachmentlocation\v!rightedge \endcsname{\inrightedge {\scrn_attachment_flush}} +\defcsname\??attachmentlocation\v!leftmargin \endcsname{\inleftmargin {\scrn_attachment_flush}} +\defcsname\??attachmentlocation\v!rightmargin\endcsname{\inrightmargin{\scrn_attachment_flush}} +\defcsname\??attachmentlocation\v!high \endcsname{\high {\scrn_attachment_flush}} -\setvalue{\??attachmentlocation\v!none}% +\defcsname\??attachmentlocation\v!none\endcsname {\global\setbox\b_scrn_attachment_collect\hbox\bgroup \ifvoid\b_scrn_attachment_collect\else \box\b_scrn_attachment_collect @@ -265,7 +265,7 @@ \scrn_attachment_flush \egroup} -\setvalue{\??attachmentlocation\s!unknown}% +\defcsname\??attachmentlocation\s!unknown\endcsname {\ifvoid\b_scrn_attachment_collect\else \box\b_scrn_attachment_collect \fi} @@ -409,9 +409,9 @@ \mutable\let\currentcommentwidth \empty \appendtoks - \frozen\instance\setuevalue \currentcomment {\scrn_comment_argument[\currentcomment]}% - \frozen\instance\setuevalue{\e!start\currentcomment}{\scrn_comment_start [\currentcomment]}% - \frozen\instance\setuevalue{\e!stop \currentcomment}{\scrn_comment_stop }% + \frozen\protected\instance\edefcsname \currentcomment\endcsname{\scrn_comment_argument[\currentcomment]}% + \frozen\protected\instance\edefcsname\e!start\currentcomment\endcsname{\scrn_comment_start [\currentcomment]}% + \frozen\protected\instance\edefcsname\e!stop \currentcomment\endcsname{\scrn_comment_stop }% \to \everydefinecomment \tolerant\protected\def\scrn_comment_argument[#category]#spacer[#title]#spacer[#settings]#:#text%% @@ -461,7 +461,7 @@ %D Beware: comments symbols don't scale in acrobat (cf. spec but somewhat %D weird, esp because for instance attachment symbols do scale). -\setvalue{\??commentmethods\v!normal}% +\defcsname\??commentmethods\v!normal\endcsname {\edef\currentcommentsymbol{\commentparameter\c!symbol}% \edef\currentcommentwidth {\commentparameter\c!width }% \edef\currentcommentheight{\commentparameter\c!height}% @@ -527,14 +527,14 @@ % todo: dedicated margin classes -\setvalue{\??commentlocation\v!inmargin }{\inmargin {\scrn_comment_flush}} -\setvalue{\??commentlocation\v!leftedge }{\inleftedge {\scrn_comment_flush}} -\setvalue{\??commentlocation\v!rightedge }{\inrightedge {\scrn_comment_flush}} -\setvalue{\??commentlocation\v!leftmargin }{\inleftmargin {\scrn_comment_flush}} -\setvalue{\??commentlocation\v!rightmargin}{\inrightmargin{\scrn_comment_flush}} -\setvalue{\??commentlocation\v!high }{\high {\scrn_comment_flush}} +\defcsname\??commentlocation\v!inmargin \endcsname{\inmargin {\scrn_comment_flush}} +\defcsname\??commentlocation\v!leftedge \endcsname{\inleftedge {\scrn_comment_flush}} +\defcsname\??commentlocation\v!rightedge \endcsname{\inrightedge {\scrn_comment_flush}} +\defcsname\??commentlocation\v!leftmargin \endcsname{\inleftmargin {\scrn_comment_flush}} +\defcsname\??commentlocation\v!rightmargin\endcsname{\inrightmargin{\scrn_comment_flush}} +\defcsname\??commentlocation\v!high \endcsname{\high {\scrn_comment_flush}} -\setvalue{\??commentlocation\v!none}% +\defcsname\??commentlocation\v!none\endcsname {\global\setbox\b_scrn_comment_collect\hbox\bgroup \ifvoid\b_scrn_comment_collect\else \box\b_scrn_comment_collect @@ -543,7 +543,7 @@ \scrn_comment_flush \egroup} -\setvalue{\??commentlocation\s!unknown}% +\defcsname\??commentlocation\s!unknown\endcsname {\ifvoid\b_scrn_comment_collect\else \box\b_scrn_comment_collect \fi} @@ -569,8 +569,20 @@ %D %D associated actions: StartSound StopSound PauseSound ResumeSound %D -%D Todo: like external figures, also search on path, -%D although, they need to be present ar viewing time, so ... +%D Todo: like external figures, also search on path, although, they need to be +%D present at viewing time, so ... +%D +%D A soundtrack needs to be embedded explicitly with \type {\checksoundtrack} +%D although we could hook it into the first goto. +%D +%D \starttyping +%D \useexternalsoundtrack[myaudio][hastobe.wav] % surprise ... no longer mp3 +%D +%D \starttext +%D \checksoundtrack{myaudio} +%D \goto{play sound}[StartSound{myaudio}] +%D \stoptext +%D \starttyping \installcorenamespace{externalsoundtracks} diff --git a/tex/context/base/mkxl/syst-ini.mkxl b/tex/context/base/mkxl/syst-ini.mkxl index 90580c471..1f58b42a1 100644 --- a/tex/context/base/mkxl/syst-ini.mkxl +++ b/tex/context/base/mkxl/syst-ini.mkxl @@ -765,7 +765,7 @@ \tracingmacros \plusthree % > 2: obey \untraced \tracingoutput \plusone \tracingpages \plusone - \tracingparagraphs\plusone + \tracingparagraphs\plustwo \tracingrestores \plusone \tracinggroups \plusone \tracingifs \plusone diff --git a/tex/context/base/mkxl/typo-duc.lmt b/tex/context/base/mkxl/typo-duc.lmt index 00f761c19..2e3638946 100644 --- a/tex/context/base/mkxl/typo-duc.lmt +++ b/tex/context/base/mkxl/typo-duc.lmt @@ -31,7 +31,7 @@ if not modules then modules = { } end modules ['typo-duc'] = { -- todo: no need for a max check -- todo: collapse bound similar ranges (not ok yet) -- todo: combine some sweeps --- todo: removing is not needed when we inject at the same spot (only chnage the dir property) +-- todo: removing is not needed when we inject at the same spot (only change the dir property) -- todo: isolated runs (isolating runs are similar to bidi=local in the basic analyzer) -- todo: check unicode addenda (from the draft): diff --git a/tex/context/interface/mkii/keys-en.xml b/tex/context/interface/mkii/keys-en.xml index 011f03598..3496d40b5 100644 --- a/tex/context/interface/mkii/keys-en.xml +++ b/tex/context/interface/mkii/keys-en.xml @@ -210,6 +210,8 @@ + + @@ -1242,6 +1244,7 @@ + @@ -1271,6 +1274,7 @@ + @@ -1372,6 +1376,7 @@ + diff --git a/tex/context/interface/mkii/keys-it.xml b/tex/context/interface/mkii/keys-it.xml index 3b7577a1f..78a2254d4 100644 --- a/tex/context/interface/mkii/keys-it.xml +++ b/tex/context/interface/mkii/keys-it.xml @@ -1376,6 +1376,7 @@ + diff --git a/tex/context/interface/mkii/keys-nl.xml b/tex/context/interface/mkii/keys-nl.xml index 2312b7675..7e3779cfe 100644 --- a/tex/context/interface/mkii/keys-nl.xml +++ b/tex/context/interface/mkii/keys-nl.xml @@ -210,6 +210,8 @@ + + @@ -1242,6 +1244,7 @@ + @@ -1373,6 +1376,7 @@ + diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index dc5249fca..a87c6272c 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 2021-07-27 19:24 +-- merge date : 2021-07-30 00:41 do -- begin closure to overcome local limits and interference -- cgit v1.2.3