2020-07-13 16:31:00

20 files changed, 52 insertions, 29 deletions

diff --git a/doc/context/documents/general/manuals/luametatex.pdf b/doc/context/documents/general/manuals/luametatex.pdf
index e4db222c9..d9c50e96b 100644
--- a/doc/context/documents/general/manuals/luametatex.pdf
+++ b/doc/context/documents/general/manuals/luametatex.pdf
diff --git a/doc/context/documents/general/qrcs/setup-cs.pdf b/doc/context/documents/general/qrcs/setup-cs.pdf
index 7d9f46424..75875aee7 100644
--- a/doc/context/documents/general/qrcs/setup-cs.pdf
+++ b/doc/context/documents/general/qrcs/setup-cs.pdf
diff --git a/doc/context/documents/general/qrcs/setup-de.pdf b/doc/context/documents/general/qrcs/setup-de.pdf
index 7c92252e8..72782f896 100644
--- a/doc/context/documents/general/qrcs/setup-de.pdf
+++ b/doc/context/documents/general/qrcs/setup-de.pdf
diff --git a/doc/context/documents/general/qrcs/setup-en.pdf b/doc/context/documents/general/qrcs/setup-en.pdf
index ca7a9bbbf..59f603a12 100644
--- a/doc/context/documents/general/qrcs/setup-en.pdf
+++ b/doc/context/documents/general/qrcs/setup-en.pdf
diff --git a/doc/context/documents/general/qrcs/setup-fr.pdf b/doc/context/documents/general/qrcs/setup-fr.pdf
index 0cee0b9bb..4d16223da 100644
--- a/doc/context/documents/general/qrcs/setup-fr.pdf
+++ b/doc/context/documents/general/qrcs/setup-fr.pdf
diff --git a/doc/context/documents/general/qrcs/setup-it.pdf b/doc/context/documents/general/qrcs/setup-it.pdf
index 482884cb6..460c7e01a 100644
--- a/doc/context/documents/general/qrcs/setup-it.pdf
+++ b/doc/context/documents/general/qrcs/setup-it.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-cs.pdf b/doc/context/documents/general/qrcs/setup-mapping-cs.pdf
index 24143049b..f97b06880 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-cs.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-cs.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-de.pdf b/doc/context/documents/general/qrcs/setup-mapping-de.pdf
index 9ce15011f..aeb65503c 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-de.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-de.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-en.pdf b/doc/context/documents/general/qrcs/setup-mapping-en.pdf
index e6e63a011..6d75c98c2 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-en.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-en.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-fr.pdf b/doc/context/documents/general/qrcs/setup-mapping-fr.pdf
index d88b02e92..d2910f110 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-fr.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-fr.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-it.pdf b/doc/context/documents/general/qrcs/setup-mapping-it.pdf
index 289286a97..29fc39deb 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-it.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-it.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-nl.pdf b/doc/context/documents/general/qrcs/setup-mapping-nl.pdf
index 9d5a949dd..7ac7ca36d 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-nl.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-nl.pdf
diff --git a/doc/context/documents/general/qrcs/setup-mapping-ro.pdf b/doc/context/documents/general/qrcs/setup-mapping-ro.pdf
index 187a20eef..04d845ce6 100644
--- a/doc/context/documents/general/qrcs/setup-mapping-ro.pdf
+++ b/doc/context/documents/general/qrcs/setup-mapping-ro.pdf
diff --git a/doc/context/documents/general/qrcs/setup-nl.pdf b/doc/context/documents/general/qrcs/setup-nl.pdf
index 4f4ef468f..ca6381cc4 100644
--- a/doc/context/documents/general/qrcs/setup-nl.pdf
+++ b/doc/context/documents/general/qrcs/setup-nl.pdf
diff --git a/doc/context/documents/general/qrcs/setup-ro.pdf b/doc/context/documents/general/qrcs/setup-ro.pdf
index 2175a74f3..9e47f4f86 100644
--- a/doc/context/documents/general/qrcs/setup-ro.pdf
+++ b/doc/context/documents/general/qrcs/setup-ro.pdf
diff --git a/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex b/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex
index d653703a9..8c1e3ccb0 100644
--- a/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex
+++ b/doc/context/sources/general/manuals/evenmore/evenmore-tokens.tex
@@ -9,10 +9,6 @@
 
 \startchapter[title=Tokens]
 
-\usemodule[article-basic,abbreviations-logos]
-
-\starttext
-
 {\em This is mostly a wrapup of some developments, and definitely not a tutorial.}
 
 Talking deep down \TEX\ is talking about tokens and nodes. Roughly spoken, from
diff --git a/doc/context/sources/general/manuals/libraries/libraries-mkiv.tex b/doc/context/sources/general/manuals/libraries/libraries-mkiv.tex
index f1801efc0..19c045f5f 100644
--- a/doc/context/sources/general/manuals/libraries/libraries-mkiv.tex
+++ b/doc/context/sources/general/manuals/libraries/libraries-mkiv.tex
@@ -240,17 +240,18 @@ There are some optional libraries present, like:
 
 \starttabulate[|l|l|l|l|l|]
 \FL
-\BC module                  \BC library        \BC windows     \BC unix           \NC \NR
+\BC module                  \BC library        \BC windows      \BC unix           \NC \NR
 \ML
-\NC libs-imp-curl           \NC curl           \NC libcurl     \NC libcurl        \NC \NR
-\NC libs-imp-mysql          \NC mysql          \NC libmysql    \NC libmysqlclient \NC \NR
-\NC libs-imp-sqlite         \NC sqlite         \NC sqlite3     \NC sqlite3        \NC \NR
-%NC libs-imp-postgress      \NC libpg          \NC libpg       \NC libpg          \NC \NR % untested
-\NC libs-imp-zint           \NC libzint        \NC libzint     \NC libzint        \NC \NR
-\NC libs-imp-mujs           \NC libmujs        \NC libmujs     \NC libmujs        \NC \NR
-\NC libs-imp-ghostscript    \NC ghostscript    \NC gswin64     \NC libgs          \NC \NR
-\NC libs-imp-graphicsmagick \NC graphicsmagick \NC several     \NC unknown        \NC \NR
-%NC font-phb-imp-internal   \NC hb             \NC libharfbuzz \NC libharfbuzz    \NC \NR % maybe, for idris, testing uniscribe
+\NC libs-imp-curl           \NC curl           \NC libcurl      \NC libcurl        \NC \NR
+\NC libs-imp-mysql          \NC mysql          \NC libmysql     \NC libmysqlclient \NC \NR
+\NC libs-imp-sqlite         \NC sqlite         \NC sqlite3      \NC sqlite3        \NC \NR
+%NC libs-imp-postgress      \NC libpg          \NC libpg        \NC libpg          \NC \NR % untested
+\NC libs-imp-zint           \NC libzint        \NC libzint      \NC libzint        \NC \NR
+\NC libs-imp-mujs           \NC libmujs        \NC libmujs      \NC libmujs        \NC \NR
+\NC libs-imp-kpse           \NC kpse           \NC kpathsea*w64 \NC libkpathsea    \NC \NR % left-over for mkii runner
+\NC libs-imp-ghostscript    \NC ghostscript    \NC gswin64      \NC libgs          \NC \NR
+\NC libs-imp-graphicsmagick \NC graphicsmagick \NC several      \NC unknown        \NC \NR
+%NC font-phb-imp-internal   \NC hb             \NC libharfbuzz  \NC libharfbuzz    \NC \NR % maybe, for idris, testing uniscribe
 \LL
 \stoptabulate
 
diff --git a/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex b/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
index 07790af29..af058197b 100644
--- a/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
+++ b/doc/context/sources/general/manuals/luametatex/luametatex-tex.tex
@@ -796,6 +796,9 @@ The first argument is one of \type {top}, \type {bottom}, \type {first}, \type
 number. When no arguments are given the current maximum number of classes is
 returned.
 
+When \type {tex.gettoks} gets an extra argument \type {true} it will return a
+table with userdata tokens.
+
 \stopsubsection
 
 \startsubsection[title={Character code registers: \type {[get|set]*code[s]}}]
@@ -2198,22 +2201,43 @@ a bit but for passing strings conversion to and from tokens has to be done anywa
 
 \startsubsection[title={Picking up one token}]
 
-\libindex {get_next}
+\libindex {scan_next}
+\libindex {scan_next_expanded}
+\libindex {skip_next}
+\libindex {skip_next_expanded}
+\libindex {peek_next}
+\libindex {peek_next_expanded}
 \libindex {scan_token}
 \libindex {expand}
 
 The scanners look for a sequence. When you want to pick up one token from the
-input you use \type {get_next}. This creates a token with the (low level)
+input you use \type {scan_next}. This creates a token with the (low level)
 properties as discussed next. This token is just the next one. If you want to
-enforce expansion first you can use \type {scan_token}. Internally tokens are
-characterized by a number that packs a lot of information. In order to access
-the bits of information a token is wrapped in a userdata object.
+enforce expansion first you can use \type {scan_token} or the \type {_expanded}
+variants. Internally tokens are characterized by a number that packs a lot of
+information. In order to access the bits of information a token is wrapped in a
+userdata object.
 
 The \type {expand} function will trigger expansion of the next token in the
 input. This can be quite unpredictable but when you call it you probably know
 enough about \TEX\ not to be too worried about that. It basically is a call to
 the internal expand related function.
 
+\starttabulate[|lT|p|]
+\DB name \BC explanation \NC \NR
+\TB
+\NC scan_next          \NC get the next token \NC \NR
+\NC scan_next_expanded \NC get the next expanded token \NC \NR
+\NC skip_next          \NC skip the next token \NC \NR
+\NC skip_next_expanded \NC skip the next expanded token \NC \NR
+\NC peek_next          \NC get the next token and put it back in the input \NC \NR
+\NC peek_next_expanded \NC get the next expanded token and put it back in the input \NC \NR
+\LL
+\stoptabulate
+
+The peek function accept a boolean argument that triggers skipping spaces and
+alike.
+
 \stopsubsection
 
 \startsubsection[title={Creating tokens}]
@@ -2241,7 +2265,7 @@ the internal expand related function.
 \libindex{get_frozen}
 \libindex{get_user}
 
-\libindex{get_next}
+\libindex{scan_next}
 
 The creator function can be used as follows:
 
@@ -2392,16 +2416,16 @@ and are to be seen as abstractions.
 
 \startsubsection[title={Pushing back}]
 
-\libindex{get_next}
+\libindex{scan_next}
 \libindex{put_next}
 
 There is a (for now) experimental putter:
 
 \starttyping
-local t1 = token.get_next()
-local t2 = token.get_next()
-local t3 = token.get_next()
-local t4 = token.get_next()
+local t1 = token.scan_next()
+local t2 = token.scan_next()
+local t3 = token.scan_next()
+local t4 = token.scan_next()
 -- watch out, we flush in sequence
 token.put_next { t1, t2 }
 -- but this one gets pushed in front
@@ -2509,9 +2533,9 @@ When scanning from \LUA\ we are not in a mode that defines (undefined) macros at
 all. There we just get the real primitive undefined macro token.
 
 \startbuffer
-\directlua{local t = token.get_next() tex.print(t.id.." "..t.tok)}\myfirstbar
-\directlua{local t = token.get_next() tex.print(t.id.." "..t.tok)}\mysecondbar
-\directlua{local t = token.get_next() tex.print(t.id.." "..t.tok)}\mythirdbar
+\directlua{local t = token.scan_next() tex.print(t.id.." "..t.tok)}\myfirstbar
+\directlua{local t = token.scan_next() tex.print(t.id.." "..t.tok)}\mysecondbar
+\directlua{local t = token.scan_next() tex.print(t.id.." "..t.tok)}\mythirdbar
 \stopbuffer
 
 \startlines
diff --git a/doc/context/sources/general/manuals/mk/mk-environment.tex b/doc/context/sources/general/manuals/mk/mk-environment.tex
index 3472f3b04..bcbfa0851 100644
--- a/doc/context/sources/general/manuals/mk/mk-environment.tex
+++ b/doc/context/sources/general/manuals/mk/mk-environment.tex
@@ -8,6 +8,8 @@
 
 \usemodule[timing]
 
+\usemodule[obsolete-tokens]
+
 \useMPlibrary[dum]
 
 \dontcomplain
diff --git a/doc/context/sources/general/manuals/mk/mk-tokenspeak.tex b/doc/context/sources/general/manuals/mk/mk-tokenspeak.tex
index 590dbba43..8382f5cfc 100644
--- a/doc/context/sources/general/manuals/mk/mk-tokenspeak.tex
+++ b/doc/context/sources/general/manuals/mk/mk-tokenspeak.tex
@@ -212,7 +212,7 @@ The result looks like: \ctxlua {
         tokens.letters("12345"),
         tokens.egroup,
     }
-    tex.printlist (t)
+    tex.printlist(t)
     tokens.collectors.show(t)
 }