2020-07-05 23:03:00

16 files changed, 91 insertions, 53 deletions

diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii
index 45e745faa..5c058a6ea 100644
--- a/tex/context/base/mkii/cont-new.mkii
+++ b/tex/context/base/mkii/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2020.07.02 17:49}
+\newcontextversion{2020.07.05 23:00}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii
index f13b0d05b..794bd3425 100644
--- a/tex/context/base/mkii/context.mkii
+++ b/tex/context/base/mkii/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.07.02 17:49}
+\edef\contextversion{2020.07.05 23:00}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/mkiv/cldf-ini.lua b/tex/context/base/mkiv/cldf-ini.lua
index 03c9be8f9..fda8ace2a 100644
--- a/tex/context/base/mkiv/cldf-ini.lua
+++ b/tex/context/base/mkiv/cldf-ini.lua
@@ -677,7 +677,7 @@ end
 
 local containseol = patterns.containseol
 
-local t_cldl_luafunction             = newtoken("luafunctioncall",0)
+----- t_cldl_luafunction             = newtoken("luafunctioncall",0)
 local lua_expandable_call_token_code = token.command_id and token.command_id("lua_expandable_call")
 
 local sortedhashindeed = false
diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index d07ab3621..025972e38 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2020.07.02 17:49}
+\newcontextversion{2020.07.05 23:00}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index 64fec04ab..a42f6b081 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -45,7 +45,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.07.02 17:49}
+\edef\contextversion{2020.07.05 23:00}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/context.mkxl b/tex/context/base/mkiv/context.mkxl
index 1d8188b2d..fb4858e7b 100644
--- a/tex/context/base/mkiv/context.mkxl
+++ b/tex/context/base/mkiv/context.mkxl
@@ -29,7 +29,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.07.02 17:49}
+\edef\contextversion{2020.07.05 23:00}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/mult-low.lua b/tex/context/base/mkiv/mult-low.lua
index 8c09dd803..0a6462c1c 100644
--- a/tex/context/base/mkiv/mult-low.lua
+++ b/tex/context/base/mkiv/mult-low.lua
@@ -236,7 +236,7 @@ return {
         "triggergroupedcommand", "triggergroupedcommandcs",
         "simplegroupedcommand", "pickupgroupedcommand",
         --
-        "normalbaselineskip", "normallineskip", "normallineskiplimit",
+        "usedbaselineskip", "usedlineskip", "usedlineskiplimit",
         --
         "availablehsize", "localhsize", "setlocalhsize", "distributedhsize", "hsizefraction",
         --
diff --git a/tex/context/base/mkiv/spac-ver.mkiv b/tex/context/base/mkiv/spac-ver.mkiv
index 6b1abe436..6be653155 100644
--- a/tex/context/base/mkiv/spac-ver.mkiv
+++ b/tex/context/base/mkiv/spac-ver.mkiv
@@ -738,19 +738,19 @@
 \unexpanded\def\setmaxdepth
   {\maxdepth\systemmaxdepthfactor\globalbodyfontsize}
 
-\let\normalbaselineskip \relax \newskip \normalbaselineskip  % these got lost in the transition to mkiv due
-\let\normallineskip     \relax \newskip \normallineskip      % to auto-\normal* definitions and registers
-\let\normallineskiplimit\relax \newdimen\normallineskiplimit % being protected
+\newskip \usedbaselineskip
+\newskip \usedlineskip
+\newdimen\usedlineskiplimit
 
 \unexpanded\def\normalbaselines
-  {\baselineskip \normalbaselineskip
-   \lineskip     \normallineskip
-   \lineskiplimit\normallineskiplimit}
+  {\baselineskip \usedbaselineskip
+   \lineskip     \usedlineskip
+   \lineskiplimit\usedlineskiplimit}
 
 \unexpanded\def\flexiblebaselines
-  {\baselineskip \normalbaselineskip
-   \lineskip     1\normallineskip \s!plus 1\s!fill
-   \lineskiplimit\normallineskiplimit}
+  {\baselineskip \usedbaselineskip
+   \lineskip     1\usedlineskip \s!plus 1\s!fill
+   \lineskiplimit\usedlineskiplimit}
 
 \unexpanded\def\setnormalbaselines
   {\ifdim\normallineheight>\zeropoint
@@ -770,13 +770,13 @@
    \ifdim\dimexpr\minimumstrutdepth+\minimumstrutheight\relax>\zeropoint
      \openlineheight\dimexpr\openstrutheight+\openstrutdepth\relax % new
    \fi
-   \normalbaselineskip\openlineheight
+   \usedbaselineskip\openlineheight
      \ifgridsnapping\else
        \s!plus \m_spac_vertical_baseline_stretch_factor\openlineheight
        \s!minus\m_spac_vertical_baseline_shrink_factor \openlineheight
      \fi
-   \normallineskip\minimumlinedistance\relax % \onepoint\relax
-   \normallineskiplimit\zeropoint\relax
+   \usedlineskip\minimumlinedistance\relax % \onepoint\relax
+   \usedlineskiplimit\zeropoint\relax
    \normalbaselines}
 
 \unexpanded\def\spacing#1% vertical
@@ -818,9 +818,9 @@
       \openstrutheight    \the\openstrutheight
       \openstrutdepth     \the\openstrutdepth
       \openlineheight     \the\openlineheight
-      \normalbaselineskip \the\normalbaselineskip
-      \normallineskip     \the\normallineskip
-      \normallineskiplimit\the\normallineskiplimit
+      \usedbaselineskip   \the\usedbaselineskip
+      \usedlineskip       \the\usedlineskip
+      \usedlineskiplimit  \the\usedlineskiplimit
       \noexpand\def\noexpand\normallineheight{\the\dimexpr\normallineheight}%
       \noexpand\normalbaselines}}
 
diff --git a/tex/context/base/mkiv/spac-ver.mkxl b/tex/context/base/mkiv/spac-ver.mkxl
index b69c1d33c..d5f85569a 100644
--- a/tex/context/base/mkiv/spac-ver.mkxl
+++ b/tex/context/base/mkiv/spac-ver.mkxl
@@ -738,19 +738,19 @@
 \unexpanded\def\setmaxdepth
   {\maxdepth\systemmaxdepthfactor\globalbodyfontsize}
 
-\let\normalbaselineskip \relax \newskip \normalbaselineskip  % these got lost in the transition to mkiv due
-\let\normallineskip     \relax \newskip \normallineskip      % to auto-\normal* definitions and registers
-\let\normallineskiplimit\relax \newdimen\normallineskiplimit % being protected
+\newskip \usedbaselineskip  % These used to be \normal... but that isn't pretty
+\newskip \usedlineskip      % in the token interface, so thes few now have new
+\newdimen\usedlineskiplimit % names. They are public but not really user commands.
 
 \unexpanded\def\normalbaselines
-  {\baselineskip \normalbaselineskip
-   \lineskip     \normallineskip
-   \lineskiplimit\normallineskiplimit}
+  {\baselineskip \usedbaselineskip
+   \lineskip     \usedlineskip
+   \lineskiplimit\usedlineskiplimit}
 
 \unexpanded\def\flexiblebaselines
-  {\baselineskip \normalbaselineskip
-   \lineskip     1\normallineskip \s!plus 1\s!fill
-   \lineskiplimit\normallineskiplimit}
+  {\baselineskip \usedbaselineskip
+   \lineskip     1\usedlineskip \s!plus 1\s!fill
+   \lineskiplimit\usedlineskiplimit}
 
 \unexpanded\def\setnormalbaselines
   {\ifdim\normallineheight>\zeropoint
@@ -770,13 +770,13 @@
    \ifdim\dimexpr\minimumstrutdepth+\minimumstrutheight\relax>\zeropoint
      \openlineheight\dimexpr\openstrutheight+\openstrutdepth\relax % new
    \fi
-   \normalbaselineskip\openlineheight
+   \usedbaselineskip\openlineheight
      \ifgridsnapping\else
        \s!plus \m_spac_vertical_baseline_stretch_factor\openlineheight
        \s!minus\m_spac_vertical_baseline_shrink_factor \openlineheight
      \fi
-   \normallineskip\minimumlinedistance\relax % \onepoint\relax
-   \normallineskiplimit\zeropoint\relax
+   \usedlineskip\minimumlinedistance\relax % \onepoint\relax
+   \usedlineskiplimit\zeropoint\relax
    \normalbaselines}
 
 \unexpanded\def\spacing#1% vertical
@@ -818,9 +818,9 @@
       \openstrutheight    \the\openstrutheight
       \openstrutdepth     \the\openstrutdepth
       \openlineheight     \the\openlineheight
-      \normalbaselineskip \the\normalbaselineskip
-      \normallineskip     \the\normallineskip
-      \normallineskiplimit\the\normallineskiplimit
+      \usedbaselineskip   \the\usedbaselineskip
+      \usedlineskip       \the\usedlineskip
+      \usedlineskiplimit  \the\usedlineskiplimit
       \noexpand\def\noexpand\normallineheight{\the\dimexpr\normallineheight}%
       \noexpand\normalbaselines}}
 
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index 0f1852d0b..1a4f8cf7b 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index 1b0676045..a49f8d2a7 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
diff --git a/tex/context/base/mkiv/syst-aux.mkiv b/tex/context/base/mkiv/syst-aux.mkiv
index 43d9c6b31..824590d9e 100644
--- a/tex/context/base/mkiv/syst-aux.mkiv
+++ b/tex/context/base/mkiv/syst-aux.mkiv
@@ -4979,7 +4979,7 @@
 % experimental
 
 \unexpanded\def\gobblenested#1#2#3%
-  {\normalexpanded{\def\next##1\csname#2\endcsname}{\csname#3\endcsname}%
+  {\normalexpanded{\def\noexpand\next##1\csname#2\endcsname}{\csname#3\endcsname}%
    \next}%
 
 %D The next one simply expands the pickup up tokens.
diff --git a/tex/context/base/mkiv/syst-aux.mkxl b/tex/context/base/mkiv/syst-aux.mkxl
index 2c84fe2c6..6574ba9b0 100644
--- a/tex/context/base/mkiv/syst-aux.mkxl
+++ b/tex/context/base/mkiv/syst-aux.mkxl
@@ -3185,7 +3185,7 @@
    \globalpushmacro\recurseaction
    \unexpanded\gdef\recurseaction##1##2{#4}%
    \normalexpanded{\ifcmpnum#3\zerocount
-     \ifnum#1<#2\relax
+     \ifnum#1<#2\relax\relax % so we catch \number\numexpr xx without \relax's
        \syst_helpers_stepwise_exit
      \else
        \syst_helpers_stepwise_reverse
@@ -3193,12 +3193,12 @@
    \or
      \syst_helpers_stepwise_exit
    \or
-     \ifnum#2<#1\relax
+     \ifnum#2<#1\relax\relax % so we catch \number\numexpr xx without \relax's
        \syst_helpers_stepwise_exit
      \else
        \syst_helpers_stepwise_recurse
      \fi
-   \fi{\number#1}{\number#2}{\number#3}}
+   \fi{\number#1}{\number#2}{\number#3}}%
    \globalpopmacro\recurselevel
    \globalpopmacro\recurseaction
    \global\advance\outerrecurse\minusone}
diff --git a/tex/context/base/mkiv/util-prs.lua b/tex/context/base/mkiv/util-prs.lua
index 3154bf6c2..8915c3e9e 100644
--- a/tex/context/base/mkiv/util-prs.lua
+++ b/tex/context/base/mkiv/util-prs.lua
@@ -10,7 +10,7 @@ local lpeg, table, string = lpeg, table, string
 local P, R, V, S, C, Ct, Cs, Carg, Cc, Cg, Cf, Cp = lpeg.P, lpeg.R, lpeg.V, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cs, lpeg.Carg, lpeg.Cc, lpeg.Cg, lpeg.Cf, lpeg.Cp
 local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns
 local concat, gmatch, find = table.concat, string.gmatch, string.find
-local tostring, type, next, rawset = tostring, type, next, rawset
+local tonumber, tostring, type, next, rawset = tonumber, tostring, type, next, rawset
 local mod, div = math.mod, math.div
 
 utilities         = utilities or {}
@@ -538,13 +538,15 @@ function parsers.csvsplitter(specification)
     specification   = specification and setmetatableindex(specification,defaultspecification) or defaultspecification
     local separator = specification.separator
     local quotechar = specification.quote
+    local numbers   = specification.numbers
     local separator = S(separator ~= "" and separator or ",")
     local whatever  = C((1 - separator - newline)^0)
     if quotechar and quotechar ~= "" then
         local quotedata = nil
         for chr in gmatch(quotechar,".") do
             local quotechar = P(chr)
-            local quoteword = quotechar * C((1 - quotechar)^0) * quotechar
+            local quoteitem = (1 - quotechar)^0
+            local quoteword = quotechar * (numbers and (quoteitem/tonumber) or C(quoteitem)) * quotechar
             if quotedata then
                 quotedata = quotedata + quoteword
             else
@@ -559,29 +561,29 @@ function parsers.csvsplitter(specification)
     end
 end
 
--- and this is a slightly patched version of a version posted by Philipp Gesang
-
--- local mycsvsplitter = parsers.rfc4180splitter()
-
 -- local crap = [[
 -- first,second,third,fourth
 -- "1","2","3","4"
--- "a","b","c","d"
--- "foo","bar""baz","boogie","xyzzy"
+-- "5","6","7","8"
 -- ]]
+--
+-- local mycsvsplitter = parsers.csvsplitter { numbers = true }
+--
+-- local list = mycsvsplitter(crap) inspect(list)
 
--- local list, names = mycsvsplitter(crap,true)   inspect(list) inspect(names)
--- local list, names = mycsvsplitter(crap)        inspect(list) inspect(names)
+-- and this is a slightly patched version of a version posted by Philipp Gesang
 
 function parsers.rfc4180splitter(specification)
     specification     = specification and setmetatableindex(specification,defaultspecification) or defaultspecification
+    local numbers     = specification.numbers
     local separator   = specification.separator --> rfc: COMMA
     local quotechar   = P(specification.quote)  -->      DQUOTE
     local dquotechar  = quotechar * quotechar   -->      2DQUOTE
                       / specification.quote
     local separator   = S(separator ~= "" and separator or ",")
+    local whatever    = (dquotechar + (1 - quotechar))^0
     local escaped     = quotechar
-                      * Cs((dquotechar + (1 - quotechar))^0)
+                      * (numbers and (whatever/tonumber) or Cs(whatever))
                       * quotechar
     local non_escaped = C((1 - quotechar - newline - separator)^1)
     local field       = escaped + non_escaped + Cc("")
@@ -601,6 +603,18 @@ function parsers.rfc4180splitter(specification)
     end
 end
 
+-- local mycsvsplitter = parsers.rfc4180splitter { numbers = true }
+--
+-- local crap = [[
+-- first,second,third,fourth
+-- "1","2","3","4"
+-- "a","b","c","d"
+-- "foo","bar""baz","boogie","xyzzy"
+-- ]]
+--
+-- local list, names = mycsvsplitter(crap,true)   inspect(list) inspect(names)
+-- local list, names = mycsvsplitter(crap)        inspect(list) inspect(names)
+
 -- parsers.stepper("1,7-",9,function(i) print(">>>",i) end)
 -- parsers.stepper("1-3,7,8,9")
 -- parsers.stepper("1-3,6,7",function(i) print(">>>",i) end)
diff --git a/tex/context/modules/mkiv/s-system-tokens.mkxl b/tex/context/modules/mkiv/s-system-tokens.mkxl
index e33b11a70..fd16bade6 100644
--- a/tex/context/modules/mkiv/s-system-tokens.mkxl
+++ b/tex/context/modules/mkiv/s-system-tokens.mkxl
@@ -31,6 +31,30 @@
 \setupbodyfont[8pt]
 
 \starttext
+
     \showsystemtokens
+
+    Some tokens cannot be created yet because they are more complex but eventually the creator will
+    deal with them. The indices are sometimes the same as internal but in most cases nicely start
+    at zero. When they start at another value, then there is an internal token or state that is not
+    user accessible. You always need to use symbolic constants as the numbers can change.
+
+    The first column is a classifier (which is experimental and categories might change when we feel
+    the need).
+
+    \startcolumns[n=5]
+    \starttabulate[|T||]
+    \NC 0 \NC unused    \NC \NR
+    \NC 1 \NC regular   \NC \NR
+    \NC 2 \NC character \NC \NR
+    \NC 3 \NC register  \NC \NR
+    \NC 4 \NC internal  \NC \NR
+    \NC 5 \NC reference \NC \NR
+    \NC 6 \NC data      \NC \NR
+    \NC 7 \NC token     \NC \NR
+    \NC 8 \NC node      \NC \NR
+    \stoptabulate
+    \stopcolumns
+
 \stoptext
 
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 4ae9c30f8..88fb53499 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date  : 2020-07-02 17:49
+-- merge date  : 2020-07-05 23:00
 
 do -- begin closure to overcome local limits and interference