2020-08-09 19:54:00

20 files changed, 258 insertions, 53 deletions

diff --git a/tex/context/base/mkiv/back-out.lmt b/tex/context/base/mkiv/back-out.lmt
index 36787bca2..af66d18c4 100644
--- a/tex/context/base/mkiv/back-out.lmt
+++ b/tex/context/base/mkiv/back-out.lmt
@@ -14,11 +14,11 @@ local context             = context
 -- tokens.scanners.....
 
 local get                 = token.get_index
-local scaninteger         = token.scan_int
-local scanstring          = token.scan_string
-local scankeyword         = token.scan_keyword
-local scantokenlist       = token.scan_tokenlist
------ scancode            = token.scan_code
+local scanners            = tokens.scanners
+local scaninteger         = scanners.integer
+local scanstring          = scanners.string
+local scankeyword         = scanners.keyword
+local scantokenlist       = scanners.tokenlist
 
 local tokentostring       = token.to_string
 
diff --git a/tex/context/base/mkiv/cldf-ini.lua b/tex/context/base/mkiv/cldf-ini.lua
index fda8ace2a..62a4d8e8a 100644
--- a/tex/context/base/mkiv/cldf-ini.lua
+++ b/tex/context/base/mkiv/cldf-ini.lua
@@ -375,13 +375,17 @@ interfaces.storedscanners = storedscanners
 
 storage.register("interfaces/storedscanners", storedscanners, "interfaces.storedscanners")
 
-local function registerscanner(name,action,protected,public,value)
+local function registerscanner(name,action,protected,public,value,condition) -- todo: combine value and condition
     rawset(interfacescanners,name,action)
     local n = storedscanners[name]
     n = registerfunction("interfaces.scanners."..name,true,n)
     storedscanners[name] = n
     name = public and name or (privatenamespace .. name)
-    setluatoken(name,n,"global",protected and "protected" or "",value and "value" or "")
+    setluatoken(name,n,"global", -- always
+        protected and "protected" or "",
+        value     and "value"     or "",
+        condition and "condition" or ""
+    )
 end
 
 interfaces.registerscanner = registerscanner
diff --git a/tex/context/base/mkiv/cldf-lmt.lmt b/tex/context/base/mkiv/cldf-lmt.lmt
index a660dbb3b..a9e312423 100644
--- a/tex/context/base/mkiv/cldf-lmt.lmt
+++ b/tex/context/base/mkiv/cldf-lmt.lmt
@@ -26,6 +26,7 @@ local scanboolean       = scanners.boolean
 local scandimen         = scanners.dimen
 local scanfloat         = scanners.float
 local scaninteger       = scanners.integer
+local scancardinal      = scanners.cardinal
 local scannumber        = scanners.luanumber
 local scanluainteger    = scanners.luainteger
 local scanluacardinal   = scanners.luacardinal
@@ -33,6 +34,7 @@ local scanluanumber     = scanners.luanumber
 local scanargument      = scanners.argument
 local scantoken         = scanners.token
 local scancsname        = scanners.csname
+local scankeyword       = scanners.keyword
 
 local getindex          = token.get_index
 
@@ -719,3 +721,119 @@ implement {
 
 context.hashedtables  = setmetatableindex(function(t,k) return tables[k] end)
 context.indexedtables = context.hashedtables
+
+-- I played with adding bitwise operator to the expr scanners but in the end didn't
+-- keep the code ... the problem is in what symbols ... so for now a more verbose
+-- and/or/xor is okay. Extending the expression scanner would add a little overhead
+-- to all expressions while an logical operator would gain less that 50% so for now
+-- we don't bother. Providing \ifbitwiseand makes sense for performance reasons as
+-- if a bit beter than testing the bitwise and with an \ifcase.
+
+local reporterror = logs.reporterror
+
+local function youcant(cmd)
+    reporterror("you can't use \\%s this way, it's an integer value", cmd)
+end
+
+implement {
+    name    = "bitwiseset",
+    public  = true,
+    value   = true,
+    actions = function(what)
+        local a = scancardinal()
+        if what == "value" then
+            return cardinal_code, a
+        else
+            youcant("bitwiseset")
+        end
+    end
+}
+
+implement {
+    name    = "bitwiseand",
+    public  = true,
+    value   = true,
+    actions = function(what)
+        local a = scancardinal()
+        scankeyword("with")
+        local b = scancardinal()
+        if what == "value" then
+            return cardinal_code, a & b
+        else
+            youcant("bitwiseand")
+        end
+    end
+}
+
+implement {
+    name    = "bitwiseor",
+    public  = true,
+    value   = true,
+    actions = function(what)
+        local a = scancardinal()
+        scankeyword("with")
+        local b = scancardinal()
+        if what == "value" then
+            return cardinal_code, a | b
+        else
+            youcant("bitwiseor")
+        end
+    end
+}
+
+implement {
+    name    = "bitwisexor",
+    public  = true,
+    value   = true,
+    actions = function(what)
+        local a = scancardinal()
+        scankeyword("with")
+        local b = scancardinal()
+        if what == "value" then
+            return cardinal_code, a ~ b
+        else
+            youcant("bitwisexor")
+        end
+    end
+}
+
+implement {
+    name    = "bitwisenot",
+    public  = true,
+    value   = true,
+    actions = function(what)
+        local a = scancardinal()
+        if what == "value" then
+            return cardinal_code, ~ a
+        else
+            youcant("bitwisenot")
+        end
+    end
+}
+
+implement {
+    name    = "bitwisenil",
+    public  = true,
+    value   = true,
+    actions = function(what)
+        local a = scancardinal()
+        scankeyword("with")
+        local b = scancardinal()
+        if what == "value" then
+            return cardinal_code, a & (~ b)
+        else
+            youcant("bitwisenil")
+        end
+    end
+}
+
+implement {
+    name      = "ifbitwiseand",
+    public    = true,
+    condition = true,
+    actions   = function(what)
+        local a = scancardinal()
+        local b = scancardinal()
+        return boolean_code, (a & b) ~= 0
+    end
+}
diff --git a/tex/context/base/mkiv/cldf-scn.lua b/tex/context/base/mkiv/cldf-scn.lua
index fc4ea1e4a..8fd4f2978 100644
--- a/tex/context/base/mkiv/cldf-scn.lua
+++ b/tex/context/base/mkiv/cldf-scn.lua
@@ -77,7 +77,7 @@ function interfaces.implement(specification)
     if knownscanner(name) and not specification.overload then
         report("warning: interface scanner %a is overloaded",name)
     end
-    register(name,scanner,specification.protected,specification.public,specification.value)
+    register(name,scanner,specification.protected,specification.public,specification.value,specification.condition)
     if private then
         return
     end
diff --git a/tex/context/base/mkiv/cont-log.mkiv b/tex/context/base/mkiv/cont-log.mkiv
index 75d87eb81..42de5b434 100644
--- a/tex/context/base/mkiv/cont-log.mkiv
+++ b/tex/context/base/mkiv/cont-log.mkiv
@@ -289,7 +289,7 @@
 \unexpanded\def\MPXL{MpXL} % MarkFourty : LuaMetaFun
 \unexpanded\def\MPLX{MpLX} % MarkSixty  : LuaMetaFun
 
-\unexpanded\def\LMTX{lmtx} % for LuaMetaTeX
+\unexpanded\def\LMTX{LMTX} % for LuaMetaTeX
 
 \appendtoks
     \def\ConTeXt   {ConTeXt}%
diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index 6049160c8..5a5e0acb5 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2020.08.07 23:34}
+\newcontextversion{2020.08.09 19:51}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index 7ced40420..0a7c70fe9 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -45,7 +45,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.08.07 23:34}
+\edef\contextversion{2020.08.09 19:51}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/context.mkxl b/tex/context/base/mkiv/context.mkxl
index 313bf87ae..531cea7b3 100644
--- a/tex/context/base/mkiv/context.mkxl
+++ b/tex/context/base/mkiv/context.mkxl
@@ -29,7 +29,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2020.08.07 23:34}
+\edef\contextversion{2020.08.09 19:51}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/lang-ini.mkiv b/tex/context/base/mkiv/lang-ini.mkiv
index 946fe6845..45aea36fb 100644
--- a/tex/context/base/mkiv/lang-ini.mkiv
+++ b/tex/context/base/mkiv/lang-ini.mkiv
@@ -430,6 +430,12 @@
 
 \let\dohyphens\relax
 
+\unexpanded\def\usehyphensparameter#1%
+  {\edef\p_hyphens{#1\c!hyphens}%
+   \ifx\p_hyhens\v!no
+     \nohyphens
+   \fi}
+
 \unexpanded\def\lang_basics_synchronize_min_max % maybe store this at the lua end
   {% these values are stored along with glyph nodes
    \lefthyphenmin \numexpr0\languageparameter\s!lefthyphenmin +\hyphenminoffset\relax
diff --git a/tex/context/base/mkiv/lang-ini.mkxl b/tex/context/base/mkiv/lang-ini.mkxl
index 490d35997..b7a589b47 100644
--- a/tex/context/base/mkiv/lang-ini.mkxl
+++ b/tex/context/base/mkiv/lang-ini.mkxl
@@ -409,6 +409,46 @@
 \newcount\hyphenstate
 \newcount\hyphenminoffset
 
+% This is the old implementation: a hack that sets the min values, because we want
+% to keep the language set:
+%
+% \unexpanded\def\nohyphens % nicer for url's
+%   {\ifx\dohyphens\relax
+%      \unexpanded\edef\dohyphens
+%        {\hyphenminoffset\the\hyphenminoffset\relax
+%         \lang_basics_synchronize_min_max}%
+%    \fi
+%    \hyphenminoffset\plusthousand
+%    \lang_basics_synchronize_min_max}
+%
+% \let\dohyphens\relax
+
+% But this one is nicer because we do keep the language set as well as don't mess
+% with the min values (it's more efficient too). We might get some more bits in
+% this mode (engine specific).
+%
+% \let \hyphenationenabledcode   \plusone
+% \let \hyphenationuppercasecode \plustwo
+%
+% \protected\def\dohyphens
+%   {\hyphenationmode \numexpr
+%      \hyphenationenabledcode
+%     +\hyphenationuppercasecode
+%    \relax}
+
+\unexpanded\def\dohyphens{\hyphenationmode\plusthree}
+\unexpanded\def\nohyphens{\hyphenationmode\zerocount}
+
+\unexpanded\def\usehyphensparameter#1%
+  {\edef\p_hyphens{#1\c!hyphens}%
+   \ifx\p_hyhens\v!no
+     \nohyphens
+   \fi}
+
+\dohyphens
+
+%D The rest stays the same as in mkiv:
+
 \unexpanded\def\lesshyphens
   {\advance\hyphenminoffset\plusone
    \lang_basics_synchronize_min_max}
@@ -419,17 +459,6 @@
    \fi
    \lang_basics_synchronize_min_max}
 
-\unexpanded\def\nohyphens % nicer for url's
-  {\ifx\dohyphens\relax
-     \unexpanded\edef\dohyphens
-       {\hyphenminoffset\the\hyphenminoffset\relax
-        \lang_basics_synchronize_min_max}%
-   \fi
-   \hyphenminoffset\plusthousand
-   \lang_basics_synchronize_min_max}
-
-\let\dohyphens\relax
-
 \unexpanded\def\lang_basics_synchronize_min_max % maybe store this at the lua end
   {% these values are stored along with glyph nodes
    \lefthyphenmin \numexpr0\languageparameter\s!lefthyphenmin +\hyphenminoffset\relax
diff --git a/tex/context/base/mkiv/mult-low.lua b/tex/context/base/mkiv/mult-low.lua
index f96199655..9275f2a56 100644
--- a/tex/context/base/mkiv/mult-low.lua
+++ b/tex/context/base/mkiv/mult-low.lua
@@ -509,5 +509,8 @@ return {
         "boxlinels", "boxliners", "boxlinelh", "boxlinerh",
         "boxlinelp", "boxlinerp", "boxlinein",
         "boxrangewd", "boxrangeht", "boxrangedp",
+        --
+        "bitwiseset", "bitwiseand", "bitwiseor", "bitwisexor", "bitwisenot", "bitwisenil",
+        "ifbitwiseand",
     }
 }
diff --git a/tex/context/base/mkiv/mult-prm.lua b/tex/context/base/mkiv/mult-prm.lua
index c56b121c0..b38fcbb7d 100644
--- a/tex/context/base/mkiv/mult-prm.lua
+++ b/tex/context/base/mkiv/mult-prm.lua
@@ -299,9 +299,11 @@ return {
   "hpack",
   "hyphenationbounds",
   "hyphenationmin",
+  "hyphenationmode",
   "hyphenpenaltymode",
   "ifabsdim",
   "ifabsnum",
+  "ifand",
   "ifarguments",
   "ifboolean",
   "ifchkdim",
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index 469411908..d5418eb8e 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index 66ab0b9cf..9ce114508 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
diff --git a/tex/context/base/mkiv/strc-syn.mkiv b/tex/context/base/mkiv/strc-syn.mkiv
index 7f71e88f5..cdede1d0d 100644
--- a/tex/context/base/mkiv/strc-syn.mkiv
+++ b/tex/context/base/mkiv/strc-syn.mkiv
@@ -383,6 +383,7 @@
 
 \startsetups[\??simplelistrenderings::\v!text]
     \begingroup
+    \usehyphensparameter\simplelistparameter
     \usesimpleliststyleandcolor\c!textstyle\c!textcolor
     \simplelistparameter\c!textcommand{\currentsynonymmeaning}%
     \endgroup
@@ -663,7 +664,8 @@
 
 \setupsynonyms
   [\v!abbreviation]
-  [\c!textstyle=\v!capital]
+  [\c!hyphens=\v!no, % new, not yet in setup definitions
+   \c!textstyle=\v!capital]
 
 \definesorting
   [\v!logo]
diff --git a/tex/context/base/mkiv/supp-box.lmt b/tex/context/base/mkiv/supp-box.lmt
index 95af07d9f..ac04e5b08 100644
--- a/tex/context/base/mkiv/supp-box.lmt
+++ b/tex/context/base/mkiv/supp-box.lmt
@@ -257,42 +257,54 @@ local function processsplit(specification)
     local command = specification.command or "ruledhbox"
     local method  = specification.method  or "word"
     local spaced  = specification.spaced
+    if command then
+        command = context[command]
+    end
     if method == "char" or method == "character" then
+        if spaced then
+            spaced = context.space
+        end
         local words = lpegmatch(split_char,str)
         for i=1,#words do
             local word = words[i]
             if word == " " then
                 if spaced then
-                    context.space()
+                    spaced()
                 end
             elseif command then
-                context[command](word)
+                command(word)
             else
                 context(word)
             end
         end
     elseif method == "word" then
+        if spaced then
+            spaced = context.space
+        end
         local words = lpegmatch(split_word,str)
         for i=1,#words do
             local word = words[i]
             if spaced and i > 1 then
-                context.space()
+                spaced()
             end
             if command then
-                context[command](word)
+                command(word)
             else
                 context(word)
             end
         end
     elseif method == "line" then
+        if spaced then
+            spaced = context.par
+        end
         local words = lpegmatch(split_line,str)
         for i=1,#words do
             local word = words[i]
             if spaced and i > 1 then
-                context.par()
+                spaced()
             end
             if command then
-                context[command](word)
+                command(word)
             else
                 context(word)
             end
@@ -1112,7 +1124,6 @@ do
         return { width = w, height = h, depth = d }
     end
 
-
     local getters_one = {
 
         ["wd"]  = function(p,l,found) return dimension_value, p.width end,
@@ -1188,7 +1199,7 @@ do
         implement {
             name      = name,
             public    = true,
-            protected = true,
+         -- protected = true,
             value     = true,
             actions   = function(what) return boxline(action,what) end,
         }
@@ -1198,7 +1209,7 @@ do
         implement {
             name      = name,
             public    = true,
-            protected = true,
+         -- protected = true,
             value     = true,
             actions   = function(what) return boxrange(action,what) end,
         }
@@ -1207,7 +1218,7 @@ do
     implement {
         name      = "boxlines",
         public    = true,
-        protected = true,
+     -- protected = true,
         value     = true,
         actions   = boxlinecount,
     }
diff --git a/tex/context/base/mkiv/syst-lua.lmt b/tex/context/base/mkiv/syst-lua.lmt
index 29f91ae75..effba14d9 100644
--- a/tex/context/base/mkiv/syst-lua.lmt
+++ b/tex/context/base/mkiv/syst-lua.lmt
@@ -334,10 +334,11 @@ do
     local setmacro    = tokens.setters.macro
     local vrbcatcodes = tex.vrbcatcodes
 
-    interfaces.implement {
+    implement {
         name      = "openin",
         public    = true,
-        protected = true,
+     -- protected = true,
+        value     = true,
         actions   = function()
             local n = scaninteger()
                       scankeyword("=")
@@ -355,11 +356,11 @@ do
         end,
     }
 
-    interfaces.implement {
+    implement {
         name      = "closein",
         public    = true,
-        protected = true,
-        value     = "none",
+     -- protected = true,
+        value     = true,
         actions   = function()
             local n = scaninteger()
             local c = channels[n]
@@ -370,11 +371,11 @@ do
         end,
     }
 
-    interfaces.implement {
+    implement {
         name      = "read",
         public    = true,
-        protected = true,
-        value     = "none",
+     -- protected = true,
+        value     = true,
         actions   = function(prefix)
             local n = scaninteger()
                       scankeyword("to")
@@ -389,10 +390,11 @@ do
         end,
     }
 
-    interfaces.implement {
+    implement {
         name      = "readline",
         public    = true,
-        protected = true,
+     -- protected = true,
+        value     = true,
         actions   = function(prefix)
             local n = scaninteger()
                       scankeyword("to")
@@ -407,8 +409,19 @@ do
         end,
     }
 
-    callback.register("if_end_of_file", function(n)
-        return not channels[n]
-    end)
+ -- callback.register("if_end_of_file", function(n)
+ --     return not channels[n]
+ -- end)
+
+    implement {
+        name      = "ifeof",
+        public    = true,
+     -- protected = true,
+        condition = true,
+        actions   = function(prefix)
+            local n = scaninteger()
+            return tokens.values.boolean, not channels[n]
+        end,
+    }
 
 end
diff --git a/tex/context/base/mkiv/toks-ini.lua b/tex/context/base/mkiv/toks-ini.lua
index 350fe6947..834bc773d 100644
--- a/tex/context/base/mkiv/toks-ini.lua
+++ b/tex/context/base/mkiv/toks-ini.lua
@@ -45,7 +45,8 @@ local scan_string      = token.scan_string
 local scan_argument    = token.scan_argument
 local scan_delimited   = token.scan_delimited
 local scan_tokenlist   = token.scan_tokenlist or scan_string
-local scan_int         = token.scan_int
+local scan_integer     = token.scan_integer or token.scan_int
+local scan_cardinal    = token.scan_cardinal
 local scan_code        = token.scan_code
 local scan_token_code  = token.scan_token_code
 local scan_dimen       = token.scan_dimen
@@ -64,9 +65,9 @@ local scan_number      = token.scan_number -- not defined
 local scan_csname      = token.scan_csname
 local scan_real        = token.scan_real
 local scan_float       = token.scan_float
-local scan_luanumber   = token.scan_luanumber   or scan_float -- only lmtx
-local scan_luainteger  = token.scan_luainteger  or scan_int   -- only lmtx
-local scan_luacardinal = token.scan_luacardinal or scan_int   -- only lmtx
+local scan_luanumber   = token.scan_luanumber   or scan_float    -- only lmtx
+local scan_luainteger  = token.scan_luainteger  or scan_integer  -- only lmtx
+local scan_luacardinal = token.scan_luacardinal or scan_cardinal -- only lmtx
 
 local set_macro        = token.set_macro
 local set_char         = token.set_char
@@ -176,13 +177,14 @@ tokens.scanners = { -- these expand
     glue           = scan_glue,
     gluevalues     = function() return scan_glue(false,false,true) end,
     gluespec       = scan_skip,
-    integer        = scan_int,
+    integer        = scan_integer,
+    cardinal       = scan_cardinal,
     real           = scan_real,
     float          = scan_float,
     luanumber      = scan_luanumber,
     luainteger     = scan_luainteger,
     luacardinal    = scan_luacardinal,
-    count          = scan_int,
+    count          = scan_integer,
     string         = scan_string,
     argument       = scan_argument,
     delimited      = scan_delimited,
diff --git a/tex/context/base/mkiv/toks-scn.lua b/tex/context/base/mkiv/toks-scn.lua
index 4377bece1..b6f4d78d3 100644
--- a/tex/context/base/mkiv/toks-scn.lua
+++ b/tex/context/base/mkiv/toks-scn.lua
@@ -27,6 +27,7 @@ local scanverbatim   = scanners.verbatim
 local scantokenlist  = scanners.tokenlist
 local scantoks       = scanners.toks
 local scaninteger    = scanners.integer
+local scancardinal   = scanners.cardinal
 local scannumber     = scanners.number
 local scankeyword    = scanners.keyword
 local scankeywordcs  = scanners.keywordcs
@@ -219,6 +220,7 @@ local shortcuts = {
     scantokenlist     = scantokenlist,
     scantoks          = scantoks,
     scaninteger       = scaninteger,
+    scancardinal      = scancardinal,
     scannumber        = scannumber,
     scantable         = scantable, -- not directly useable
     scankeyword       = scankeyword,
diff --git a/tex/context/base/mkiv/trac-log.lua b/tex/context/base/mkiv/trac-log.lua
index bb154067f..d20a974e1 100644
--- a/tex/context/base/mkiv/trac-log.lua
+++ b/tex/context/base/mkiv/trac-log.lua
@@ -1154,3 +1154,16 @@ if tex then
     end
 
 end
+
+-- for now ... maybe directly but now we get the context too
+
+if tex.error then
+
+    local texerror   = tex.error
+    local formatters = string.formatters
+
+    function logs.reporterror(fmt,first,...)
+        texerror(first and formatters[fmt](first,...) or fmt)
+    end
+
+end