2021-03-27 19:06:00

19 files changed, 219 insertions, 54 deletions

diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii
index 160f34997..8fd36b843 100644
--- a/tex/context/base/mkii/cont-new.mkii
+++ b/tex/context/base/mkii/cont-new.mkii
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2021.03.25 13:59}
+\newcontextversion{2021.03.27 19:03}
 
 %D This file is loaded at runtime, thereby providing an
 %D excellent place for hacks, patches, extensions and new
diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii
index 99edd79c5..4f3ab902a 100644
--- a/tex/context/base/mkii/context.mkii
+++ b/tex/context/base/mkii/context.mkii
@@ -20,7 +20,7 @@
 %D your styles an modules.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2021.03.25 13:59}
+\edef\contextversion{2021.03.27 19:03}
 
 %D For those who want to use this:
 
diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index b05ae7ee2..cd9476787 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2021.03.25 13:59}
+\newcontextversion{2021.03.27 19:03}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index b1d8cad34..5e957e8bf 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -45,7 +45,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2021.03.25 13:59}
+\edef\contextversion{2021.03.27 19:03}
 
 %D Kind of special:
 
diff --git a/tex/context/base/mkiv/luat-cnf.lua b/tex/context/base/mkiv/luat-cnf.lua
index 16448c429..f61ec43d2 100644
--- a/tex/context/base/mkiv/luat-cnf.lua
+++ b/tex/context/base/mkiv/luat-cnf.lua
@@ -24,7 +24,7 @@ texconfig.expand_depth    =    10000
 texconfig.half_error_line =      125
 texconfig.max_print_line  =   100000
 texconfig.max_strings     =   500000
-texconfig.hash_extra      =   200000
+texconfig.hash_extra      =   250000
 texconfig.function_size   =    32768
 texconfig.properties_size =    10000
 
diff --git a/tex/context/base/mkiv/mtx-context-setters.tex b/tex/context/base/mkiv/mtx-context-setters.tex
new file mode 100644
index 000000000..3201ae3ba
--- /dev/null
+++ b/tex/context/base/mkiv/mtx-context-setters.tex
@@ -0,0 +1,32 @@
+%D \module
+%D   [       file=mtx-context-setters,
+%D        version=0000.00.00,
+%D          title=\CONTEXT\ Extra Trickry,
+%D       subtitle=Show Trackers,
+%D         author=Hans Hagen,
+%D           date=\currentdate,
+%D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
+%C
+%C This module is part of the \CONTEXT\ macro||package and is
+%C therefore copyrighted by \PRAGMA. See mreadme.pdf for
+%C details.
+
+% begin help
+%
+% usage: context --extra=setters [--pattern=] [trackers|directives|experiments]
+%
+% --pattern     : filter by pattern
+% --trackers    : show trackers
+% --directives  : show directives
+% --experiments : show experiments
+%
+% end help
+
+\starttext
+
+\doifdocumentargument {trackers}    { \ctxlua {trackers   .show(environment.arguments.pattern) } }
+\doifdocumentargument {directives}  { \ctxlua {directives .show(environment.arguments.pattern) } }
+\doifdocumentargument {experiments} { \ctxlua {experiments.show(environment.arguments.pattern) } }
+
+\stoptext
+
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index 68e0cd0a1..3f6d1a605 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index 378081cf9..880d4b9c3 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
diff --git a/tex/context/base/mkiv/trac-set.lua b/tex/context/base/mkiv/trac-set.lua
index 3974830fa..4edb5d129 100644
--- a/tex/context/base/mkiv/trac-set.lua
+++ b/tex/context/base/mkiv/trac-set.lua
@@ -165,6 +165,9 @@ local function register_setter(t,what,...)
             end
             local s = fnc -- else wrong reference
             fnc = function(value) set(t,s,value) end
+        elseif typ == "table" then
+            functions.values = fnc
+            fnc = nil
         elseif typ ~= "function" then
             fnc = nil
         end
@@ -216,35 +219,42 @@ local function list_setter(t) -- pattern
     return user, system
 end
 
-local function show_setter(t)
+local function show_setter(t,pattern)
     local list = list_setter(t)
     t.report()
     for k=1,#list do
         local name = list[k]
-        local functions = t.data[name]
-        if functions then
-            local value   = functions.value
-            local default = functions.default
-            local modules = #functions
-            if default == nil then
-                default = "unset"
-            elseif type(default) == "table" then
-                default = concat(default,"|")
-            else
-                default = tostring(default)
+        if not pattern or find(name,pattern) then
+            local functions = t.data[name]
+            if functions then
+                local value   = functions.value
+                local default = functions.default
+                local values  = functions.values
+                local modules = #functions
+                if default == nil then
+                    default = "unset"
+                elseif type(default) == "table" then
+                    default = concat(default,"|")
+                else
+                    default = tostring(default)
+                end
+                if value == nil then
+                    value = "unset"
+                elseif type(value) == "table" then
+                    value = concat(value,"|")
+                else
+                    value = tostring(value)
+                end
+                t.report(name)
+                t.report("    modules : %i",modules)
+                t.report("    default : %s",default)
+                t.report("    value   : %s",value)
+            if values then
+                local v = { } for i=1,#values do v[i] = tostring(values[i]) end
+                t.report("    values  : % t",v)
             end
-            if value == nil then
-                value = "unset"
-            elseif type(value) == "table" then
-                value = concat(value,"|")
-            else
-                value = tostring(value)
+                t.report()
             end
-            t.report(name)
-            t.report("    modules : %i",modules)
-            t.report("    default : %s",default)
-            t.report("    value   : %s",value)
-            t.report()
         end
     end
 end
@@ -272,6 +282,11 @@ local function setter_value(setter,name)
     return d and (d.value or d.default)
 end
 
+local function setter_values(setter,name)
+    local d = setter.data[name]
+    return d and d.values
+end
+
 local function new_setter(name) -- we could use foo:bar syntax (but not used that often)
     local setter -- we need to access it in setter itself
     setter = {
@@ -286,6 +301,7 @@ local function new_setter(name) -- we could use foo:bar syntax (but not used tha
         show     = function(...)         show_setter    (setter,...) end,
         default  = function(...)  return setter_default (setter,...) end,
         value    = function(...)  return setter_value   (setter,...) end,
+        values   = function(...)  return setter_values  (setter,...) end,
     }
     data[name] = setter
     return setter
diff --git a/tex/context/base/mkxl/cont-new.mkxl b/tex/context/base/mkxl/cont-new.mkxl
index b79c1fe25..c09bfb8dc 100644
--- a/tex/context/base/mkxl/cont-new.mkxl
+++ b/tex/context/base/mkxl/cont-new.mkxl
@@ -13,7 +13,7 @@
 
 % \normalend % uncomment this to get the real base runtime
 
-\newcontextversion{2021.03.25 13:59}
+\newcontextversion{2021.03.27 19:03}
 
 %D This file is loaded at runtime, thereby providing an excellent place for hacks,
 %D patches, extensions and new features. There can be local overloads in cont-loc
diff --git a/tex/context/base/mkxl/context.mkxl b/tex/context/base/mkxl/context.mkxl
index 09a0d8dff..3bb67e070 100644
--- a/tex/context/base/mkxl/context.mkxl
+++ b/tex/context/base/mkxl/context.mkxl
@@ -29,7 +29,7 @@
 %D {YYYY.MM.DD HH:MM} format.
 
 \immutable\edef\contextformat {\jobname}
-\immutable\edef\contextversion{2021.03.25 13:59}
+\immutable\edef\contextversion{2021.03.27 19:03}
 
 %overloadmode 1 % check frozen / warning
 %overloadmode 2 % check frozen / error
diff --git a/tex/context/base/mkxl/font-ots.lmt b/tex/context/base/mkxl/font-ots.lmt
index 6d9ee1a00..d9ff6635c 100644
--- a/tex/context/base/mkxl/font-ots.lmt
+++ b/tex/context/base/mkxl/font-ots.lmt
@@ -544,9 +544,9 @@ local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfou
         setchar(start,char)
         return head, start
     end
-    if inhibited(start,stop) then
-        return head, start
-    end
+--     if inhibited(start,stop) then
+--         return head, start
+--     end
     local prev = getprev(start)
     local next = getnext(stop)
     local comp = start
@@ -777,7 +777,12 @@ end
 -- simplified. Anyway, it can be way messier: marks that get removed as well as
 -- marks that are kept.
 
+-- Blocking is nasty: (ff\zwj l) vs (ff\zwj l) vs (ffl) vs (f\zwj fl}
+
 function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skiphash)
+if start and has_glyph_option(start,no_right_ligature_code) then
+    return head, start, false, false
+end
     local current = getnext(start)
     if not current then
         return head, start, false, nil
@@ -786,6 +791,9 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip
     local startchar = getchar(start)
     if skiphash and skiphash[startchar] then
         while current do
+if current and has_glyph_option(current,no_left_ligature_code) then
+    break
+end
             local nxt, char = isnextchar(current,currentfont,currentdynamic,currentscale,currentxscale,currentyscale)
             if char then
                 local lg = ligature[char]
@@ -819,6 +827,9 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip
         local discfound = false
         local hasmarks  = marks[startchar]
         while current do
+if current and has_glyph_option(current,no_left_ligature_code) then
+    break
+end
             local nxt, char, id = isnextchar(current,currentfont,currentdynamic,currentscale,currentxscale,currentyscale)
             if char then
                 if skiphash and skiphash[char] then
@@ -1475,6 +1486,9 @@ assume rather stupid ligatures (no complex disc nodes).</p>
 -- compare to handlers.gsub_ligature which is more complex ... why
 
 function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup,rlmode,skiphash,chainindex)
+if start and has_glyph_option(start,no_right_ligature_code) then
+    return head, start, false, 0, false
+end
     local mapping = currentlookup.mapping
     if mapping == nil then
         mapping = getmapping(dataset,sequence,currentlookup)
@@ -1493,6 +1507,9 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup
             local last            = stop
             local nofreplacements = 1
             while current do
+if current and has_glyph_option(current,no_left_ligature_code) then
+    break
+end
                 -- todo: ischar ... can there really be disc nodes here?
 --                 local id = getid(current)
 --                 if id == disc_code then
diff --git a/tex/context/base/mkxl/lang-hup.lmt b/tex/context/base/mkxl/lang-hup.lmt
index 41548d84d..c42f0ffd8 100644
--- a/tex/context/base/mkxl/lang-hup.lmt
+++ b/tex/context/base/mkxl/lang-hup.lmt
@@ -56,7 +56,7 @@ local function collect(head)
     while head do
         local nxt, char, id = isnextglyph(head)
         if char then
-            local u = characters[id][char].unicode
+            local u = characters[id][char].unicode -- we could cache it
             if type(u) == "table" then
                 for i=1,#u do
                  -- word[#word+1] = utfchar(u[i])
@@ -196,19 +196,36 @@ trackers.register("hyphenation.applied.console", function(v)
     end
 end)
 
+-- local c, f = isglyph(current)
+-- local char = chardata[f][c]
+-- if char and type(char.unicode) == "table" then -- hackery test
+
+local ligature_code = 0x8000 + nodes.glyphcodes.ligature
+local ligature_mode = false
+
+local color_n = { "red",     "green",     "blue"     }
+local color_l = { "darkred", "darkgreen", "darkblue" }
+
 function nodes.handlers.visualizehyphenation(head)
     for current, id, subtype, list in nexthlist, head do
         if list and subtype == line_code then
-            for n in nextglyph, list do
-                local d = getdiscpart(n)
-                if d == 0 then
-                    -- nothing
-                elseif d == 1 then
-                    setcolor(n,"red")   -- pre
-                elseif d == 2 then
-                    setcolor(n,"blue")  -- post
-                elseif d == 3 then
-                    setcolor(n,"green") -- replace
+            if ligature_mode then
+                for n in nextglyph, list do
+                    local d = getdiscpart(n)
+                    if d > 0 and d < 4 then
+                        if getsubtype(n) == ligature_code then
+                            setcolor(n,color_l[d])
+                        else
+                            setcolor(n,color_n[d])
+                        end
+                    end
+                end
+            else
+                for n in nextglyph, list do
+                    local d = getdiscpart(n)
+                    if d > 0 and d < 4 then
+                        setcolor(n,color_n[d])
+                    end
                 end
             end
         end
@@ -216,6 +233,7 @@ function nodes.handlers.visualizehyphenation(head)
     return head
 end
 
-trackers.register("hyphenation.applied.visualize", function(v)
+trackers.register("hyphenation.applied.visualize", { true, false, "ligatures" }, function(v)
     setaction("finalizers","nodes.handlers.visualizehyphenation",v)
+    ligature_mode = v == "ligatures"
 end)
diff --git a/tex/context/base/mkxl/lang-ini.lmt b/tex/context/base/mkxl/lang-ini.lmt
index 5f2dd1a55..2e2d5e57d 100644
--- a/tex/context/base/mkxl/lang-ini.lmt
+++ b/tex/context/base/mkxl/lang-ini.lmt
@@ -524,7 +524,7 @@ function languages.postexhyphenchar(what) return postexhyphenchar(tolang(what))
 -- e['user-friendly'] = 'user=friend-ly'
 -- e['exceptionally-friendly'] = 'excep-tionally=friend-ly'
 
-local invalid = { "{", "}", "-" }
+local invalid = { "{", "}", "(", ")", "-", " " }
 
 local function collecthjcodes(data,str)
     local found = data.extras and data.extras.characters or { }
@@ -564,6 +564,15 @@ function languages.setexceptions(tag,str)
     end
 end
 
+local function setwordhandler(tag,action)
+    local data, instance = resolve(tag)
+    if data then
+        instance:setwordhandler(action)
+    end
+end
+
+languages.setwordhandler = setwordhandler
+
 function languages.hyphenate(tag,str)
     -- todo: does this still work?
     local data, instance = resolve(tag)
@@ -574,6 +583,75 @@ function languages.hyphenate(tag,str)
     end
 end
 
+-- This code is here for some testing (and discussion) but it might end up in its
+-- own module. I wrote it after listening to the end March 2021 live concert of
+-- Mandoki Soulmates: Hungarian Pictures (music is the greatest unifier) with his
+-- usual incredible international lineup. After that, and realizing that we needed
+-- to deal better with some language issues as follow up on a mailing list thread, I
+-- needed only a few loops of relistening the concert to implement it. In
+-- restrospect this was a language feature that should have been there a while ago.
+
+do
+
+    local nuts       = nodes.nuts
+    local nextglyph  = nuts.traversers.glyph
+    local setoptions = nuts.setoptions
+    local glyphoptioncodes = tex.glyphoptioncodes
+
+    local cache = table.setmetatableindex(function(t,k)
+        local v = 0
+        for s in gmatch(k,"%w+") do
+            local o = glyphoptioncodes[s]
+            if o then
+                v = v | o
+            end
+        end
+        t[k] = v
+        return v
+    end)
+
+    local function checkglyphoptions(options)
+        for word, list in next, options do
+            if type(list) == "string" then
+                options[word] = options[list]
+            else
+                for index, option in next, list do
+                    if type(option) == "string" then
+                        list[index] = cache[option]
+                    end
+                end
+            end
+        end
+    end
+
+    function languages.setoptionhandler(tag,options)
+        if type(options) == "table" then
+            checkglyphoptions(options) -- checks in place!
+            setwordhandler(tag,
+                function(n,word,length,first,last)
+                    local o = options[word]
+                    if o then
+                        local index = 0
+                        for g, c in nextglyph, first do
+                            index = index + 1
+                            local t = o[index]
+                            if t then
+                                setoptions(g,t)
+                            end
+                            if g == last then
+                                break
+                            end
+                        end
+                    end
+                    return false
+            end)
+        else
+            setwordhandler(tag)
+        end
+    end
+
+end
+
 -- hyphenation.define        ("zerolanguage")
 -- hyphenation.loadpatterns  ("zerolanguage") -- else bug
 -- hyphenation.loadexceptions("zerolanguage") -- else bug
diff --git a/tex/context/base/mkxl/luat-cod.lmt b/tex/context/base/mkxl/luat-cod.lmt
index 71f03c0e2..9a74e4238 100644
--- a/tex/context/base/mkxl/luat-cod.lmt
+++ b/tex/context/base/mkxl/luat-cod.lmt
@@ -33,7 +33,7 @@ texconfig.max_print_line  =   100000
 texconfig.max_strings     =   500000
 texconfig.max_pool        = 10000000
 
-texconfig.hash_extra      =   200000
+texconfig.hash_extra      =   250000
 
 texconfig.expand_depth    =    10000
 texconfig.function_size   =    32768
diff --git a/tex/context/base/mkxl/mlib-svg.lmt b/tex/context/base/mkxl/mlib-svg.lmt
index 8409be11b..885d2cd77 100644
--- a/tex/context/base/mkxl/mlib-svg.lmt
+++ b/tex/context/base/mkxl/mlib-svg.lmt
@@ -1677,12 +1677,13 @@ do
     local f_rotation_stop   = formatters[") rotatedaround((0,0),-angle((%N,%N))) ;"]
     local f_rotation_angle  = formatters[") rotatedaround((0,0),-%N) ;"]
 
+    local s_offset_start    = "draw image ( "
+    local f_offset_stop     = formatters[") shifted (%N,%N) ;"]
+    local s_size_start      = "draw image ( "
+    local f_size_stop       = formatters[") xysized (%N,%N) ;"]
+
     local handleoffset, handlesize do
 
-        local s_offset_start = "draw image ( "
-        local f_offset_stop  = formatters[") shifted (%N,%N) ;"]
-        local s_size_start   = "draw image ( "
-        local f_size_stop    = formatters[") xysized (%N,%N) ;"]
 
         handleoffset = function(at)
             local x = asnumber_vx(rawget(at,"x"))
diff --git a/tex/context/base/mkxl/trac-vis.lmt b/tex/context/base/mkxl/trac-vis.lmt
index 44723e38f..8b2380b70 100644
--- a/tex/context/base/mkxl/trac-vis.lmt
+++ b/tex/context/base/mkxl/trac-vis.lmt
@@ -889,6 +889,8 @@ local ruledglyph do
     -- have a frozen color anyway or i need a more detailed cache .. below is a more
     -- texie approach
 
+ -- local ligature_code = 0x8000 + nodes.glyphcodes.ligature
+
     ruledglyph = function(head,current,previous) -- wrong for vertical glyphs
         local wd = getwidth(current)
         if wd ~= 0 then
@@ -905,6 +907,7 @@ local ruledglyph do
                 line   = linewidth,
                 type   = "box",
             },new_kern(-wd))
+         -- if getsubtype(n) == ligature_code then
             local c, f = isglyph(current)
             local char = chardata[f][c]
             if char and type(char.unicode) == "table" then -- hackery test
diff --git a/tex/context/modules/mkiv/x-mathml.mkiv b/tex/context/modules/mkiv/x-mathml.mkiv
index 7bfc75c50..58669eb2a 100644
--- a/tex/context/modules/mkiv/x-mathml.mkiv
+++ b/tex/context/modules/mkiv/x-mathml.mkiv
@@ -797,7 +797,7 @@
 
 \startxmlsetups mml:apply:inverse
     \settrue\xmlinversefunction
-    \xmlsetup{#1}{mml:\xmlfilter{#1}{/mml:apply/*[2]/name()}}
+    \xmlsetup{#1}{mml:\xmlfilter{#1}{/mml:apply/*[2]/tag()}}
 \stopxmlsetups
 
 % condition
@@ -1219,7 +1219,7 @@
 \startxmlsetups mml:factorof   \xmlsetup{#1}{mml:relation} \stopxmlsetups
 
 \startxmlsetups mml:relation
-    \edef\mmlapplyaction{\xmlfilter{#1}{/*/name()}}
+    \edef\mmlapplyaction{\xmlfilter{#1}{/*/tag()}}
     \MMLcreset \xmlsetup{#1}{mml:relation:\xmlattdef{#1}{align}{\MMLrelationalign}}
 \stopxmlsetups
 
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index c616a4e68..c981601b9 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date  : 2021-03-25 13:59
+-- merge date  : 2021-03-27 19:03
 
 do -- begin closure to overcome local limits and interference