2015-01-01 13:17:00

12 files changed, 285 insertions, 400 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index e642b5be1..fa756369d 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2014.12.31 20:42}
+\newcontextversion{2015.01.01 13:15}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 5b1834d62..abae22c5d 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index abac94514..2f3248eb6 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -28,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2014.12.31 20:42}
+\edef\contextversion{2015.01.01 13:15}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/publ-aut.lua b/tex/context/base/publ-aut.lua
index 01b41c167..f203d1f79 100644
--- a/tex/context/base/publ-aut.lua
+++ b/tex/context/base/publ-aut.lua
@@ -384,16 +384,6 @@ function commands.btxauthor(dataset,tag,field,settings)
     end
 end
 
--- We can consider creating a hashtable key -> entry but I wonder if it ever
--- pays off.
-
-local compare  = sorters.comparers.basic -- (a,b)
------ compare  = sorters.basicsorter -- (a,b)
-local strip    = sorters.strip
-local splitter = sorters.splitters.utf
-
--- authors(s) | year | journal | title | pages
-
 local function components(snippet,short)
     local vons       = snippet.vons
     local surnames   = snippet.surnames
@@ -445,162 +435,25 @@ local function writer(key,snippets)
     return concat(snippets," ",1,s)
 end
 
-local function newsplitter(splitter)
-    return table.setmetatableindex({},function(t,k) -- could be done in the sorter but seldom that many shared
-        local v = splitter(k,true)                  -- in other cases
-        t[k] = v
-        return v
-    end)
-end
+
+publications.writers   .author = writer
+publications.casters   .author = splitauthorstring
+publications.components.author = components
+
+-- sharedmethods.author = {
+--     { field = "key",     default = "",     unknown = "" },
+--     { field = "author",  default = "",     unknown = "" },
+--     { field = "title",   default = "",     unknown = "" },
+-- }
 
 -- Analysis of the APA by Alan:
 --
 -- first : key author editor publisher title           journal volume number pages
 -- second: year suffix                 title month day journal volume number
 
--- local function indexer(dataset,list,method)
---     local current  = datasets[dataset]
---     local luadata  = current.luadata
---     local details  = current.details
---     local result   = { }
---     local splitted = newsplitter(splitter) -- saves mem
---     local snippets = { } -- saves mem
---     local field    = "author" -- todo
---     for i=1,#list do
---         -- either { tag, tag, ... } or { { tag, index }, { tag, index } }
---         local li    = list[i]
---         local tag   = type(li) == "string" and li or li[1]
---         local index = tostring(i)
---         local entry = luadata[tag]
---         if entry then
---             local value   = getcasted(current,tag,field) or ""
---             local mainkey = writer(value,snippets)
---             local detail  = details[tag]
---             result[i] = {
---                 index  = i,
---                 split  = {
---                     splitted[entry.key           or ""    ],
---                     splitted[strip(mainkey)               ],
---                     splitted[entry.year          or "9998"],
---                     splitted[detail.suffix       or " "   ],
---                     splitted[entry.month         or "13"  ],
---                     splitted[entry.day           or "32"  ],
---                     splitted[strip(entry.journal or ""   )],
---                     splitted[strip(entry.volume  or ""   )],
---                     splitted[strip(entry.number  or ""   )],
---                     splitted[strip(entry.title   or ""   )],
---                     splitted[entry.pages         or ""    ],
---                     splitted[index],
---                 },
---             }
---         else
---             result[i] = {
---                 index  = i,
---                 split  = {
---                     splitted[""],     -- key
---                     splitted[""],     -- mainkey
---                     splitted["9999"], -- year
---                     splitted[" "],    -- suffix
---                     splitted["14"],   -- month
---                     splitted["33"],   -- day
---                     splitted[""],     -- journal
---                     splitted[""],     -- volume
---                     splitted[""],     -- number
---                     splitted[""],     -- title
---                     splitted[""],     -- pages
---                     splitted[index],  -- index
---                 },
---             }
---         end
---     end
---     return result
--- end
-
--- if needed we can optimize this one: chekc if it's detail or something else
--- and use direct access, but in practice it's fast enough
-
-local template = [[
-local type, tostring = type, tostring
-
-local writers  = publications.writers
-local datasets = publications.datasets
-local getter   = publications.getfaster -- (current,data,details,field,categories,types)
-local strip    = sorters.strip
-local splitter = sorters.splitters.utf
-
-local function newsplitter(splitter)
-    return table.setmetatableindex({},function(t,k) -- could be done in the sorter but seldom that many shared
-        local v = splitter(k,true)                  -- in other cases
-        t[k] = v
-        return v
-    end)
-end
-
-return function(dataset,list,method) -- indexer
-    local current       = datasets[dataset]
-    local luadata       = current.luadata
-    local details       = current.details
-    local specification = publications.currentspecification
-    local categories    = specification.categories
-    local types         = specification.types
-    local splitted      = newsplitter(splitter) -- saves mem
-    local snippets      = { } -- saves mem
-    local result        = { }
-
-%helpers%
-
-    for i=1,#list do
-        -- either { tag, tag, ... } or { { tag, index }, { tag, index } }
-        local li    = list[i]
-        local tag   = type(li) == "string" and li or li[1]
-        local index = tostring(i)
-        local entry = luadata[tag]
-        if entry then
-            local detail  = details[tag]
-            result[i] = {
-                index  = i,
-                split  = {
-                    splitted[entry.key or ""],
-
-%getters%
-
-                    splitted[index],
-                },
-            }
-        else
-            result[i] = {
-                index  = i,
-                split  = {
-                    splitted[""],
-
-%unknowns%
-
-                    splitted[index],
-                },
-            }
-        end
-    end
-    return result
-end
-]]
-
-local f_getter = formatters["splitted[strip(getter(current,entry,detail,%q,categories,types) or %q)], -- %s"]
-local f_writer = formatters["splitted[strip(writer_%s(getter(current,entry,detail,%q,categories,types) or %q,snippets))], -- %s"]
-local f_helper = formatters["local writer_%s = writers[%q] -- %s: %s"]
-local f_value  = formatters["splitted[%q], -- %s"]
-
-local function indexer(dataset,list,method)
-    local writers       = publications.writers
-    local specification = publications.currentspecification
-    ----- categories    = specification.categories
-    local types         = specification.types
-
-    local getters  = { }
-    local unknowns = { }
-    local helpers  = { }
-
-    local sequence = {
-        -- always key first (or should that be made explicit)
+publications.sortmethods.authoryear = {
+    sequence = {
+        { field = "key",     default = "",     unknown = "" },
         { field = "author",  default = "",     unknown = "" },
         { field = "year",    default = "9998", unknown = "9999" },
         { field = "suffix",  default = " ",    unknown = " " },
@@ -611,68 +464,5 @@ local function indexer(dataset,list,method)
         { field = "number",  default = "",     unknown = "" },
         { field = "title",   default = "",     unknown = "" },
         { field = "pages",   default = "",     unknown = "" },
-        -- always index last (to catch duplicates)
-    }
-
-    for i=1,#sequence do
-        local step    = sequence[i]
-        local field   = step.field   or "?"
-        local default = step.default or ""
-        local unknown = step.unknown or ""
-        local fldtype = types[field]
-        local writer  = fldtype and writers[fldtype]
-        if writer then
-            local h = #helpers + 1
-            getters[i] = f_writer(h,field,default,field)
-            helpers[h] = f_helper(h,fldtype,field,fldtype)
-        else
-            getters[i] = f_getter(field,default,field)
-        end
-        unknowns[i] = f_value(unknown,field)
-    end
-
-    local code = utilities.templates.replace(template, {
-        helpers  = concat(helpers, "\n"),
-        getters  = concat(getters, "\n"),
-        unknowns = concat(unknowns,"\n"),
-    })
-
- -- print(code)
-
-    local action, error = loadstring(code)
-    if type(action) == "function" then
-        action = action()
-    end
-    if type(action) == "function" then
-        return action(dataset,list,method) or { }
-    else
-        return { }
-    end
-
-end
-
--- local function sorted(dataset,list) -- experimental
---     local valid = indexer(dataset,list,sorttype)
---     if #valid == 0 or #valid ~= #list then
---         return list
---     else
---         sorters.sort(valid,function(a,b) return a ~= b and compare(a,b) == -1 end)
---         for i=1,#valid do
---             valid[i] = valid[i].index
---         end
---         return valid
---     end
--- end
-
-local function sorted(dataset,valid) -- experimental
-    sorters.sort(valid,compare)
-    return valid
-end
-
--- made public
-
-publications.indexers  .author = indexer
-publications.writers   .author = writer
-publications.sorters   .author = sorted
-publications.casters   .author = splitauthorstring
-publications.components.author = components
+    },
+}
diff --git a/tex/context/base/publ-dat.lua b/tex/context/base/publ-dat.lua
index dcaabb015..e360ca1fe 100644
--- a/tex/context/base/publ-dat.lua
+++ b/tex/context/base/publ-dat.lua
@@ -72,11 +72,11 @@ publications.loaders    = loaders
 local casters           = { }
 publications.casters    = casters
 
-local sorters           = { }
-publications.sorters    = sorters
-
-local indexers          = { }
-publications.indexers   = indexers
+-- local sorters           = { }
+-- publications.sorters    = sorters
+--
+-- local indexers          = { }
+-- publications.indexers   = indexers
 
 local components        = { }
 publications.components = components -- register components
diff --git a/tex/context/base/publ-ini.lua b/tex/context/base/publ-ini.lua
index 120957034..e09ba3a38 100644
--- a/tex/context/base/publ-ini.lua
+++ b/tex/context/base/publ-ini.lua
@@ -69,13 +69,9 @@ local v_middle           = variables.middle
 local v_inbetween        = variables.inbetween
 local v_yes              = variables.yes
 local v_all              = variables.all
-local v_short            = variables.short
 local v_cite             = variables.cite
 local v_default          = variables.default
-local v_reference        = variables.reference
 local v_dataset          = variables.dataset
-local v_author           = variables.author or "author"
-local v_editor           = variables.editor or "editor"
 
 local numbertochar       = converters.characters
 
@@ -84,10 +80,9 @@ local logspushtarget     = logs.pushtarget
 local logspoptarget      = logs.poptarget
 local csname_id          = token.csname_id
 
-local basicsorter        = sorters.basicsorter -- (a,b)
-local sortcomparer       = sorters.comparers.basic -- (a,b)
-local sortstripper       = sorters.strip
-local sortsplitter       = sorters.splitters.utf
+----- basicsorter        = sorters.basicsorter -- (a,b)
+----- sortstripper       = sorters.strip
+----- sortsplitter       = sorters.splitters.utf
 
 local manipulators       = typesetters.manipulators
 local splitmanipulation  = manipulators.splitspecification
@@ -891,7 +886,7 @@ do
                                     s[#s+1] = { tag, year, u, i }
                                 end
                             else
-                                report("author typecast expected for fiel %a",field)
+                                report("author typecast expected for field %a",field)
                             end
                         else
                             --- no spec so let's forget about it
@@ -1503,158 +1498,6 @@ do
         filtermethod(dataset,rendering,keyword)
     end
 
-    -- experiment
-
-    local splitspec = lpeg.splitat(S(":."))
-    local splitter  = sorters.splitters.utf
-    local strip     = sorters.strip
-
-    local function newsplitter(splitter)
-        return setmetatableindex({},function(t,k) -- could be done in the sorter but seldom that many shared
-            local v = splitter(k,true)                  -- in other cases
-            t[k] = v
-            return v
-        end)
-    end
-
-    local template = [[
-        local strip   = sorters.strip
-        local writers = publications.writers
-        return function(entry,detail,splitted,i) -- snippets
-            return {
-                index = i,
-                split = { %s, splitted[tostring(i)] }
-            }
-        end
-    ]]
-
-    local function byspec(dataset,list,method) -- todo: yearsuffix
-        local luadata  = datasets[dataset].luadata
-        local details  = datasets[dataset].details
-        local result   = { }
-        local splitted = newsplitter(splitter) -- saves mem
-     -- local snippets = { } -- saves mem
-        local fields   = settings_to_array(method)
-        for i=1,#fields do
-            local f = settings_to_array(fields[i])
-            local r = { }
-            for i=1,#f do
-                local a, b = lpegmatch(splitspec,f[i])
-                if b then
-                    if a == "detail" or a == "entry" then
-                        local t = currentspecification.types[b]
-                        local w = t and writers[t]
-                        if w then
-                            r[#r+1] = formatters["(%s.%s and writers[%q](%s.%s))"](a,b,t,a,b)
-                        else
-                            r[#r+1] = formatters["%s.%s"](a,b,a,b)
-                        end
-                    end
-                elseif a then
-                    r[#r+1] = formatters["%s"](a)
-                end
-            end
-            r[#r+1] = '""'
-            fields[i] = "splitted[strip(" .. concat(r," or ") .. ")]"
-        end
-        local action  = formatters[template](concat(fields,", "))
-        local prepare = loadstring(action)
-        if prepare then
-            prepare = prepare()
-            local dummy = { }
-            for i=1,#list do
-                -- either { tag, tag, ... } or { { tag, index }, { tag, index } }
-                local li     = list[i]
-                local tag    = type(li) == "string" and li or li[1]
-                local entry  = luadata[tag]
-                local detail = details[tag]
-                if entry and detail then
-                    result[i] = prepare(entry,detail,splitted,i) -- ,snippets)
-                else
-                    result[i] = prepare(dummy,dummy,splitted,i) -- ,snippets)
-                end
-            end
-        end
-        return result
-    end
-
-    lists.sorters = {
-        [v_short] = function(dataset,rendering,list)
-            local shorts = rendering.shorts
-            local function compare(a,b)
-                local aa, bb = a and a[1], b and b[1]
-                if aa and bb then
-                    aa, bb = shorts[aa], shorts[bb]
-                    return aa and bb and aa < bb
-                end
-                return false
-            end
-            sort(list,compare)
-        end,
-        [v_reference] = function(dataset,rendering,list)
-            local function compare(a,b)
-                local aa, bb = a and a[1], b and b[1]
-                if aa and bb then
-                    return aa and bb and aa < bb
-                end
-                return false
-            end
-            sort(list,compare)
-        end,
-        [v_dataset] = function(dataset,rendering,list)
-            local function compare(a,b)
-                local aa, bb = a and a[1], b and b[1]
-                if aa and bb then
-                    aa, bb = list[aa].index or 0, list[bb].index or 0
-                    return aa and bb and aa < bb
-                end
-                return false
-            end
-            sort(list,compare)
-        end,
-        [v_default] = function(dataset,rendering,list,sorttype) -- experimental
-            if sorttype == "" or sorttype == v_default then
-                local function compare(a,b)
-                    local aa, bb = a and a[3], b and b[3]
-                    if aa and bb then
-                        return aa and bb and aa < bb
-                    end
-                    return false
-                end
-                sort(list,compare)
-            else
-                local valid = byspec(dataset,list,sorttype)
-                if #valid == 0 or #valid ~= #list then
-                    -- nothing to sort
-                else
-                    -- if needed we can wrap compare and use the list directly but this is cleaner
-                    sorters.sort(valid,sortcomparer)
-                    for i=1,#valid do
-                        local v = valid[i]
-                        valid[i] = list[v.index]
-                    end
-                    return valid
-                end
-            end
-        end,
-        [v_author] = function(dataset,rendering,list)
-            -- there is no real need to go vi aindex as the list itself can be sorted ... todo
-            local valid = publications.indexers.author(dataset,list)
-            if #valid == 0 or #valid ~= #list then
-                -- nothing to sort
-            else
-                -- if needed we can wrap compare and use the list directly but this is cleaner
---                 sorters.sort(valid,publications.sorters.author)
-                local valid = publications.sorters.author(dataset,valid)
-                for i=1,#valid do
-                    local v = valid[i]
-                    valid[i] = list[v.index]
-                end
-                return valid
-            end
-        end,
-    }
-
     -- for determining width
 
     local lastreferencenumber = 0 -- document wide
@@ -1666,7 +1509,7 @@ do
         local forceall  = rendering.criterium == v_all
         local repeated  = rendering.repeated == v_yes
         local sorttype  = rendering.sorttype or v_default
-        local sorter    = lists.sorters[sorttype] or lists.sorters[v_default]
+        local sorter    = lists.sorters[sorttype]
         local current   = datasets[dataset]
         local luadata   = current.luadata
         local details   = current.details
diff --git a/tex/context/base/publ-ini.mkiv b/tex/context/base/publ-ini.mkiv
index aaee08d53..801a274d3 100644
--- a/tex/context/base/publ-ini.mkiv
+++ b/tex/context/base/publ-ini.mkiv
@@ -40,6 +40,7 @@
 
 \registerctxluafile{publ-dat}{1.001}
 \registerctxluafile{publ-ini}{1.001}
+\registerctxluafile{publ-sor}{1.001}
 \registerctxluafile{publ-aut}{1.001}
 \registerctxluafile{publ-usr}{1.001}
 \registerctxluafile{publ-oth}{1.001} % this could become an option
diff --git a/tex/context/base/publ-sor.lua b/tex/context/base/publ-sor.lua
new file mode 100644
index 000000000..5f0764472
--- /dev/null
+++ b/tex/context/base/publ-sor.lua
@@ -0,0 +1,243 @@
+if not modules then modules = { } end modules ['publ-sor'] = {
+    version   = 1.001,
+    comment   = "this module part of publication support",
+    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+    copyright = "PRAGMA ADE / ConTeXt Development Team",
+    license   = "see context related readme files"
+}
+
+-- if needed we can optimize this one: chekc if it's detail or something else
+-- and use direct access, but in practice it's fast enough
+
+local type         = type
+local concat       = table.concat
+local formatters   = string.formatters
+local compare      = sorters.comparers.basic -- (a,b)
+
+local publications = publications
+local writers      = publications.writers
+
+local variables    = interfaces.variables
+local v_short      = variables.short
+local v_default    = variables.reference
+local v_reference  = variables.reference
+local v_dataset    = variables.dataset
+
+local report       = logs.reporter("publications","sorters")
+
+local trace_sorters  trackers.register("publications.sorters",function(v) trace_sorters = v end)
+
+-- authors(s) | year | journal | title | pages
+
+local template = [[
+local type, tostring = type, tostring
+
+local writers  = publications.writers
+local datasets = publications.datasets
+local getter   = publications.getfaster -- (current,data,details,field,categories,types)
+local strip    = sorters.strip
+local splitter = sorters.splitters.utf
+
+local function newsplitter(splitter)
+    return table.setmetatableindex({},function(t,k) -- could be done in the sorter but seldom that many shared
+        local v = splitter(k,true)                  -- in other cases
+        t[k] = v
+        return v
+    end)
+end
+
+return function(dataset,list,method) -- indexer
+    local current       = datasets[dataset]
+    local luadata       = current.luadata
+    local details       = current.details
+    local specification = publications.currentspecification
+    local categories    = specification.categories
+    local types         = specification.types
+    local splitted      = newsplitter(splitter) -- saves mem
+    local snippets      = { } -- saves mem
+    local result        = { }
+
+%helpers%
+
+    for i=1,#list do
+        -- either { tag, tag, ... } or { { tag, index }, { tag, index } }
+        local li    = list[i]
+        local tag   = type(li) == "string" and li or li[1]
+        local index = tostring(i)
+        local entry = luadata[tag]
+        if entry then
+            local detail  = details[tag]
+            result[i] = {
+                index  = i,
+                split  = {
+
+%getters%
+
+                },
+            }
+        else
+            result[i] = {
+                index  = i,
+                split  = {
+
+%unknowns%
+
+                },
+            }
+        end
+    end
+    return result
+end
+]]
+
+local f_getter = formatters["splitted[strip(getter(current,entry,detail,%q,categories,types) or %q)], -- %s"]
+local f_writer = formatters["splitted[strip(writer_%s(getter(current,entry,detail,%q,categories,types) or %q,snippets))], -- %s"]
+local f_helper = formatters["local writer_%s = writers[%q] -- %s: %s"]
+local f_value  = formatters["splitted[%q], -- %s"]
+local s_index  = "splitted[index], -- the order in the list, always added"
+
+-- there is no need to cache this in specification
+
+local sharedmethods      = { }
+publications.sortmethods = sharedmethods
+
+local function sortsequence(dataset,list,sorttype)
+    local specification = publications.currentspecification
+    local types         = specification.types
+    local sortmethods   = specification.sortmethods
+    local method        = sortmethods and sortmethods[sorttype] or sharedmethods[sorttype]
+    local sequence      = method and method.sequence
+
+    if not sequence and type(sorttype) == "string" then
+        local list = utilities.parsers.settings_to_array(sorttype)
+        if #list > 0 then
+            sequence = { }
+            for i=1,#list do
+                sequence[i] = { field = list[i] }
+            end
+        end
+        if trace_sorters then
+            report("creating sequence from method %a",sorttype)
+        end
+    end
+
+    if sequence then
+
+        local getters  = { }
+        local unknowns = { }
+        local helpers  = { }
+
+        if trace_sorters then
+            report("initializing method %a",sorttype)
+        end
+
+        for i=1,#sequence do
+            local step    = sequence[i]
+            local field   = step.field   or "?"
+            local default = step.default or ""
+            local unknown = step.unknown or ""
+            local fldtype = types[field]
+            local writer  = fldtype and writers[fldtype]
+
+            if trace_sorters then
+                report("% 3i : field %a, type %a, default %a, unknown %a",i,field,fldtype,default,unknown)
+            end
+
+            if writer then
+                local h = #helpers + 1
+                getters[i] = f_writer(h,field,default,field)
+                helpers[h] = f_helper(h,fldtype,field,fldtype)
+            else
+                getters[i] = f_getter(field,default,field)
+            end
+            unknowns[i] = f_value(unknown,field)
+        end
+
+        unknowns[#unknowns+1] = s_index
+        getters [#getters +1] = s_index
+
+        local code = utilities.templates.replace(template, {
+            helpers  = concat(helpers, "\n"),
+            getters  = concat(getters, "\n"),
+            unknowns = concat(unknowns,"\n"),
+        })
+
+     -- print(code)
+
+        local action, error = loadstring(code)
+        if type(action) == "function" then
+            action = action()
+        end
+        if type(action) == "function" then
+            local valid = action(dataset,list,method)
+            if valid and #valid > 0 then
+                sorters.sort(valid,compare)
+                return valid
+            end
+        end
+    else
+        report("invalid sort method %a",sorttype)
+    end
+
+end
+
+local sorters = {
+    [v_short] = function(dataset,rendering,list)
+        local shorts = rendering.shorts
+        local function compare(a,b)
+            local aa, bb = a and a[1], b and b[1]
+            if aa and bb then
+                aa, bb = shorts[aa], shorts[bb]
+                return aa and bb and aa < bb
+            end
+            return false
+        end
+        sort(list,compare)
+    end,
+    [v_reference] = function(dataset,rendering,list)
+        local function compare(a,b)
+            local aa, bb = a and a[1], b and b[1]
+            if aa and bb then
+                return aa and bb and aa < bb
+            end
+            return false
+        end
+        sort(list,compare)
+    end,
+    [v_dataset] = function(dataset,rendering,list)
+        local function compare(a,b)
+            local aa, bb = a and a[1], b and b[1]
+            if aa and bb then
+                aa, bb = list[aa].index or 0, list[bb].index or 0
+                return aa and bb and aa < bb
+            end
+            return false
+        end
+        sort(list,compare)
+    end,
+    [v_default] = function(dataset,rendering,list,sorttype) -- experimental
+        if sorttype == "" or sorttype == v_default then
+            local function compare(a,b)
+                local aa, bb = a and a[3], b and b[3]
+                if aa and bb then
+                    return aa and bb and aa < bb
+                end
+                return false
+            end
+            sort(list,compare)
+        else
+            local valid = sortsequence(dataset,list,sorttype)
+            if #valid > 0 then
+                for i=1,#valid do
+                    local v = valid[i]
+                    valid[i] = list[v.index]
+                end
+                return valid
+            end
+        end
+    end
+}
+
+table.setmetatableindex(sorters,function(t,k) return t[v_default] end)
+
+publications.lists.sorters = sorters
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 6ded1f9a4..8cca48991 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index bb67d0ed5..23b6fa723 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/tabl-ntb.mkiv b/tex/context/base/tabl-ntb.mkiv
index 04d4cadbd..77b0be2fa 100644
--- a/tex/context/base/tabl-ntb.mkiv
+++ b/tex/context/base/tabl-ntb.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-%D This module as a more modern variant in xtables but as we follow a bit
+%D This module has a more modern variant in xtables but as we follow a bit
 %D different approach with settings there, this mechanism will stay. In fact
 %D each of them has its advantages. This module could be sped up a bit and made
 %D more efficient by delegating some housekeeping to \LUA\ but it's not worth
@@ -344,6 +344,7 @@
 \newif\ifenableTBLbreak     \enableTBLbreakfalse
 \newif\ifmultipleTBLheads   \multipleTBLheadsfalse
 \newif\iftightTBLrowspan    \tightTBLrowspantrue
+\newif\iftightTBLcolspan    \tightTBLcolspantrue
 
 \newif\iftraceTABLE         \traceTABLEfalse
 
@@ -1514,7 +1515,7 @@
             % eigenlijk moet dit alleen als de kolom wordt overspannen door een
             % vorige, maw extra dubbele loop en status var
             \ifnum\c_tabl_ntb_encountered_max=\c_tabl_ntb_maximum_col % *nx* bah
-              \advance\!!counta \plusone % settign maxwidth to a large value also works
+              \advance\!!counta \plusone % setting maxwidth to a large value also works
             \fi
           \fi
         \fi
@@ -1652,6 +1653,10 @@
 \unexpanded\def\tabl_ntb_cell_process_x#1#2[#3]#4%
   {}
 
+% problem: when span doesn't break we can have a span that is the sum of
+% cells but still to small .. chicken egg problem ... for that we should
+% also have a smallest width run
+
 \unexpanded\def\tabl_ntb_cell_process_a#1#2[#3]#4% grouping added ! ! !
   {\bgroup
    \tabl_ntb_setup_cell{#1}{#2}%
@@ -1676,10 +1681,13 @@
        \fi
      \fi
      \tabl_ntb_spn_doifelse\c_tabl_ntb_col
-       \donothing
-       {\ifdim\tabl_ntb_get_wid\c_tabl_ntb_col<\wd\scratchbox
+       {\iftightTBLcolspan\donefalse\else\donetrue\fi}%
+       \donetrue
+     \ifdone
+       \ifdim\tabl_ntb_get_wid\c_tabl_ntb_col<\wd\scratchbox
           \tabl_ntb_set_wid\c_tabl_ntb_col{\the\wd\scratchbox}%
-        \fi}% auto set
+        \fi
+     \fi
    \fi
    \scratchcounter\numexpr\c_tabl_ntb_row+\plusone\relax
    \scratchdimen\tabl_ntb_get_hei\scratchcounter\relax
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 3d5443875..523de8fd4 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 12/31/14 20:42:33
+-- merge date  : 01/01/15 13:15:11
 
 do -- begin closure to overcome local limits and interference