beta 2014.02.07 00:57

21 files changed, 139 insertions, 37 deletions

diff --git a/tex/context/base/char-def.lua b/tex/context/base/char-def.lua
index 9642d1736..3e1d56009 100644
--- a/tex/context/base/char-def.lua
+++ b/tex/context/base/char-def.lua
@@ -1829,7 +1829,9 @@ characters.data={
   direction="on",
   linebreak="al",
   mathclass="topaccent",
+  mathfiller="barfill",
   mathname="bar",
+  mathmleq=0x203E,
   specials={ "compat", 0x0020, 0x0304 },
   unicodeslot=0x00AF,
  },
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 587b9e5a5..63bf53a9e 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2014.02.01 14:22}
+\newcontextversion{2014.02.07 00:57}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index b12d442b4..be0d634a3 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 36a0e8645..abfc44390 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -28,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2014.02.01 14:22}
+\edef\contextversion{2014.02.07 00:57}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/core-env.lua b/tex/context/base/core-env.lua
index 8cbc93038..2cc84299b 100644
--- a/tex/context/base/core-env.lua
+++ b/tex/context/base/core-env.lua
@@ -31,6 +31,7 @@ tex.systemmodes         = allocate { }
 tex.constants           = allocate { }
 tex.conditionals        = allocate { }
 tex.ifs                 = allocate { }
+tex.isdefined           = allocate { }
 
 local modes             = { }
 local systemmodes       = { }
@@ -78,6 +79,10 @@ setmetatableindex(tex.ifs, function(t,k)
     return csname_id(k) ~= undefined and create(k)[2] == iftrue -- inefficient, this create, we need a helper
 end)
 
+setmetatableindex(tex.isdefined, function(t,k)
+    return csname_id(k) ~= undefined
+end)
+
 function context.setconditional(name,value)
     if value then
         context.settruevalue(name)
diff --git a/tex/context/base/file-job.mkvi b/tex/context/base/file-job.mkvi
index ce0d54ece..fa395a32e 100644
--- a/tex/context/base/file-job.mkvi
+++ b/tex/context/base/file-job.mkvi
@@ -274,7 +274,7 @@
 
 %D Relatively new (might move as it depends on setups):
 
-\newtoks\everysetupdocument
+%newtoks\everysetupdocument
 
 \unexpanded\def\startdocument % todo: dostarttagged\t!document
   {\dosingleargument\syst_structure_document_start}
diff --git a/tex/context/base/font-mis.lua b/tex/context/base/font-mis.lua
index 75cf62dde..63cae37f3 100644
--- a/tex/context/base/font-mis.lua
+++ b/tex/context/base/font-mis.lua
@@ -22,7 +22,7 @@ local handlers = fonts.handlers
 handlers.otf   = handlers.otf or { }
 local otf      = handlers.otf
 
-otf.version    = otf.version or 2.750
+otf.version    = otf.version or 2.751
 otf.cache      = otf.cache   or containers.define("fonts", "otf", otf.version, true)
 
 function otf.loadcached(filename,format,sub)
diff --git a/tex/context/base/font-otf.lua b/tex/context/base/font-otf.lua
index d6373cef9..eb28bc368 100644
--- a/tex/context/base/font-otf.lua
+++ b/tex/context/base/font-otf.lua
@@ -48,7 +48,7 @@ local otf                = fonts.handlers.otf
 
 otf.glists               = { "gsub", "gpos" }
 
-otf.version              = 2.750 -- beware: also sync font-mis.lua
+otf.version              = 2.751 -- beware: also sync font-mis.lua
 otf.cache                = containers.define("fonts", "otf", otf.version, true)
 
 local fontdata           = fonts.hashes.identifiers
@@ -239,7 +239,7 @@ local valid_fields = table.tohash {
     "upos",
     "use_typo_metrics",
     "uwidth",
- -- "validation_state",
+    "validation_state",
     "version",
     "vert_base",
     "weight",
@@ -1772,6 +1772,13 @@ actions["check metadata"] = function(data,filename,raw)
             ttftables[i].data = "deleted"
         end
     end
+    --
+    if metadata.validation_state and table.contains(metadata.validation_state,"bad_ps_fontname") then
+        local name = file.nameonly(filename)
+        metadata.fontname = "bad-fontname-" .. name
+        metadata.fullname = "bad-fullname-" .. name
+    end
+    --
 end
 
 actions["cleanup tables"] = function(data,filename,raw)
diff --git a/tex/context/base/font-pat.lua b/tex/context/base/font-pat.lua
index 9733c9ada..049853796 100644
--- a/tex/context/base/font-pat.lua
+++ b/tex/context/base/font-pat.lua
@@ -38,7 +38,7 @@ local report   = patches.report
 -- library) the palatino arabic fonts don't have the mkmk features properly
 -- set up.
 
-register("after","rehash features","^palatino.*arabic", function patch(data,filename)
+register("after","rehash features","^palatino.*arabic", function (data,filename)
     local gpos = data.gpos
     if gpos then
         for k=1,#gpos do
diff --git a/tex/context/base/luat-cnf.lua b/tex/context/base/luat-cnf.lua
index 3672c603e..4ad6cd69d 100644
--- a/tex/context/base/luat-cnf.lua
+++ b/tex/context/base/luat-cnf.lua
@@ -23,7 +23,7 @@ texconfig.half_error_line =     50 --    50 -- obsolete
 texconfig.expand_depth    =  10000 -- 10000
 texconfig.hash_extra      = 100000 --     0
 texconfig.nest_size       =   1000 --    50
-texconfig.max_in_open     =    500 --    15
+texconfig.max_in_open     =    500 --    15 -- in fact it's limited to 127
 texconfig.max_print_line  =  10000 --    79
 texconfig.max_strings     = 500000 -- 15000
 texconfig.param_size      =  25000 --    60
diff --git a/tex/context/base/math-fen.mkiv b/tex/context/base/math-fen.mkiv
index 94d93e4af..fe959cc1e 100644
--- a/tex/context/base/math-fen.mkiv
+++ b/tex/context/base/math-fen.mkiv
@@ -144,7 +144,7 @@
 \definemathfence [mirroredangle]       [mirrored] [\c!right="27E8,\c!left="27E9]
 \definemathfence [mirroreddoubleangle] [mirrored] [\c!right="27EA,\c!left="27EB]
 \definemathfence [mirroredsolidus]     [mirrored] [\c!right="2044,\c!left="2044]
-\definemathfence [mirrorednothing]     [mirorred]
+\definemathfence [mirrorednothing]     [mirrored]
 
 %D A bonus:
 
diff --git a/tex/context/base/math-ini.lua b/tex/context/base/math-ini.lua
index 6be06e634..4da1cded2 100644
--- a/tex/context/base/math-ini.lua
+++ b/tex/context/base/math-ini.lua
@@ -470,6 +470,7 @@ mathematics.utfmathclass   = utfmathclass
 mathematics.utfmathstretch = utfmathstretch
 mathematics.utfmathcommand = utfmathcommand
 mathematics.utfmathfiller  = utfmathfiller
+mathematics.utfmathaccent  = utfmathaccent
 
 -- interfaced
 
diff --git a/tex/context/base/page-str.mkiv b/tex/context/base/page-str.mkiv
index 200a71377..a8fab9c6c 100644
--- a/tex/context/base/page-str.mkiv
+++ b/tex/context/base/page-str.mkiv
@@ -29,8 +29,6 @@
 %D
 %D Remark: marknotes are gone, at least for a while.
 
-\writestatus{loading}{ConTeXt Page Macros / Page Streams}
-
 \registerctxluafile{page-str}{1.001}
 
 \unprotect
diff --git a/tex/context/base/publ-ini.lua b/tex/context/base/publ-ini.lua
index 924caf561..6bf6714da 100644
--- a/tex/context/base/publ-ini.lua
+++ b/tex/context/base/publ-ini.lua
@@ -47,7 +47,7 @@ local concat, sort = table.concat, table.sort
 local utfsub = utf.sub
 local formatters = string.formatters
 local allocate = utilities.storage.allocate
-local settings_to_array = utilities.parsers.settings_to_array
+local settings_to_array, settings_to_set = utilities.parsers.settings_to_array, utilities.parsers.settings_to_set
 local sortedkeys, sortedhash = table.sortedkeys, table.sortedhash
 local lpegmatch = lpeg.match
 local P, C, Ct = lpeg.P, lpeg.C, lpeg.Ct
@@ -338,7 +338,10 @@ function publications.enhance(dataset) -- for the moment split runs (maybe publi
         if type(tags) == "table" then
             sort(tags)
             for i=1,#tags do
-                details[tags[i]].short = short .. numbertochar(i)
+--                 details[tags[i]].short = short .. numbertochar(i)
+local detail = details[tags[i]]
+detail.short  = short
+detail.suffix = numbertochar(i)
             end
         else
             details[tags].short = short
@@ -352,6 +355,13 @@ function publications.enhance(dataset) -- for the moment split runs (maybe publi
             details[tag].pages = first and last and { first, last } or pages
         end
     end
+    -- keywords
+    for tag, entry in next, luadata do
+        local keyword = entry.keyword
+        if keyword then
+            details[tag].keyword = settings_to_set(keyword)
+        end
+    end
     statistics.stoptiming(publications)
 end
 
@@ -733,7 +743,7 @@ end)
 
 -- why shorts vs tags: only for sorting
 
-function lists.register(dataset,tag,short)
+function lists.register(dataset,tag,short) -- needs checking now that we split
     local r = renderings[dataset]
     if not short or short == "" then
         short = tag
@@ -759,19 +769,49 @@ function lists.setmethod(dataset,method)
     r.done   = { }
 end
 
+local function validkeyword(dataset,tag,keyword)
+    local ds = datasets[dataset]
+    if not ds then
+        report("unknown dataset %a",dataset)
+        return
+    end
+    local dt = ds.details[tag]
+    if not dt then
+        report("no details for tag %a",tag)
+        return
+    end
+    local kw = dt.keyword
+    if kw then
+-- inspect(keyword)
+-- inspect(kw)
+        for k in next, keyword do
+            if kw[k] then
+                return true
+            end
+        end
+    end
+end
+
 function lists.collectentries(specification)
     local dataset = specification.btxdataset
     if not dataset then
         return
     end
     local rendering = renderings[dataset]
+-- specification.names = "btx"
     local method = rendering.method
     if method == v_none then
         return
     end
 -- method=v_local --------------------
     local result  = structures.lists.filter(specification)
--- inspect(result)
+    --
+    local keyword = specification.keyword
+    if keyword and keyword ~= "" then
+        keyword = settings_to_set(keyword)
+    else
+        keyword = nil
+    end
     lists.result  = result
     local section = sections.currentid()
     local list    = rendering.list
@@ -784,9 +824,11 @@ function lists.collectentries(specification)
             if u and u.btxset == dataset then
                 local tag = u.btxref
                 if tag and done[tag] ~= section then
-                    done[tag]     = section
-                    alldone[tag]  = true
-                    list[#list+1] = { tag, listindex }
+                    if not keyword or validkeyword(dataset,tag,keyword) then
+                        done[tag]     = section
+                        alldone[tag]  = true
+                        list[#list+1] = { tag, listindex }
+                    end
                 end
             end
         end
@@ -797,9 +839,11 @@ function lists.collectentries(specification)
             if u and u.btxset == dataset then
                 local tag = u.btxref
                 if tag and not alldone[tag] and done[tag] ~= section then
-                    done[tag]     = section
-                    alldone[tag]  = true
-                    list[#list+1] = { tag, listindex }
+                    if not keyword or validkeyword(dataset,tag,keyword) then
+                        done[tag]     = section
+                        alldone[tag]  = true
+                        list[#list+1] = { tag, listindex }
+                    end
                 end
             end
         end
@@ -812,14 +856,18 @@ function lists.collectentries(specification)
             if u and u.btxset == dataset then
                 local tag = u.btxref
                 if tag then
-                    list[#list+1] = { tag, listindex }
+                    if not keyword or validkeyword(dataset,tag,keyword) then
+                        list[#list+1] = { tag, listindex }
+                    end
                 end
             end
         end
     elseif method == v_dataset then
-        dataset = datasets[dataset]
-        for tag, data in table.sortedhash(dataset.luadata) do
-            list[#list+1] = { tag }
+        local luadata = datasets[dataset].luadata
+        for tag, data in table.sortedhash(luadata) do
+            if not keyword or validkeyword(dataset,tag,keyword) then
+                list[#list+1] = { tag }
+            end
         end
     end
 end
@@ -1287,9 +1335,19 @@ function citevariants.authornum(dataset,tags)
     lists.resolve(dataset,tags) -- left/right ?
 end
 
+-- function citevariants.short(dataset,tags)
+--     local short = getdetail(dataset,tags,"short")
+--     if short then
+--         context(short)
+--     end
+-- end
+
 function citevariants.short(dataset,tags)
     local short = getdetail(dataset,tags,"short")
-    if short then
+    local suffix = getdetail(dataset,tags,"suffix")
+    if suffix then
+        context(short .. suffix)
+    elseif short then
         context(short)
     end
 end
@@ -1349,9 +1407,19 @@ function listvariants.num(dataset,block,tags,variant,listindex)
     ctx_btxdirectlink(f_destination(dataset,block,tags),listindex) -- not okay yet
 end
 
+-- function listvariants.short(dataset,block,tags,variant,listindex)
+--     local short = getdetail(dataset,tags,variant,variant)
+--     if short then
+--         context(short)
+--     end
+-- end
+
 function listvariants.short(dataset,block,tags,variant,listindex)
-    local short = getdetail(dataset,tags,variant,variant)
-    if short then
+    local short  = getdetail(dataset,tags,"short","short")
+    local suffix = getdetail(dataset,tags,"suffix","suffix")
+    if suffix then
+        context(short .. suffix)
+    elseif short then
         context(short)
     end
 end
diff --git a/tex/context/base/publ-ini.mkiv b/tex/context/base/publ-ini.mkiv
index fc2b00055..42226695c 100644
--- a/tex/context/base/publ-ini.mkiv
+++ b/tex/context/base/publ-ini.mkiv
@@ -423,8 +423,9 @@
        \ctxcommand{btxcollectlistentries {
            names        = "btx",
            criterium    = "\currentbtxcriterium",
-           number       = "\namedlistparameter\currentbtxrendering\c!number",
+           number       = "\btxrenderingparameter\c!number",
            btxdataset   = "\currentbtxdataset",
+           keyword      = "\btxrenderingparameter\c!keyword",
        }}%
        % next we analyze the width
        \ifx\btx_reference_inject_indeed\relax \else
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index e089e9b11..3050b0acb 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index af6e21167..fab84e1ba 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/typo-bld.lua b/tex/context/base/typo-bld.lua
index 04bf226c1..b5668d362 100644
--- a/tex/context/base/typo-bld.lua
+++ b/tex/context/base/typo-bld.lua
@@ -173,7 +173,6 @@ function constructors.disable()
     enabled = false
 end
 
-
 callbacks.register('linebreak_filter', processor, "breaking paragraps into lines")
 
 statistics.register("linebreak processing time", function()
diff --git a/tex/context/base/x-mathml.lua b/tex/context/base/x-mathml.lua
index ee18d4776..e2bb62d12 100644
--- a/tex/context/base/x-mathml.lua
+++ b/tex/context/base/x-mathml.lua
@@ -83,7 +83,7 @@ local o_replacements = { -- in main table
  -- [utfchar(0xF103C)] = "\\mmlleftdelimiter<",
     [utfchar(0xF1026)] = "\\mmlchar{38}",
  -- [utfchar(0xF103E)] = "\\mmlleftdelimiter>",
-
+-- [utfchar(0xAF)] = '\\mmlchar{"203E}', -- 0x203E
 }
 
 local simpleoperatorremapper = utf.remapper(o_replacements)
@@ -479,7 +479,7 @@ end
 function mathml.mo(id)
     local str = xmlcontent(getid(id)) or ""
     local rep = gsub(str,"&.-;","") -- todo
-    context(simpleoperatorremapper(rep))
+    context(simpleoperatorremapper(rep) or rep)
 end
 
 function mathml.mi(id)
@@ -833,3 +833,13 @@ function mathml.cpolar_a(root)
     end
     context.right(false,")")
 end
+
+-- crap .. maybe in char-def a mathml overload
+
+local mathmleq = {
+    [utfchar(0x00AF)] = utfchar(0x203E),
+}
+
+function mathml.extensible(chr)
+    context(mathmleq[chr] or chr)
+end
diff --git a/tex/context/base/x-mathml.mkiv b/tex/context/base/x-mathml.mkiv
index ec8fd74e4..320d19251 100644
--- a/tex/context/base/x-mathml.mkiv
+++ b/tex/context/base/x-mathml.mkiv
@@ -2283,7 +2283,7 @@
 \unexpanded\def\mmloverbs#1{\mmlexecuteifdefined\mmlbasecommand\relax{\mmlunexpandedsecond{#1}}\relax}
 
 \startxmlsetups mml:mover
-    \edef\mmlovertoken{\xmlraw{#1}{/mml:*[2]}}% /text()
+    \edef\mmlovertoken{\mmlextensible{\xmlraw{#1}{/mml:*[2]}}}% /text()
     \doifelseutfmathabove\mmlovertoken {
         \edef\mmlovercommand{\utfmathcommandabove\mmlovertoken}
         \mmloverof{#1}
@@ -2321,13 +2321,18 @@
 %   % \limits % spoils spacing
 % \stopxmlsetups
 
+% do this in lua
+
+\def\mmlextensible#1{\ctxmodulemathml{extensible(\!!bs#1\!!es)}}
+
 \unexpanded\def\mmlunderuf#1{\mmlexecuteifdefined\mmlundercommand\relax  {\mmlunexpandedfirst {#1}}\relax}
 \unexpanded\def\mmlunderus#1{\mmlexecuteifdefined\mmlundercommand        {\mmlunexpandedsecond{#1}}\relax}
 \unexpanded\def\mmlunderbf#1{\mmlexecuteifdefined\mmlbasecommand         {\mmlunexpandedfirst {#1}}\relax}
-\unexpanded\def\mmlunderbs#1{\mmlexecuteifdefined\mmlbasecommand \relax{}{\mmlunexpandedsecond{#1}}\relax}
+%unexpanded\def\mmlunderbs#1{\mmlexecuteifdefined\mmlbasecommand \relax{}{\mmlunexpandedsecond{#1}}\relax}
+\unexpanded\def\mmlunderbs#1{\mmlexecuteifdefined\mmlbasecommand \relax  {\mmlunexpandedsecond{#1}}\relax}
 
 \startxmlsetups mml:munder
-    \edef\mmlundertoken{\xmlraw{#1}{/mml:*[2]}}% /text()
+    \edef\mmlundertoken{\mmlextensible{\xmlraw{#1}{/mml:*[2]}}}% /text()
     \doifelseutfmathbelow\mmlundertoken {%
         \edef\mmlundercommand{\utfmathcommandbelow\mmlundertoken}
         \mmlunderuf{#1}
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 3986118a0..8c31750c8 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 02/01/14 14:22:42
+-- merge date  : 02/07/14 00:57:35
 
 do -- begin closure to overcome local limits and interference
 
@@ -6450,7 +6450,7 @@ local report_otf=logs.reporter("fonts","otf loading")
 local fonts=fonts
 local otf=fonts.handlers.otf
 otf.glists={ "gsub","gpos" }
-otf.version=2.750 
+otf.version=2.751 
 otf.cache=containers.define("fonts","otf",otf.version,true)
 local fontdata=fonts.hashes.identifiers
 local chardata=characters and characters.data 
@@ -6602,6 +6602,7 @@ local valid_fields=table.tohash {
   "upos",
   "use_typo_metrics",
   "uwidth",
+  "validation_state",
   "version",
   "vert_base",
   "weight",
@@ -7914,6 +7915,11 @@ actions["check metadata"]=function(data,filename,raw)
       ttftables[i].data="deleted"
     end
   end
+  if metadata.validation_state and table.contains(metadata.validation_state,"bad_ps_fontname") then
+    local name=file.nameonly(filename)
+    metadata.fontname="bad-fontname-"..name
+    metadata.fullname="bad-fullname-"..name
+  end
 end
 actions["cleanup tables"]=function(data,filename,raw)
   data.resources.indices=nil