2015-05-13 13:06:00

12 files changed, 237 insertions, 32 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index ec97951b7..1e4e683e9 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2015.05.12 17:57}
+\newcontextversion{2015.05.13 13:04}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 73287d375..61e0c872c 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index de3e6c701..05fbde607 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -39,7 +39,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2015.05.12 17:57}
+\edef\contextversion{2015.05.13 13:04}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/publ-aut.lua b/tex/context/base/publ-aut.lua
index 185d99efc..5a9d48551 100644
--- a/tex/context/base/publ-aut.lua
+++ b/tex/context/base/publ-aut.lua
@@ -852,7 +852,7 @@ publications.components.author = components
 
 publications.sortmethods.authoryear = {
     sequence = {
-        { field = "key",     default = "ZZZZ", unknown = "ZZZZ" },
+     -- { field = "key",     default = "ZZZZ", unknown = "ZZZZ" },
         { field = "author",  default = "",     unknown = "" },
         { field = "year",    default = "9998", unknown = "9999" },
      -- { field = "suffix",  default = " ",    unknown = " " },
diff --git a/tex/context/base/publ-imp-replacements.lua b/tex/context/base/publ-imp-replacements.lua
index b15b17e30..e0ac249fc 100644
--- a/tex/context/base/publ-imp-replacements.lua
+++ b/tex/context/base/publ-imp-replacements.lua
@@ -1,3 +1,10 @@
+-- Many bibtex databases are poluted. This is a side effect of 7 bit encoding on the
+-- one hand and tweaking the outcome at the other. The worst examples are the use
+-- of \rlap on whole names. We found that trying to cope with all can one drive insane
+-- so we stopped at some point. Clean up your mess or pay the price. But, you can load
+-- this file (and similar ones) to help you out. There is simply no reward in trying
+-- to deal with it ourselves.
+
 return {
     name         = "replacements",
     version      = "1.00",
@@ -5,7 +12,10 @@ return {
     author       = "Alan Braslau and Hans Hagen",
     copyright    = "ConTeXt development team",
     replacements = {
-        [ [[Th\^e\llap{\raise0.5ex\hbox{\'{\relax}}}]] ] = "Thánh",
+        [ [[\emdash]] ] = "—",
+        [ [[\endash]] ] = "–",
+        [ [[{\emdash}]] ] = "—",
+        [ [[{\endash}]] ] = "–",
         [ [[Th\^e\llap{\raise 0.5ex\hbox{\'{\relax}}}]] ] = "Thánh",
         [ [[Th{\^e}\llap{\raise0.5ex\hbox{\'{\relax}}}]] ] = "Thánh",
         [ [[Th{\^e}\llap{\raise 0.5ex\hbox{\'{\relax}}}]] ] = "Thánh",
diff --git a/tex/context/base/publ-sor.lua b/tex/context/base/publ-sor.lua
index ba353a9bc..b617af760 100644
--- a/tex/context/base/publ-sor.lua
+++ b/tex/context/base/publ-sor.lua
@@ -74,6 +74,8 @@ return function(dataset,list,method) -- indexer
         local index = tostring(i)
         local entry = luadata[tag]
         if entry then
+            -- maybe optional: if entry.key then push the keygetter
+            -- in slot 1 and ignore (e.g. author)
             local detail  = details[tag]
             result[i] = {
                 index  = i,
@@ -366,3 +368,10 @@ end
 table.setmetatableindex(sorters,function(t,k) return anything end)
 
 publications.lists.sorters = sorters
+
+-- publications.sortmethods.key = {
+--     sequence = {
+--         { field = "key",   default = "", unknown = "" },
+--         { field = "index", default = "", unknown = "" },
+--     },
+-- }
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index bc804670a..84a3ebc91 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index 99c615fd9..cb6bcb95b 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/context/base/typo-chr.lua b/tex/context/base/typo-chr.lua
index 94095828d..db8579c84 100644
--- a/tex/context/base/typo-chr.lua
+++ b/tex/context/base/typo-chr.lua
@@ -88,15 +88,32 @@ if not modules then modules = { } end modules ['typo-chr'] = {
 
 local insert, remove = table.insert, table.remove
 
-local nodecodes   = nodes.nodecodes
-local glyph_code  = nodecodes.glyph
+local nodecodes       = nodes.nodecodes
+local whatsitcodes    = nodes.whatsitcodes
+local glyph_code      = nodecodes.glyph
+local whatsit_code    = nodecodes.whatsit
+local localpar_code   = whatsitcodes.localpar
 
-local texnest     = tex.nest
-local free_node   = node.free
+local texnest         = tex.nest
+local free_node       = node.free
+local flush_list      = node.flush_list
 
-local punctuation = characters.is_punctuation
+local settexattribute = tex.setattribute
+local punctuation     = characters.is_punctuation
 
-local stack       = { }
+local variables       = interfaces.variables
+local v_all           = variables.all
+local v_reset         = variables.reset
+
+local a_marked        = attributes.numbers['marked']
+local lastmarked      = 0
+local marked          = {
+    [v_all]   = 1,
+    [""]      = 1,
+    [v_reset] = attributes.unsetvalue,
+}
+
+local stack           = { }
 
 local function pickup()
     local list = texnest[texnest.ptr]
@@ -143,6 +160,69 @@ local function pickuppunctuation(specification)
     end
 end
 
+-- I played with nested marked content but it makes no sense and gives
+-- complex code. Also, it's never needed so why bother.
+
+local function pickup(head,tail,str)
+    local attr = marked[str]
+    local last = tail
+    if last[a_marked] == attr then
+        local first = last
+        while true do
+            local prev = first.prev
+            if prev and prev[a_marked] == attr then
+                if prev.id == whatsit_code and prev.subtype == localpar_code then
+                    break
+                else
+                    first = prev
+                end
+            else
+                break
+            end
+        end
+        return first, last
+    end
+end
+
+local actions = {
+    remove = function(specification)
+        local list = texnest[texnest.ptr]
+        if list then
+            local head = list.head
+            local tail = list.tail
+            local first, last = pickup(head,tail,specification.mark)
+            if first then
+                if first == head then
+                    list.head = nil
+                    list.tail = nil
+                else
+                    local prev = first.prev
+                    list.tail  = prev
+                    prev.next  = nil
+                end
+                flush_list(first)
+            end
+        end
+    end,
+}
+
+local function pickupmarkedcontent(specification)
+    local action = actions[specification.action or "remove"]
+    if action then
+        action(specification)
+    end
+end
+
+local function markcontent(str)
+    local currentmarked = marked[str or v_all]
+    if not currentmarked then
+        lastmarked    = lastmarked + 1
+        currentmarked = lastmarked
+        marked[str]   = currentmarked
+    end
+    settexattribute(a_marked,currentmarked)
+end
+
 interfaces.implement {
     name      = "pickuppunctuation",
     actions   = pickuppunctuation,
@@ -153,3 +233,19 @@ interfaces.implement {
     }
 }
 
+interfaces.implement {
+    name      = "pickupmarkedcontent",
+    actions   = pickupmarkedcontent,
+    arguments = {
+        {
+            { "action" },
+            { "mark" }
+        }
+    }
+}
+
+interfaces.implement {
+    name      = "markcontent",
+    actions   = markcontent,
+    arguments = "string",
+}
diff --git a/tex/context/base/typo-chr.mkiv b/tex/context/base/typo-chr.mkiv
index 9d8dd7c20..dc0c68664 100644
--- a/tex/context/base/typo-chr.mkiv
+++ b/tex/context/base/typo-chr.mkiv
@@ -1,9 +1,9 @@
 
 %D \module
-%D   [       file=typo-cap,
-%D        version=2009.03.27, % code moved from core-spa.mkiv
+%D   [       file=typo-chr,
+%D        version=2015.01.01, % or about that time
 %D          title=\CONTEXT\ Typesetting Macros,
-%D       subtitle=Capping,
+%D       subtitle=Cleaning Up Mess,
 %D         author=Hans Hagen,
 %D           date=\currentdate,
 %D      copyright={PRAGMA ADE \& \CONTEXT\ Development Team}]
@@ -16,13 +16,67 @@
 
 \unprotect
 
-%D Maybe we need a more clever system: either command or style mode etc. so
-%D that we can avoid the grouped mess in a simple style switch.
+%D This is a very experimental feature, mostly for Alan and me to play with in
+%D publication hell.
+%D
+%D \startbuffer
+%D before, after\par
+%D before,\removepunctuation after\par
+%D before\markcontent[gone]{\darkred   gone}\removemarkedcontent[gone]after\par
+%D before\markcontent[kept]{\darkgreen kept}\removemarkedcontent[gone]after\par
+%D \markcontent[gone]{\darkred   gone}\removemarkedcontent[gone]after\par
+%D \markcontent[kept]{\darkgreen kept}\removemarkedcontent[gone]after\par
+%D \stopbuffer
+%D
+%D \typebuffer \blank \getbuffer\blank
+%D
+%D This feature is paragraph based and is only to be used for small snippets of text,
+%D for instance when combining bit and pieces where keeping a state is complex compared
+%D to cleaning up unwanted stuff.
 
 \registerctxluafile{typo-chr}{1.001}
 
-\unexpanded\def\removepunctuation{\clf_pickuppunctuation action {remove}\relax}
-\unexpanded\def\pushpunctuation  {\clf_pickuppunctuation action   {push}\relax}
-\unexpanded\def\poppunctuation   {\clf_pickuppunctuation action    {pop}\relax}
+\definesystemattribute[marked][public]
 
-\protect
+\unexpanded\def\removepunctuation
+  {\clf_pickuppunctuation action{remove}\relax} % the first experiment
+
+\unexpanded\def\pushpunctuation
+  {\clf_pickuppunctuation action{push}\relax} % the first experiment
+
+\unexpanded\def\poppunctuation
+  {\clf_pickuppunctuation action{pop}\relax} % the first experiment
+
+\unexpanded\def\markcontent
+  {\dosingleempty\typo_marked_mark}
+
+\def\typo_marked_mark[#1]#2%
+  {\dontleavehmode
+   \bgroup
+   \clf_markcontent{#1}%
+   \bgroup
+   #2% double grouping makes aftergroups work ok
+   \egroup
+   \egroup}
+
+\unexpanded\def\startmarkedcontent
+  {\dontleavehmode
+   \bgroup
+   \dosingleempty\typo_marked_start}
+
+\def\typo_marked_start[#1]%
+  {\clf_markcontent{#1}%
+   % double grouping makes aftergroups work ok
+   \bgroup}
+
+\unexpanded\def\stopmarkedcontent
+  {\egroup
+   \egroup}
+
+\unexpanded\def\removemarkedcontent
+  {\dosingleempty\typo_marked_remove}
+
+\def\typo_marked_remove[#1]%
+  {\clf_pickupmarkedcontent action{remove}mark{#1}\relax}
+
+\protect \endinput
diff --git a/tex/context/base/typo-itc.lua b/tex/context/base/typo-itc.lua
index f37549dc8..c6528eee8 100644
--- a/tex/context/base/typo-itc.lua
+++ b/tex/context/base/typo-itc.lua
@@ -126,6 +126,42 @@ local function okay(data,current,font,prevchar,previtalic,char,what)
     return true
 end
 
+-- maybe: with_attributes(current,n) :
+--
+-- local function correction_kern(kern,n)
+--     return with_attributes(new_correction_kern(kern),n)
+-- end
+
+local function correction_glue(glue,n)
+    local g = new_correction_glue(glue)
+    if n then
+        setfield(g,"attr",getfield(n,"attr"))
+    end
+    return g
+end
+
+local function correction_kern(kern,n)
+    local k = new_correction_kern(kern)
+    if n then
+        local a = getfield(n,"attr")
+        if a then -- maybe not
+            setfield(k,"attr",a) -- can be a marked content (border case)
+        end
+    end
+    return k
+end
+
+local function correction_glue(n,glue)
+    local g = new_correction_glue(glue)
+    if n then
+        local a = getfield(n,"attr")
+        if a then -- maybe not
+            setfield(g,"attr",a) -- can be a marked content (border case)
+        end
+    end
+    return g
+end
+
 function italics.handler(head)
 
     local prev            = nil
@@ -160,7 +196,7 @@ function italics.handler(head)
             if font ~= lastfont then
                 if previtalic ~= 0 then
                     if okay(data,current,font,prevchar,previtalic,char,"glyph") then
-                        insert_node_after(prevhead,prev,new_correction_kern(previtalic))
+                        insert_node_after(prevhead,prev,correction_kern(previtalic,current))
                         done = true
                     end
                 elseif previnserted and data then
@@ -172,7 +208,7 @@ function italics.handler(head)
                     --
                     if replaceitalic ~= 0 then
                         if okay(data,replace,font,replacechar,replaceitalic,char,"replace") then
-                            insert_node_after(replacehead,replace,new_correction_kern(replaceitalic))
+                            insert_node_after(replacehead,replace,correction_kern(replaceitalic,current))
                             done = true
                         end
                         replaceitalic = 0
@@ -185,7 +221,7 @@ function italics.handler(head)
                     --
                     if postitalic ~= 0 then
                         if okay(data,post,font,postchar,postitalic,char,"post") then
-                            insert_node_after(posthead,post,new_correction_kern(postitalic))
+                            insert_node_after(posthead,post,correction_kern(postitalic,current))
                             done = true
                         end
                         postitalic = 0
@@ -338,7 +374,7 @@ function italics.handler(head)
                 if trace_italics then
                     report_italics("inserting %p between %s italic %C and glue",previtalic,"glyph",prevchar)
                 end
-                previnserted = new_correction_glue(previtalic) -- maybe just add ? else problem with penalties
+                previnserted = correction_glue(previtalic,current) -- maybe just add ? else problem with penalties
                 previtalic   = 0
                 done         = true
                 insert_node_after(prevhead,prev,previnserted)
@@ -347,7 +383,7 @@ function italics.handler(head)
                     if trace_italics then
                         report_italics("inserting %p between %s italic %C and glue",replaceitalic,"replace",replacechar)
                     end
-                    replaceinserted = new_correction_kern(replaceitalic) -- needs to be a kern
+                    replaceinserted = correction_kern(replaceitalic,current) -- needs to be a kern
                     replaceitalic   = 0
                     done            = true
                     insert_node_after(replacehead,replace,replaceinserted)
@@ -356,7 +392,7 @@ function italics.handler(head)
                     if trace_italics then
                         report_italics("inserting %p between %s italic %C and glue",postitalic,"post",postchar)
                     end
-                    postinserted = new_correction_kern(postitalic) -- needs to be a kern
+                    postinserted = correction_kern(postitalic,current) -- needs to be a kern
                     postitalic   = 0
                     done         = true
                     insert_node_after(posthead,post,postinserted)
@@ -375,7 +411,7 @@ function italics.handler(head)
                 if trace_italics then
                     report_italics("inserting %p between %s italic %C and whatever",previtalic,"glyph",prevchar)
                 end
-                insert_node_after(prevhead,prev,new_correction_kern(previtalic))
+                insert_node_after(prevhead,prev,correction_kern(previtalic,current))
                 previnserted    = nil
                 previtalic      = 0
                 replaceinserted = nil
@@ -388,7 +424,7 @@ function italics.handler(head)
                     if trace_italics then
                         report_italics("inserting %p between %s italic %C and whatever",replaceritalic,"replace",replacechar)
                     end
-                    insert_node_after(replacehead,replace,new_correction_kern(replaceitalic))
+                    insert_node_after(replacehead,replace,correction_kern(replaceitalic,current))
                     previnserted    = nil
                     previtalic      = 0
                     replaceinserted = nil
@@ -401,7 +437,7 @@ function italics.handler(head)
                     if trace_italics then
                         report_italics("inserting %p between %s italic %C and whatever",postitalic,"post",postchar)
                     end
-                    insert_node_after(posthead,post,new_correction_kern(postitalic))
+                    insert_node_after(posthead,post,correction_kern(postitalic,current))
                     previnserted    = nil
                     previtalic      = 0
                     replaceinserted = nil
@@ -419,21 +455,21 @@ function italics.handler(head)
             if trace_italics then
                 report_italics("inserting %p between %s italic %C and end of list",previtalic,"glyph",prevchar)
             end
-            insert_node_after(prevhead,prev,new_correction_kern(previtalic))
+            insert_node_after(prevhead,prev,correction_kern(previtalic,current))
             done = true
         else
             if replaceitalic ~= 0 then
                 if trace_italics then
                     report_italics("inserting %p between %s italic %C and end of list",replaceitalic,"replace",replacechar)
                 end
-                insert_node_after(replacehead,replace,new_correction_kern(replaceitalic))
+                insert_node_after(replacehead,replace,correction_kern(replaceitalic,current))
                 done = true
             end
             if postitalic ~= 0 then
                 if trace_italics then
                     report_italics("inserting %p between %s italic %C and end of list",postitalic,"post",postchar)
                 end
-                insert_node_after(posthead,post,new_correction_kern(postitalic))
+                insert_node_after(posthead,post,correction_kern(postitalic,current))
                 done = true
             end
         end
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index ead4e2aea..de74d99a4 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 05/12/15 17:57:50
+-- merge date  : 05/13/15 13:04:20
 
 do -- begin closure to overcome local limits and interference