beta 2013.12.20 00:55

10 files changed, 171 insertions, 70 deletions

diff --git a/tex/context/base/char-tex.lua b/tex/context/base/char-tex.lua
index 4c397ab3b..3fa140c7d 100644
--- a/tex/context/base/char-tex.lua
+++ b/tex/context/base/char-tex.lua
@@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['char-tex'] = {
 local lpeg = lpeg
 
 local find = string.find
-local P, C, R, S, Cs, Cc = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cs, lpeg.Cc
+local P, C, R, S, V, Cs, Cc = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.V, lpeg.Cs, lpeg.Cc
 local U, lpegmatch = lpeg.patterns.utf8, lpeg.match
 
 local allocate, mark = utilities.storage.allocate, utilities.storage.mark
@@ -150,53 +150,80 @@ local accent_map = allocate { -- incomplete
     --  ̰ Ḛ
 }
 
-local accents = table.concat(table.keys(accent_map))
+-- local accents = table.concat(table.keys(accentmapping)) -- was _map
 
-local function remap_accents(a,c,braced)
-    local m = accent_map[a]
+local function remap_accent(a,c,braced)
+    local m = accentmapping[a]
     if m then
-        return c .. m
-    elseif braced then
+        local n = m[c]
+        if n then
+            return n
+        end
+    end
+--     local m = accent_map[a]
+--     if m then
+--         return c .. m
+--     elseif braced then -- or #c > 0
+    if braced then -- or #c > 0
         return "\\" .. a .. "{" .. c .. "}"
     else
-        return "\\" .. a .. c
+        return "\\" .. a .. " " .. c
     end
 end
 
 local command_map = allocate {
-    ["i"] = "ı",
-    ["l"] = "ł",
+    ["i"]  = "ı",
+    ["l"]  = "ł",
+    ["ss"] = "ß",
+    ["ae"] = "æ",
+    ["AE"] = "Æ",
+    ["oe"] = "œ",
+    ["OE"] = "Œ",
 }
 
-local function remap_commands(c)
-    local m = command_map[c]
-    if m then
-        return m
-    else
-        return "\\" .. c
-    end
-end
+-- no need for U here
 
 local spaces   = P(" ")^0
-local accents  = ( P('\\') * C(S(accents)) * spaces * (P("{") * C(U) * P("}" * Cc(true)) + C(U) * Cc(false)) ) / remap_accents
-local commands = ( P('\\') * C(R("az","AZ")^1) + P("{") * P('\\') * C(R("az","AZ")^1) * spaces * P("}") )/ remap_commands
+local no_l     = P("{") / ""
+local no_r     = P("}") / ""
+local no_b     = P('\\') / ""
 
-local convert_accents  = Cs((accents  + P(1))^0)
-local convert_commands = Cs((commands + P(1))^0)
+local lUr      = P("{") * C(R("az","AZ")) * P("}")
 
-local no_l = P("{") / ""
-local no_r = P("}") / ""
+local accents_1 = [["'.=^`~]]
+local accents_2 = [[Hckruv]]
 
-local convert_accents_strip  = Cs((no_l * accents  * no_r + accents  + P(1))^0)
-local convert_commands_strip = Cs((no_l * commands * no_r + commands + P(1))^0)
+local accent   = P('\\') * (
+    C(S(accents_1)) * (lUr * Cc(true) + C(R("az","AZ")) * Cc(false)) +
+    C(S(accents_2)) *  lUr * Cc(true)
+) / remap_accent
+
+local csname  = P('\\') * C(R("az","AZ")^1)
+
+local command  = (
+    csname +
+    P("{") * csname * spaces * P("}")
+) / command_map -- remap_commands
+
+local both_1 = Cs { "run",
+    accent  = accent,
+    command = command,
+    run     = (V("accent") + no_l * V("accent") * no_r + V("command") + P(1))^0,
+}
+
+local both_2 = Cs { "run",
+    accent  = accent,
+    command = command,
+    run     = (V("accent") + V("command") + no_l * ( V("accent") + V("command") ) * no_r + P(1))^0,
+}
 
 function characters.tex.toutf(str,strip)
-    if not find(str,"\\") then -- we can start at the found position
+    if not find(str,"\\") then
         return str
     elseif strip then
-        return lpegmatch(convert_accents_strip,lpegmatch(convert_commands_strip,str))
+        return lpegmatch(both_1,str)
     else
-        return lpegmatch(convert_accents,      lpegmatch(convert_commands,      str))
+        return lpegmatch(both_2,str)
     end
 end
 
@@ -206,6 +233,9 @@ end
 -- print(characters.tex.toutf([[{\" {e}}]],true))
 -- print(characters.tex.toutf([[{\l}]],true))
 -- print(characters.tex.toutf([[{\l }]],true))
+-- print(characters.tex.toutf([[\v{r}]],true))
+-- print(characters.tex.toutf([[fo{\"o}{\ss}ar]],true))
+-- print(characters.tex.toutf([[H{\'a}n Th\^e\llap{\raise 0.5ex\hbox{\'{\relax}}} Th{\'a}nh]],true))
 
 function characters.tex.defineaccents()
     for accent, group in next, accentmapping do
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 03a26311b..12841a278 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2013.12.19 00:11}
+\newcontextversion{2013.12.20 00:55}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index a9b04a923..64ce0b25d 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 516567e89..976760be7 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -11,6 +11,9 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
+% Welcome to context, pronounced as kontekst (rather dutch) and not as
+% conτεχt.
+
 \catcode`\{=1 \catcode`\}=2 \catcode`\#=6
 
 %D From the next string (which is set by the script that assembles the
@@ -25,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2013.12.19 00:11}
+\edef\contextversion{2013.12.20 00:55}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/font-otf.lua b/tex/context/base/font-otf.lua
index b2ca54615..e52946059 100644
--- a/tex/context/base/font-otf.lua
+++ b/tex/context/base/font-otf.lua
@@ -99,8 +99,27 @@ registerdirective("fonts.otf.loader.syncspace",     function(v) syncspace     =
 registerdirective("fonts.otf.loader.forcenotdef",   function(v) forcenotdef   = v end)
 registerdirective("fonts.otf.loader.overloadkerns", function(v) overloadkerns = v end)
 
+function otf.fileformat(filename)
+    local leader = lower(io.loadchunk(filename,4))
+    local suffix = lower(file.suffix(filename))
+    if leader == "otto" then
+        return "opentype", "otf", suffix == "otf"
+    elseif leader == "ttcf" then
+        return "truetype", "ttc", suffix == "ttc"
+    elseif suffix == "ttc" then
+        return "truetype", "ttc", true
+    else
+        return "truetype", "ttf", suffix == "ttf"
+    end
+end
+
 local function otf_format(filename)
-    return formats[lower(file.suffix(filename))]
+    local format, suffix, okay = otf.fileformat(filename)
+    if not okay then
+        report_otf("font %a is actually an %a file",filename,format)
+    end
+ -- return formats[lower(file.suffix(filename))]
+    return suffix
 end
 
 local function load_featurefile(raw,featurefile)
diff --git a/tex/context/base/font-sel.lua b/tex/context/base/font-sel.lua
index 7926d8dd3..2881917eb 100644
--- a/tex/context/base/font-sel.lua
+++ b/tex/context/base/font-sel.lua
@@ -494,7 +494,7 @@ local function definefontsynonym(data,alternative,index,fallback)
         end
         for _, entry in next, fontdata do
             local designsize = entry["designsize"] or 100
-            if designsize == 100 or designsize == 120 or designsize == 0 then
+            if designsize == 100 or designsize == 120 or designsize == 0 or #fontdata == 1 then
                 local filepath, filename = splitbase(entry["filename"])
                 registerdesignsizes( fontfile, "default", filename )
                 break
@@ -600,7 +600,7 @@ local function definemathfontfallback(data,alternative,index)
         for _, entry in next, fontdata do
             local filename   = entry["filename"]
             local designsize = entry["designsize"] or 100
-            if designsize == 100 or designsize == 120 or designsize == 0 then
+            if designsize == 100 or designsize == 120 or designsize == 0 or #fontdata == 1 then
                 context.definefontfallback( { fallback }, { formatters["file:%s*%s"](filename,features) }, { range }, { rscale = rscale, check = check, force = force, offset = offset } )
                 break
             end
diff --git a/tex/context/base/node-ltp.lua b/tex/context/base/node-ltp.lua
index 62f922c2a..c52e001df 100644
--- a/tex/context/base/node-ltp.lua
+++ b/tex/context/base/node-ltp.lua
@@ -7,7 +7,7 @@ if not modules then modules = { } end modules ['node-par'] = {
     comment   = "a translation of the built in parbuilder, initial convertsin by Taco Hoekwater",
 }
 
--- todo: remove nest_stack from  linebreak.w
+-- todo: remove nest_stack from linebreak.w
 -- todo: use ex field as signal (index in ?)
 -- todo: attr driven unknown/on/off
 -- todo: permit global steps i.e. using an attribute that sets min/max/step and overloads the font parameters
@@ -526,11 +526,26 @@ local function kern_stretch_shrink(p,d)
     return 0, 0
 end
 
-local function kern_stretch_shrink(p,d)
-    -- maybe make it an option in luatex where we also need to check for attribute fontkern but in general
-    -- it makes no sense to scale kerns
-    return 0, 0
-end
+-- local function kern_stretch_shrink(p,d)
+--     -- maybe make it an option in luatex where we also need to check for attribute fontkern but in general
+--     -- it makes no sense to scale kerns
+--     return 0, 0
+-- end
+
+local expand_kerns = false
+-- local expand_kerns = "both"
+
+directives.register("builders.paragraphs.adjusting.kerns",function(v)
+    if not v then
+        expand_kerns = false
+    elseif v == "stretch" or v == "shrink" then
+        expand_kerns = v
+    elseif v == "both" then
+        expand_kerns = true
+    else
+        expand_kerns = toboolean(v,true) or false
+    end
+end)
 
 -- state:
 
@@ -540,7 +555,7 @@ local function check_expand_pars(checked_expansion,f)
         checked_expansion[f] = false
         return false
     end
-expansion.step = 1
+-- expansion.step = 1
     local step    = expansion.step    or 0
     local stretch = expansion.stretch or 0
     local shrink  = expansion.shrink  or 0
@@ -588,7 +603,7 @@ local function check_expand_lines(checked_expansion,f)
         checked_expansion[f] = false
         return false
     end
-expansion.step = 1
+-- expansion.step = 1
     local step    = expansion.step    or 0
     local stretch = expansion.stretch or 0
     local shrink  = expansion.shrink  or 0
@@ -730,15 +745,21 @@ local function add_to_width(line_break_dir,checked_expansion,s) -- split into tw
                 size = size + s.depth + s.height
             end
         elseif id == kern_code then
-            if checked_expansion and s.subtype == kerning_code then
-                local d = s.kern
-                if d ~= 0 then
+            local d = s.kern
+            if d ~= 0 then 
+                if checked_expansion and expand_kerns and (s.subtype == kerning_code or s[a_fontkern]) then
                     local stretch, shrink = kern_stretch_shrink(s,d)
-                    adjust_stretch = adjust_stretch + stretch
-                    adjust_shrink  = adjust_shrink  + shrink
+                    if expand_kerns == "stretch" then
+                        adjust_stretch = adjust_stretch + stretch
+                    elseif expand_kerns == "shrink" then
+                        adjust_shrink  = adjust_shrink  + shrink
+                    else
+                        adjust_stretch = adjust_stretch + stretch
+                        adjust_shrink  = adjust_shrink  + shrink
+                    end
                 end
+                size = size + d
             end
-            size = size + s.kern
         elseif id == rule_code then
             size = size + s.width
         else
@@ -946,6 +967,7 @@ local function initialize_line_break(head,display)
         ignored_dimen                = tex.pdfignoreddimen      or 0, -- this will go away
 
         baseline_skip                = tex.baselineskip         or 0,
+        lineskip                     = tex.lineskip             or 0,
         line_skip_limit              = tex.lineskiplimit        or 0,
 
         prev_depth                   = texnest[texnest.ptr].prevdepth,
@@ -2281,13 +2303,19 @@ function constructors.methods.basic(head,d)
                 if cur_p.subtype == userkern_code then
                     kern_break(par,cur_p,first_p, checked_expansion)
                 else
-                    active_width.size = active_width.size + cur_p.kern
-                    if checked_expansion and cur_p.subtype == kerning_code then
-                        local d = cur_p.kern
-                        if d ~= 0 then
+                    local d = cur_p.kern
+                    of d ~= 0 then 
+                        active_width.size = active_width.size + d
+                        if checked_expansion and expand_kerns and (cur_p.subtype == kerning_code or cur_p[a_fontkern]) then
                             local stretch, shrink = kern_stretch_shrink(cur_p,d)
-                            active_width.adjust_stretch = active_width.adjust_stretch + stretch
-                            active_width.adjust_shrink  = active_width.adjust_shrink  + shrink
+                            if expand_kerns == "stretch" then
+                                active_width.adjust_stretch = active_width.adjust_stretch + stretch
+                            elseif expand_kerns == "shrink" then
+                                active_width.adjust_shrink  = active_width.adjust_shrink  + shrink
+                            else
+                                active_width.adjust_stretch = active_width.adjust_stretch + stretch
+                                active_width.adjust_shrink  = active_width.adjust_shrink  + shrink
+                            end
                         end
                     end
                 end
@@ -2794,17 +2822,21 @@ local function hpack(head,width,method,direction) -- fast version when head = ni
             local kern = current.kern
             if kern == 0 then
                 -- no kern
-            elseif current.subtype == kerning_code then -- check p.kern
-                if cal_expand_ratio then
+            else
+                if cal_expand_ratio and expand_kerns and current.subtype == kerning_code or current[a_fontkern] then -- check p.kern
                     local stretch, shrink = kern_stretch_shrink(current,kern)
-                    font_stretch = font_stretch + stretch
-                    font_shrink  = font_shrink + shrink
+                    if expand_kerns == "stretch" then
+                        font_stretch = font_stretch + stretch
+                    elseif expand_kerns == "shrink" then
+                        font_shrink  = font_shrink + shrink
+                    else
+                        font_stretch = font_stretch + stretch
+                        font_shrink  = font_shrink + shrink
+                    end
                     expansion_index = expansion_index + 1
                     expansion_stack[expansion_index] = current
                 end
                 natural = natural + kern
-            else
-                natural = natural + kern
             end
             current = current.next
         elseif id == disc_code then
@@ -2973,7 +3005,7 @@ local function hpack(head,width,method,direction) -- fast version when head = ni
         -- natural width smaller than requested width
         local order = (total_stretch[4] ~= 0 and 4 or total_stretch[3] ~= 0 and 3) or
                       (total_stretch[2] ~= 0 and 2 or total_stretch[1] ~= 0 and 1) or 0
-        local correction = 0
+--         local correction = 0
         if cal_expand_ratio and order == 0 and font_stretch > 0 then -- check sign of font_stretch
             font_expand_ratio = delta/font_stretch
 
@@ -2996,17 +3028,17 @@ local function hpack(head,width,method,direction) -- fast version when head = ni
                         setnodecolor(g,"hz:positive")
                     end
                     e = font_expand_ratio * data.glyphstretch / 1000
-                    correction = correction + (e / 1000) * g.width
+--                     correction = correction + (e / 1000) * g.width
                 else
                     local kern = g.kern
                     local stretch, shrink = kern_stretch_shrink(g,kern)
                     e = font_expand_ratio * stretch / 1000
-                    correction = correction + (e / 1000) * kern
+--                     correction = correction + (e / 1000) * kern
                 end
                 g.expansion_factor = e
             end
         end
-        delta = delta - correction
+--         delta = delta - correction
         local tso = total_stretch[order]
         if tso ~= 0 then
             hlist.glue_sign  = 1
@@ -3033,7 +3065,7 @@ local function hpack(head,width,method,direction) -- fast version when head = ni
         -- natural width larger than requested width
         local order = total_shrink[4] ~= 0 and 4 or total_shrink[3] ~= 0 and 3
                    or total_shrink[2] ~= 0 and 2 or total_shrink[1] ~= 0 and 1 or 0
-        local correction = 0
+--         local correction = 0
         if cal_expand_ratio and order == 0 and font_shrink > 0 then -- check sign of font_shrink
             font_expand_ratio = delta/font_shrink
 
@@ -3060,17 +3092,17 @@ local function hpack(head,width,method,direction) -- fast version when head = ni
                  -- local eps = g.width - (1 + d / 1000000) * g.width
                  -- correction = correction + eps
                  -- e = d
-                    correction = correction + (e / 1000) * g.width
+--                     correction = correction + (e / 1000) * g.width
                 else
                     local kern = g.kern
                     local stretch, shrink = kern_stretch_shrink(g,kern)
                     e = font_expand_ratio * shrink / 1000
-                    correction = correction + (e / 1000) * kern
+--                     correction = correction + (e / 1000) * kern
                 end
                 g.expansion_factor = e
             end
         end
-        delta = delta - correction
+--         delta = delta - correction
         local tso = total_shrink[order]
         if tso ~= 0 then
             hlist.glue_sign  = 2
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index ee9b2305c..b5cd4633c 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index c4a37bf21..b504da2ad 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 321bb7700..688cfecb5 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 12/19/13 00:11:25
+-- merge date  : 12/20/13 00:55:14
 
 do -- begin closure to overcome local limits and interference
 
@@ -6435,8 +6435,25 @@ registerdirective("fonts.otf.loader.pack",function(v) packdata=v end)
 registerdirective("fonts.otf.loader.syncspace",function(v) syncspace=v end)
 registerdirective("fonts.otf.loader.forcenotdef",function(v) forcenotdef=v end)
 registerdirective("fonts.otf.loader.overloadkerns",function(v) overloadkerns=v end)
+function otf.fileformat(filename)
+  local leader=lower(io.loadchunk(filename,4))
+  local suffix=lower(file.suffix(filename))
+  if leader=="otto" then
+    return "opentype","otf",suffix=="otf"
+  elseif leader=="ttcf" then
+    return "truetype","ttc",suffix=="ttc"
+  elseif suffix=="ttc" then
+    return "truetype","ttc",true
+  else
+    return "truetype","ttf",suffix=="ttf"
+  end
+end
 local function otf_format(filename)
-  return formats[lower(file.suffix(filename))]
+  local format,suffix,okay=otf.fileformat(filename)
+  if not okay then
+    report_otf("font %a is actually an %a file",filename,format)
+  end
+  return suffix
 end
 local function load_featurefile(raw,featurefile)
   if featurefile and featurefile~="" then
@@ -8831,7 +8848,7 @@ nodes.injections=nodes.injections or {}
 local injections=nodes.injections
 local nodecodes=nodes.nodecodes
 local glyph_code=nodecodes.glyph
-local kern_disc=nodecodes.disc
+local disc_code=nodecodes.disc
 local kern_code=nodecodes.kern
 local nuts=nodes.nuts
 local nodepool=nuts.pool