2016-04-21 19:39:00

10 files changed, 117 insertions, 174 deletions

diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv
index 6e3e5ba60..caa9c31df 100644
--- a/tex/context/base/mkiv/cont-new.mkiv
+++ b/tex/context/base/mkiv/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2016.04.21 12:13}
+\newcontextversion{2016.04.21 19:35}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv
index 17eefd0fe..9f4ef2327 100644
--- a/tex/context/base/mkiv/context.mkiv
+++ b/tex/context/base/mkiv/context.mkiv
@@ -39,7 +39,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2016.04.21 12:13}
+\edef\contextversion{2016.04.21 19:35}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/mkiv/font-mis.lua b/tex/context/base/mkiv/font-mis.lua
index 9d9c39c58..06bf22049 100644
--- a/tex/context/base/mkiv/font-mis.lua
+++ b/tex/context/base/mkiv/font-mis.lua
@@ -6,11 +6,6 @@ if not modules then modules = { } end modules ['font-mis'] = {
     license   = "see context related readme files"
 }
 
-local next = next
-local lower, strip = string.lower, string.strip
-
--- also used in other scripts so we need to check some tables:
-
 fonts          = fonts or { }
 
 fonts.helpers  = fonts.helpers or { }
@@ -22,96 +17,35 @@ local handlers = fonts.handlers
 handlers.otf   = handlers.otf or { }
 local otf      = handlers.otf
 
-otf.version    = otf.version or 2.824
-otf.cache      = otf.cache   or containers.define("fonts", "otf", otf.version, true)
-
-local fontloader    = fontloader
-local font_to_table = fontloader.to_table
-local open_font     = fontloader.open
-local get_font_info = fontloader.info
-local close_font    = fontloader.close
+local readers  = otf.readers
 
-function otf.loadcached(filename,format,sub)
-    -- no recache when version mismatch
-    local name = file.basename(file.removesuffix(filename))
-    if sub == "" then sub = false end
-    local hash = name
-    if sub then
-        hash = hash .. "-" .. sub
-    end
-    hash = containers.cleanname(hash)
-    local data = containers.read(otf.cache, hash)
-    if data and not data.verbose then
-        otf.enhancers.unpack(data)
-        return data
-    else
-        return nil
-    end
-end
+if readers then
 
-local featuregroups = { "gsub", "gpos" }
+    otf.version = otf.version or 3.018
+    otf.cache   = otf.cache   or containers.define("fonts", "otl", otf.version, true)
 
-function fonts.helpers.getfeatures(name,t,script,language) -- maybe per font type
-    local t = lower(t or (name and file.suffix(name)) or "")
-    if t == "otf" or t == "ttf" or t == "ttc" or t == "dfont" then
-        local filename = resolvers.findfile(name,t) or ""
+    function fonts.helpers.getfeatures(name,save)
+        local filename = resolvers.findfile(name) or ""
         if filename ~= "" then
-            local data = otf.loadcached(filename)
-            if data and data.resources and data.resources.features then
-                return  data.resources.features
+            local name      = file.removesuffix(file.basename(filename))
+            local cleanname = containers.cleanname(name)
+            local data      = containers.read(otf.cache,cleanname)
+            if data then
+                readers.unpack(data)
             else
-                local ff = open_font(filename)
-                if ff then
-                    local data = font_to_table(ff)
-                    close_font(ff)
-                    local features = { }
-                    for k=1,#featuregroups do
-                        local what = featuregroups[k]
-                        local dw = data[what]
-                        if dw then
-                            local f = { }
-                            features[what] = f
-                            for i=1,#dw do
-                                local d = dw[i]
-                                local dfeatures = d.features
-                                if dfeatures then
-                                    for i=1,#dfeatures do
-                                        local df = dfeatures[i]
-                                        local tag = strip(lower(df.tag))
-                                        local ft = f[tag] if not ft then ft = {} f[tag] = ft end
-                                        local dfscripts = df.scripts
-                                        for i=1,#dfscripts do
-                                            local ds = dfscripts[i]
-                                            local scri = strip(lower(ds.script))
-                                            local fts = ft[scri] if not fts then fts = {} ft[scri] = fts end
-                                            local dslangs = ds.langs
-                                            for i=1,#dslangs do
-                                                local lang = dslangs[i]
-                                                lang = strip(lower(lang))
-                                                if scri == script then
-                                                    if lang == language then
-                                                        fts[lang] = 'sl'
-                                                    else
-                                                        fts[lang] = 's'
-                                                    end
-                                                else
-                                                    if lang == language then
-                                                        fts[lang] = 'l'
-                                                    else
-                                                        fts[lang] = true
-                                                    end
-                                                end
-                                            end
-                                        end
-                                    end
-                                end
-                            end
-                        end
-                    end
-                    return features
+                data = readers.loadfont(filename)
+                if data and save then
+                    containers.write(otf.cache,cleanname,data)
                 end
             end
+            return data and data.resources and data.resources.features
         end
     end
-    return nil, nil
+
+else
+
+    function fonts.helpers.getfeatures(name)
+        -- not supported
+    end
+
 end
diff --git a/tex/context/base/mkiv/font-otr.lua b/tex/context/base/mkiv/font-otr.lua
index 27bb6a621..a9ad73906 100644
--- a/tex/context/base/mkiv/font-otr.lua
+++ b/tex/context/base/mkiv/font-otr.lua
@@ -60,15 +60,15 @@ if not modules then modules = { } end modules ['font-otr'] = {
 -- faster but it might not be the real bottleneck as we still need to juggle data. It
 -- is probably more memory efficient as no intermediate strings are involved.
 
-if not characters then
-    require("char-def")
-    require("char-ini")
-end
+-- if not characters then
+--     require("char-def")
+--     require("char-ini")
+-- end
 
 local next, type, unpack = next, type, unpack
 local byte, lower, char, strip, gsub = string.byte, string.lower, string.char, string.strip, string.gsub
 local bittest = bit32.btest
-local concat, remove, unpack = table.concat, table.remov, table.unpack
+local concat, remove, unpack, fastcopy = table.concat, table.remov, table.unpack, table.fastcopy
 local floor, mod, abs, sqrt, round = math.floor, math.mod, math.abs, math.sqrt, math.round
 local P, R, S, C, Cs, Cc, Ct, Carg, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.Carg, lpeg.Cmt
 local lpegmatch = lpeg.match
@@ -1186,6 +1186,8 @@ local sequence = {
     { 3,  0,  6 },
     -- variants
     { 0,  5, 14 },
+    -- last resort ranges
+    { 3, 10, 13 },
 }
 
 -- local sequence = {
@@ -1388,7 +1390,7 @@ formatreaders[12] = function(f,fontdata,offset)
         local last  = readulong(f)
         local index = readulong(f)
         if trace_cmap then
-            report("format 12 from %C to %C",first,last)
+            report("format 12 from %C to %C starts at index %i",first,last,index)
         end
         for unicode=first,last do
             local glyph = glyphs[index]
@@ -1416,6 +1418,53 @@ formatreaders[12] = function(f,fontdata,offset)
     return nofdone
 end
 
+formatreaders[13] = function(f,fontdata,offset)
+    --
+    -- this fector is only used for simple fallback fonts
+    --
+    setposition(f,offset+2+2+4+4) -- skip format reserved length language
+    local mapping    = fontdata.mapping
+    local glyphs     = fontdata.glyphs
+    local duplicates = fontdata.duplicates
+    local nofgroups  = readulong(f)
+    local nofdone    = 0
+    for i=1,nofgroups do
+        local first = readulong(f)
+        local last  = readulong(f)
+        local index = readulong(f)
+        if first < privateoffset then
+            if trace_cmap then
+                report("format 13 from %C to %C get index %i",first,last,index)
+            end
+            local glyph   = glyphs[index]
+            local unicode = glyph.unicode
+            if not unicode then
+                unicode = first
+                glyph.unicode = unicode
+                first = first + 1
+            end
+            local list     = duplicates[unicode]
+            mapping[index] = unicode
+            if not list then
+                list = { }
+                duplicates[unicode] = list
+            end
+            if last >= privateoffset then
+                local limit = privateoffset - 1
+                report("format 13 from %C to %C pruned to %C",first,last,limit)
+                last = limit
+            end
+            for unicode=first,last do
+                list[unicode] = true
+            end
+            nofdone = nofdone + last - first + 1
+        else
+            report("format 13 from %C to %C ignored",first,last)
+        end
+    end
+    return nofdone
+end
+
 formatreaders[14] = function(f,fontdata,offset)
     if offset and offset ~= 0 then
         setposition(f,offset)
@@ -2225,7 +2274,7 @@ function readers.extend(fontdata)
     end
 end
 
---
+-- for now .. this will move to a context specific file
 
 if fonts.hashes then
 
diff --git a/tex/context/base/mkiv/font-oup.lua b/tex/context/base/mkiv/font-oup.lua
index a99aaf49a..3b6d8ea53 100644
--- a/tex/context/base/mkiv/font-oup.lua
+++ b/tex/context/base/mkiv/font-oup.lua
@@ -367,22 +367,28 @@ local function copyduplicates(fontdata)
             for u, d in next, duplicates do
                 local du = descriptions[u]
                 if du then
-                    local t  = { f_character_y(u), "@", f_index(du.index), "->" }
+                    local t = { f_character_y(u), "@", f_index(du.index), "->" }
+                    local n = 0
+                    local m = 25
                     for u in next, d do
                         if descriptions[u] then
-                            t[#t+1] = f_character_n(u)
+                            if n < m then
+                                t[n+4] = f_character_n(u)
+                            end
                         else
                             local c = copy(du)
-                         -- c.unicode = u -- maybe
+                            c.unicode = u -- better this way
                             descriptions[u] = c
-                            t[#t+1] = f_character_y(u)
+                            if n < m then
+                                t[n+4] = f_character_y(u)
+                            end
                         end
+                        n = n + 1
                     end
-                    local n = #t
-                    if n > 25 then
-                        report("duplicates: %i : %s .. %s ",n,t[1],t[n])
-                    else
+                    if n <= m then
                         report("duplicates: %i : % t",n,t)
+                    else
+                        report("duplicates: %i : % t ...",n,t)
                     end
                 else
                     -- what a mess
diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf
index d7d0a1dd6..f5692f2c4 100644
--- a/tex/context/base/mkiv/status-files.pdf
+++ b/tex/context/base/mkiv/status-files.pdf
diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf
index fb974b5a2..65ec9e7c7 100644
--- a/tex/context/base/mkiv/status-lua.pdf
+++ b/tex/context/base/mkiv/status-lua.pdf
diff --git a/tex/context/base/mkiv/task-ini.lua b/tex/context/base/mkiv/task-ini.lua
index 6039efc03..eaf342234 100644
--- a/tex/context/base/mkiv/task-ini.lua
+++ b/tex/context/base/mkiv/task-ini.lua
@@ -111,9 +111,7 @@ appendaction("math",         "builders",    "builders.kernel.mlist_to_hlist")
 ------------("math",         "builders",    "noads.handlers.italics",  nil, "nohead")            -- disabled
 appendaction("math",         "builders",    "typesetters.directions.processmath")                -- disabled (has to happen pretty late)
 
-if LUATEXVERSION >= 0.895 then
-    appendaction("finalizers",   "lists",       "typesetters.paragraphs.normalize")                  -- moved here
-end
+appendaction("finalizers",   "lists",       "typesetters.paragraphs.normalize")                  -- moved here
 appendaction("finalizers",   "lists",       "typesetters.margins.localhandler")                  -- disabled
 appendaction("finalizers",   "lists",       "builders.paragraphs.keeptogether")
 ------------("finalizers",   "lists",       "nodes.handlers.graphicvadjust")                     -- todo
diff --git a/tex/context/base/mkiv/typo-bld.lua b/tex/context/base/mkiv/typo-bld.lua
index 006576f88..ce6a65baf 100644
--- a/tex/context/base/mkiv/typo-bld.lua
+++ b/tex/context/base/mkiv/typo-bld.lua
@@ -231,74 +231,30 @@ end
 -- this will be split into contribute_filter for these 4 so at some point
 -- thecheck can go away
 
-if LUATEXVERSION >= 0.895 then
-
-    function builders.buildpage_filter(groupcode)
-        -- the next check saves 1% runtime on 1000 tufte pages
-        local head = texlists.contrib_head
-        local done = false
-        if head then
-            -- called quite often ... maybe time to remove timing
-            starttiming(builders)
-            if trace_page_builder then
-                report(groupcode,head)
-            end
-            head, done = pageactions(head,groupcode)
-            stoptiming(builders)
-         -- -- doesn't work here (not passed on?)
-         -- tex.pagegoal = tex.vsize - tex.dimen.d_page_floats_inserted_top - tex.dimen.d_page_floats_inserted_bottom
-            texlists.contrib_head = head or nil -- needs checking
-         -- tex.setlist("contrib_head",head,head and nodes.tail(head))
-            return done and head or true -- no return value needed
-        else
-            -- happens quite often
-            if trace_page_builder then
-                report(groupcode)
-            end
-            return nil, false -- no return value needed
-        end
-    end
-
-else
-
-    local build_par_codes = {
-        pre_box    = true,
-        box        = true,
-        pre_adjust = true,
-        adjust     = true,
-    }
-
-    function builders.buildpage_filter(groupcode)
-        -- the next check saves 1% runtime on 1000 tufte pages
-        local head = texlists.contrib_head
-        local done = false
-        if build_par_codes[groupcode] then
-            -- also called in vbox .. we really need another callback for these four
-            normalize(head) -- a bit weird place
+function builders.buildpage_filter(groupcode)
+    -- the next check saves 1% runtime on 1000 tufte pages
+    local head = texlists.contrib_head
+    local done = false
+    if head then
+        -- called quite often ... maybe time to remove timing
+        starttiming(builders)
+        if trace_page_builder then
+            report(groupcode,head)
         end
-        --
-        if head then
-            -- called quite often ... maybe time to remove timing
-            starttiming(builders)
-            if trace_page_builder then
-                report(groupcode,head)
-            end
-            head, done = pageactions(head,groupcode)
-            stoptiming(builders)
-         -- -- doesn't work here (not passed on?)
-         -- tex.pagegoal = tex.vsize - tex.dimen.d_page_floats_inserted_top - tex.dimen.d_page_floats_inserted_bottom
-            texlists.contrib_head = head or nil -- needs checking
-         -- tex.setlist("contrib_head",head,head and nodes.tail(head))
-            return done and head or true -- no return value needed
-        else
-            -- happens quite often
-            if trace_page_builder then
-                report(groupcode)
-            end
-            return nil, false -- no return value needed
+        head, done = pageactions(head,groupcode)
+        stoptiming(builders)
+     -- -- doesn't work here (not passed on?)
+     -- tex.pagegoal = tex.vsize - tex.dimen.d_page_floats_inserted_top - tex.dimen.d_page_floats_inserted_bottom
+        texlists.contrib_head = head or nil -- needs checking
+     -- tex.setlist("contrib_head",head,head and nodes.tail(head))
+        return done and head or true -- no return value needed
+    else
+        -- happens quite often
+        if trace_page_builder then
+            report(groupcode)
         end
+        return nil, false -- no return value needed
     end
-
 end
 
 callbacks.register('vpack_filter',      builders.vpack_filter,      "vertical spacing etc")
diff --git a/tex/context/base/mkiv/typo-lin.lua b/tex/context/base/mkiv/typo-lin.lua
index b794ed13c..40a27dd39 100644
--- a/tex/context/base/mkiv/typo-lin.lua
+++ b/tex/context/base/mkiv/typo-lin.lua
@@ -243,7 +243,7 @@ function paragraphs.normalize(head,islocal)
     end
     for line in traverse_id(hlist_code,tonut(head)) do
         if getsubtype(line) == line_code and not getprop(line,"line") then
-            normalize(line,islocal)
+            normalize(line)
         end
     end
     return head, true