beta 2014.09.27 14:46

13 files changed, 335 insertions, 50 deletions

diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv
index 58f53214e..0083e7aee 100644
--- a/tex/context/base/cont-new.mkiv
+++ b/tex/context/base/cont-new.mkiv
@@ -11,7 +11,7 @@
 %C therefore copyrighted by \PRAGMA. See mreadme.pdf for
 %C details.
 
-\newcontextversion{2014.09.26 11:42}
+\newcontextversion{2014.09.27 14:46}
 
 %D This file is loaded at runtime, thereby providing an excellent place for
 %D hacks, patches, extensions and new features.
diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf
index 71b88d27b..ae0441b81 100644
--- a/tex/context/base/context-version.pdf
+++ b/tex/context/base/context-version.pdf
diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv
index 509472134..6e183c88b 100644
--- a/tex/context/base/context.mkiv
+++ b/tex/context/base/context.mkiv
@@ -28,7 +28,7 @@
 %D up and the dependencies are more consistent.
 
 \edef\contextformat {\jobname}
-\edef\contextversion{2014.09.26 11:42}
+\edef\contextversion{2014.09.27 14:46}
 \edef\contextkind   {beta}
 
 %D For those who want to use this:
diff --git a/tex/context/base/font-agl.lua b/tex/context/base/font-agl.lua
index 5ee34b028..42a41a15d 100644
--- a/tex/context/base/font-agl.lua
+++ b/tex/context/base/font-agl.lua
@@ -657,11 +657,15 @@ end
 -- We load this table only when needed. We could use a loading mechanism
 -- return the table but there are no more vectors like this so why bother.
 
-fonts.encodings = fonts.encodings or { }
-
-fonts.encodings.agl = {
+local agl = {
     names    = names,     -- unicode -> name
     unicodes = unicodes,  -- name -> unicode
     synonyms = synonyms,  -- merged into the other two
     extras   = extras,    -- merged into the other two
 }
+
+fonts               = fonts or { }
+fonts.encodings     = fonts.encodings or { }
+fonts.encodings.agl = agl
+
+return agl
diff --git a/tex/context/base/font-cid.lua b/tex/context/base/font-cid.lua
index e4b565313..0eaacdfbd 100644
--- a/tex/context/base/font-cid.lua
+++ b/tex/context/base/font-cid.lua
@@ -74,7 +74,7 @@ local function loadcidfile(filename)
             ordering   = ordering,
             filename   = filename,
             unicodes   = unicodes,
-            names      = names
+            names      = names,
         }
     end
 end
@@ -112,15 +112,28 @@ function cid.getmap(specification)
         report_otf("invalid cidinfo specification, table expected")
         return
     end
-    local registry = specification.registry
-    local ordering = specification.ordering
+    local registry   = specification.registry
+    local ordering   = specification.ordering
     local supplement = specification.supplement
-    -- check for already loaded file
-    local filename = format(registry,ordering,supplement)
-    local found = cidmap[lower(filename)]
+    local filename   = format(registry,ordering,supplement)
+    local lowername  = lower(filename)
+    local found      = cidmap[lowername]
     if found then
         return found
     end
+    if ordering == "Identity" then
+        local found = {
+            supplement = supplement,
+            registry   = registry,
+            ordering   = ordering,
+            filename   = filename,
+            unicodes   = { },
+            names      = { },
+        }
+        cidmap[lowername] = found
+        return found
+    end
+    -- check for already loaded file
     if trace_loading then
         report_otf("cidmap needed, registry %a, ordering %a, supplement %a",registry,ordering,supplement)
     end
diff --git a/tex/context/base/font-map.lua b/tex/context/base/font-map.lua
index e5f587105..429c73597 100644
--- a/tex/context/base/font-map.lua
+++ b/tex/context/base/font-map.lua
@@ -163,6 +163,7 @@ function mappings.addtounicode(data,filename)
     local properties   = data.properties
     local descriptions = data.descriptions
     local unicodes     = resources.unicodes
+    local lookuptypes  = resources.lookuptypes
     if not unicodes then
         return
     end
@@ -175,8 +176,10 @@ function mappings.addtounicode(data,filename)
     local private       = fonts.constructors.privateoffset
     local unknown       = format("%04X",utfbyte("?"))
     local unicodevector = fonts.encodings.agl.unicodes -- loaded runtime in context
+    ----- namevector    = fonts.encodings.agl.names    -- loaded runtime in context
     local tounicode     = { }
     local originals     = { }
+    local missing       = { }
     resources.tounicode = tounicode
     resources.originals = originals
     local lumunic, uparser, oparser
@@ -311,12 +314,144 @@ function mappings.addtounicode(data,filename)
                     end
                 end
             end
+            -- check using substitutes and alternates
+            --
+            if not unicode then
+                missing[name] = true
+            end
          -- if not unicode then
          --     originals[index] = 0xFFFD
          --     tounicode[index] = "FFFD"
          -- end
         end
     end
+    if next(missing) then
+--         inspect(missing)
+        local guess  = { }
+        -- helper
+        local function check(gname,code,unicode)
+            local description = descriptions[code]
+            -- no need to add a self reference
+            local variant = description.name
+            if variant == gname then
+                return
+            end
+            -- the variant already has a unicode (normally that resultrs in a default tounicode to self)
+            local unic = unicodes[variant]
+            if unic == -1 or unic >= private or (unic >= 0xE000 and unic <= 0xF8FF) or unic == 0xFFFE or unic == 0xFFFF then
+                -- no default mapping and therefore maybe no tounicode yet
+            else
+                return
+            end
+            -- the variant already has a tounicode
+            local index = descriptions[code].index
+            if tounicode[index] then
+                return
+            end
+            -- add to the list
+            local g = guess[variant]
+            if g then
+                g[gname] = unicode
+            else
+                guess[variant] = { [gname] = unicode }
+            end
+        end
+        --
+        for unicode, description in next, descriptions do
+            local slookups = description.slookups
+            if slookups then
+                local gname = description.name
+                for tag, data in next, slookups do
+                    local lookuptype = lookuptypes[tag]
+                    if lookuptype == "alternate" then
+                        for i=1,#data do
+                            check(gname,data[i],unicode)
+                        end
+                    elseif lookuptype == "substitution" then
+                        check(gname,data,unicode)
+                    end
+                end
+            end
+            local mlookups = description.mlookups
+            if mlookups then
+                local gname = description.name
+                for tag, list in next, mlookups do
+                    local lookuptype = lookuptypes[tag]
+                    if lookuptype == "alternate" then
+                        for i=1,#list do
+                            local data = list[i]
+                            for i=1,#data do
+                                check(gname,data[i],unicode)
+                            end
+                        end
+                    elseif lookuptype == "substitution" then
+                        for i=1,#list do
+                            check(gname,list[i],unicode)
+                        end
+                    end
+                end
+            end
+        end
+        -- resolve references
+        local done = true
+        while done do
+            done = false
+            for k, v in next, guess do
+                if type(v) ~= "number" then
+                    for kk, vv in next, v do
+                        if vv == -1 or vv >= private or (vv >= 0xE000 and vv <= 0xF8FF) or vv == 0xFFFE or vv == 0xFFFF then
+                            local uu = guess[kk]
+                            if type(uu) == "number" then
+                                guess[k] = uu
+                                done = true
+                            end
+                        else
+                            guess[k] = vv
+                            done = true
+                        end
+                    end
+                end
+            end
+        end
+        -- generate tounicodes
+        for k, v in next, guess do
+            if type(v) == "number" then
+                guess[k] = tounicode16(v)
+            else
+                local t = nil
+                local l = lower(k)
+                local u = unicodes[l]
+                if not u then
+                    -- forget about it
+                elseif u == -1 or u >= private or (u >= 0xE000 and u <= 0xF8FF) or u == 0xFFFE or u == 0xFFFF then
+                    t = tounicode[descriptions[u].index]
+                else
+                 -- t = u
+                end
+                if t then
+                    guess[k] = t
+                else
+                    guess[k] = "FFFD"
+                end
+            end
+        end
+        local orphans = 0
+        local guessed = 0
+        for k, v in next, guess do
+            tounicode[descriptions[unicodes[k]].index] = v
+            if v == "FFFD" then
+                orphans = orphans + 1
+                guess[k] = false
+            else
+                guessed = guessed + 1
+                guess[k] = true
+            end
+        end
+     -- resources.nounicode = guess -- only when we test things
+        if trace_loading and orphans > 0 or guessed > 0 then
+            report_fonts("%s glyphs with no related unicode, %s guessed, %s orphans",guessed+orphans,guessed,orphans)
+        end
+    end
     if trace_mapping then
         for unic, glyph in table.sortedhash(descriptions) do
             local name  = glyph.name
diff --git a/tex/context/base/font-mis.lua b/tex/context/base/font-mis.lua
index 1b50977ea..b934837f4 100644
--- a/tex/context/base/font-mis.lua
+++ b/tex/context/base/font-mis.lua
@@ -22,7 +22,7 @@ local handlers = fonts.handlers
 handlers.otf   = handlers.otf or { }
 local otf      = handlers.otf
 
-otf.version    = otf.version or 2.760
+otf.version    = otf.version or 2.762
 otf.cache      = otf.cache   or containers.define("fonts", "otf", otf.version, true)
 
 function otf.loadcached(filename,format,sub)
diff --git a/tex/context/base/font-otf.lua b/tex/context/base/font-otf.lua
index c1b23983d..c1bb4419c 100644
--- a/tex/context/base/font-otf.lua
+++ b/tex/context/base/font-otf.lua
@@ -48,7 +48,7 @@ local otf                = fonts.handlers.otf
 
 otf.glists               = { "gsub", "gpos" }
 
-otf.version              = 2.760 -- beware: also sync font-mis.lua
+otf.version              = 2.762 -- beware: also sync font-mis.lua
 otf.cache                = containers.define("fonts", "otf", otf.version, true)
 
 local fontdata           = fonts.hashes.identifiers
@@ -255,7 +255,7 @@ local ordered_enhancers = {
     "analyze glyphs",
     "analyze math",
 
-    "prepare tounicode", -- maybe merge with prepare
+ -- "prepare tounicode",
 
     "reorganize lookups",
     "reorganize mark classes",
@@ -274,6 +274,8 @@ local ordered_enhancers = {
     "check metadata",
     "check extra features", -- after metadata
 
+    "prepare tounicode",
+
     "check encoding", -- moved
     "add duplicates",
 
@@ -684,17 +686,18 @@ actions["prepare glyphs"] = function(data,filename,raw)
                     if includesubfonts then
                         metadata.subfonts[cidindex] = somecopy(subfont)
                     end
+                    -- we have delayed loading so we cannot use next
                     for index=0,subfont.glyphcnt-1 do -- we could take the previous glyphcnt instead of 0
                         local glyph = cidglyphs[index]
                         if glyph then
                             local unicode = glyph.unicode
-if     unicode >= 0x00E000 and unicode <= 0x00F8FF then
+                            if     unicode >= 0x00E000 and unicode <= 0x00F8FF then
                                 unicode = -1
-elseif unicode >= 0x0F0000 and unicode <= 0x0FFFFD then
+                            elseif unicode >= 0x0F0000 and unicode <= 0x0FFFFD then
                                 unicode = -1
-elseif unicode >= 0x100000 and unicode <= 0x10FFFD then
+                            elseif unicode >= 0x100000 and unicode <= 0x10FFFD then
                                 unicode = -1
-end
+                            end
                             local name = glyph.name or cidnames[index]
                             if not unicode or unicode == -1 then -- or unicode >= criterium then
                                 unicode = cidunicodes[index]
@@ -736,7 +739,6 @@ end
                                 nofunicodes = nofunicodes + 1
                             end
                             indices[index] = unicode -- each index is unique (at least now)
-
                             local description = {
                              -- width       = glyph.width,
                                 boundingbox = glyph.boundingbox,
@@ -745,7 +747,6 @@ end
                                 index       = index,
                                 glyph       = glyph,
                             }
-
                             descriptions[unicode] = description
                         else
                          -- report_otf("potential problem: glyph %U is used but empty",index)
diff --git a/tex/context/base/lang-hyp.lua b/tex/context/base/lang-hyp.lua
index 3b5eac9ba..87216cc8b 100644
--- a/tex/context/base/lang-hyp.lua
+++ b/tex/context/base/lang-hyp.lua
@@ -639,25 +639,25 @@ else
  -- traditional.loadpatterns("nl","lang-nl")
  -- traditional.loadpatterns("de","lang-de")
 
-    traditional.registerpattern("nl","e1ë",      { start = 1, length = 2, before = "e",  after = "e"  } )
-    traditional.registerpattern("nl","oo1ë",     { start = 2, length = 3, before = "o",  after = "e"  } )
-    traditional.registerpattern("de","qqxc9xkqq",{ start = 3, length = 4, before = "ab", after = "cd" } )
-
-    local specification = {
-        lefthyphenmin   = 2,
-        righthyphenmin  = 2,
-        lefthyphenchar  = "<",
-        righthyphenchar = ">",
-    }
-
-    print("reëel",       traditional.injecthyphens(dictionaries.nl,"reëel",       specification),"r{e>}{<e}{eë}el")
-    print("reeëel",      traditional.injecthyphens(dictionaries.nl,"reeëel",      specification),"re{e>}{<e}{eë}el")
-    print("rooëel",      traditional.injecthyphens(dictionaries.nl,"rooëel",      specification),"r{o>}{<e}{ooë}el")
-
-    print(   "qxcxkq",   traditional.injecthyphens(dictionaries.de,   "qxcxkq",   specification),"")
-    print(  "qqxcxkqq",  traditional.injecthyphens(dictionaries.de,  "qqxcxkqq",  specification),"")
-    print( "qqqxcxkqqq", traditional.injecthyphens(dictionaries.de, "qqqxcxkqqq", specification),"")
-    print("qqqqxcxkqqqq",traditional.injecthyphens(dictionaries.de,"qqqqxcxkqqqq",specification),"")
+--     traditional.registerpattern("nl","e1ë",      { start = 1, length = 2, before = "e",  after = "e"  } )
+--     traditional.registerpattern("nl","oo1ë",     { start = 2, length = 3, before = "o",  after = "e"  } )
+--     traditional.registerpattern("de","qqxc9xkqq",{ start = 3, length = 4, before = "ab", after = "cd" } )
+
+--     local specification = {
+--         lefthyphenmin   = 2,
+--         righthyphenmin  = 2,
+--         lefthyphenchar  = "<",
+--         righthyphenchar = ">",
+--     }
+
+--     print("reëel",       traditional.injecthyphens(dictionaries.nl,"reëel",       specification),"r{e>}{<e}{eë}el")
+--     print("reeëel",      traditional.injecthyphens(dictionaries.nl,"reeëel",      specification),"re{e>}{<e}{eë}el")
+--     print("rooëel",      traditional.injecthyphens(dictionaries.nl,"rooëel",      specification),"r{o>}{<e}{ooë}el")
+
+--     print(   "qxcxkq",   traditional.injecthyphens(dictionaries.de,   "qxcxkq",   specification),"")
+--     print(  "qqxcxkqq",  traditional.injecthyphens(dictionaries.de,  "qqxcxkqq",  specification),"")
+--     print( "qqqxcxkqqq", traditional.injecthyphens(dictionaries.de, "qqqxcxkqqq", specification),"")
+--     print("qqqqxcxkqqqq",traditional.injecthyphens(dictionaries.de,"qqqqxcxkqqqq",specification),"")
 
 end
 
diff --git a/tex/context/base/s-fnt-21.mkiv b/tex/context/base/s-fnt-21.mkiv
index 588c98016..10d5a4623 100644
--- a/tex/context/base/s-fnt-21.mkiv
+++ b/tex/context/base/s-fnt-21.mkiv
@@ -47,7 +47,7 @@
 
 \endinput
 
-% \usemodule[fnt-20]
+% \usemodule[fnt-21]
 %
 % \starttext
 %
diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf
index 6898e14f2..540ac5255 100644
--- a/tex/context/base/status-files.pdf
+++ b/tex/context/base/status-files.pdf
diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf
index aa73bb159..0b92a158c 100644
--- a/tex/context/base/status-lua.pdf
+++ b/tex/context/base/status-lua.pdf
diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua
index 4aa03e6a4..98e98d806 100644
--- a/tex/generic/context/luatex/luatex-fonts-merged.lua
+++ b/tex/generic/context/luatex/luatex-fonts-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 09/26/14 11:42:21
+-- merge date  : 09/27/14 14:46:07
 
 do -- begin closure to overcome local limits and interference
 
@@ -5045,7 +5045,7 @@ local function loadcidfile(filename)
       ordering=ordering,
       filename=filename,
       unicodes=unicodes,
-      names=names
+      names=names,
     }
   end
 end
@@ -5082,10 +5082,23 @@ function cid.getmap(specification)
   local ordering=specification.ordering
   local supplement=specification.supplement
   local filename=format(registry,ordering,supplement)
-  local found=cidmap[lower(filename)]
+  local lowername=lower(filename)
+  local found=cidmap[lowername]
   if found then
     return found
   end
+  if ordering=="Identity" then
+    local found={
+      supplement=supplement,
+      registry=registry,
+      ordering=ordering,
+      filename=filename,
+      unicodes={},
+      names={},
+    }
+    cidmap[lowername]=found
+    return found
+  end
   if trace_loading then
     report_otf("cidmap needed, registry %a, ordering %a, supplement %a",registry,ordering,supplement)
   end
@@ -5224,6 +5237,7 @@ function mappings.addtounicode(data,filename)
   local properties=data.properties
   local descriptions=data.descriptions
   local unicodes=resources.unicodes
+  local lookuptypes=resources.lookuptypes
   if not unicodes then
     return
   end
@@ -5233,9 +5247,10 @@ function mappings.addtounicode(data,filename)
   unicodes['zwnj']=unicodes['zwnj']  or 0x200C
   local private=fonts.constructors.privateoffset
   local unknown=format("%04X",utfbyte("?"))
-  local unicodevector=fonts.encodings.agl.unicodes 
+  local unicodevector=fonts.encodings.agl.unicodes
   local tounicode={}
   local originals={}
+  local missing={}
   resources.tounicode=tounicode
   resources.originals=originals
   local lumunic,uparser,oparser
@@ -5354,6 +5369,123 @@ function mappings.addtounicode(data,filename)
           end
         end
       end
+      if not unicode then
+        missing[name]=true
+      end
+    end
+  end
+  if next(missing) then
+    local guess={}
+    local function check(gname,code,unicode)
+      local description=descriptions[code]
+      local variant=description.name
+      if variant==gname then
+        return
+      end
+      local unic=unicodes[variant]
+      if unic==-1 or unic>=private or (unic>=0xE000 and unic<=0xF8FF) or unic==0xFFFE or unic==0xFFFF then
+      else
+        return
+      end
+      local index=descriptions[code].index
+      if tounicode[index] then
+        return
+      end
+      local g=guess[variant]
+      if g then
+        g[gname]=unicode
+      else
+        guess[variant]={ [gname]=unicode }
+      end
+    end
+    for unicode,description in next,descriptions do
+      local slookups=description.slookups
+      if slookups then
+        local gname=description.name
+        for tag,data in next,slookups do
+          local lookuptype=lookuptypes[tag]
+          if lookuptype=="alternate" then
+            for i=1,#data do
+              check(gname,data[i],unicode)
+            end
+          elseif lookuptype=="substitution" then
+            check(gname,data,unicode)
+          end
+        end
+      end
+      local mlookups=description.mlookups
+      if mlookups then
+        local gname=description.name
+        for tag,list in next,mlookups do
+          local lookuptype=lookuptypes[tag]
+          if lookuptype=="alternate" then
+            for i=1,#list do
+              local data=list[i]
+              for i=1,#data do
+                check(gname,data[i],unicode)
+              end
+            end
+          elseif lookuptype=="substitution" then
+            for i=1,#list do
+              check(gname,list[i],unicode)
+            end
+          end
+        end
+      end
+    end
+    local done=true
+    while done do
+      done=false
+      for k,v in next,guess do
+        if type(v)~="number" then
+          for kk,vv in next,v do
+            if vv==-1 or vv>=private or (vv>=0xE000 and vv<=0xF8FF) or vv==0xFFFE or vv==0xFFFF then
+              local uu=guess[kk]
+              if type(uu)=="number" then
+                guess[k]=uu
+                done=true
+              end
+            else
+              guess[k]=vv
+              done=true
+            end
+          end
+        end
+      end
+    end
+    for k,v in next,guess do
+      if type(v)=="number" then
+        guess[k]=tounicode16(v)
+      else
+        local t=nil
+        local l=lower(k)
+        local u=unicodes[l]
+        if not u then
+        elseif u==-1 or u>=private or (u>=0xE000 and u<=0xF8FF) or u==0xFFFE or u==0xFFFF then
+          t=tounicode[descriptions[u].index]
+        else
+        end
+        if t then
+          guess[k]=t
+        else
+          guess[k]="FFFD"
+        end
+      end
+    end
+    local orphans=0
+    local guessed=0
+    for k,v in next,guess do
+      tounicode[descriptions[unicodes[k]].index]=v
+      if v=="FFFD" then
+        orphans=orphans+1
+        guess[k]=false
+      else
+        guessed=guessed+1
+        guess[k]=true
+      end
+    end
+    if trace_loading and orphans>0 or guessed>0 then
+      report_fonts("%s glyphs with no related unicode, %s guessed, %s orphans",guessed+orphans,guessed,orphans)
     end
   end
   if trace_mapping then
@@ -6706,7 +6838,7 @@ local report_otf=logs.reporter("fonts","otf loading")
 local fonts=fonts
 local otf=fonts.handlers.otf
 otf.glists={ "gsub","gpos" }
-otf.version=2.760 
+otf.version=2.762 
 otf.cache=containers.define("fonts","otf",otf.version,true)
 local fontdata=fonts.hashes.identifiers
 local chardata=characters and characters.data 
@@ -6869,7 +7001,6 @@ local ordered_enhancers={
   "prepare lookups",
   "analyze glyphs",
   "analyze math",
-  "prepare tounicode",
   "reorganize lookups",
   "reorganize mark classes",
   "reorganize anchor classes",
@@ -6882,6 +7013,7 @@ local ordered_enhancers={
   "check glyphs",
   "check metadata",
   "check extra features",
+  "prepare tounicode",
   "check encoding",
   "add duplicates",
   "cleanup tables",
@@ -7249,13 +7381,13 @@ actions["prepare glyphs"]=function(data,filename,raw)
             local glyph=cidglyphs[index]
             if glyph then
               local unicode=glyph.unicode
-if   unicode>=0x00E000 and unicode<=0x00F8FF then
+              if   unicode>=0x00E000 and unicode<=0x00F8FF then
                 unicode=-1
-elseif unicode>=0x0F0000 and unicode<=0x0FFFFD then
+              elseif unicode>=0x0F0000 and unicode<=0x0FFFFD then
                 unicode=-1
-elseif unicode>=0x100000 and unicode<=0x10FFFD then
+              elseif unicode>=0x100000 and unicode<=0x10FFFD then
                 unicode=-1
-end
+              end
               local name=glyph.name or cidnames[index]
               if not unicode or unicode==-1 then 
                 unicode=cidunicodes[index]