Merge pull request #345 from phi-gamma/master

first batch of fixes

2 files changed, 251 insertions, 82 deletions

diff --git a/src/fontloader/runtime/fontloader-basics-gen.lua b/src/fontloader/runtime/fontloader-basics-gen.lua
index c298f6d..2a68b1c 100644
--- a/src/fontloader/runtime/fontloader-basics-gen.lua
+++ b/src/fontloader/runtime/fontloader-basics-gen.lua
@@ -92,12 +92,10 @@ local remapper = {
     otf    = "opentype fonts",
     ttf    = "truetype fonts",
     ttc    = "truetype fonts",
-    dfont  = "truetype fonts", -- "truetype dictionary",
     cid    = "cid maps",
     cidmap = "cid maps",
-    fea    = "font feature files",
-    pfa    = "type1 fonts", -- this is for Khaled, in ConTeXt we don't use this!
-    pfb    = "type1 fonts", -- this is for Khaled, in ConTeXt we don't use this!
+ -- fea    = "font feature files", -- no longer supported
+    pfb    = "type1 fonts",        -- needed for vector loading
     afm    = "afm",
 }
 
diff --git a/src/fontloader/runtime/fontloader-reference.lua b/src/fontloader/runtime/fontloader-reference.lua
index e784738..5be6493 100644
--- a/src/fontloader/runtime/fontloader-reference.lua
+++ b/src/fontloader/runtime/fontloader-reference.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date  : 04/18/16 22:12:36
+-- merge date  : 04/27/16 10:18:10
 
 do -- begin closure to overcome local limits and interference
 
@@ -3693,11 +3693,8 @@ local remapper={
   otf="opentype fonts",
   ttf="truetype fonts",
   ttc="truetype fonts",
-  dfont="truetype fonts",
   cid="cid maps",
   cidmap="cid maps",
-  fea="font feature files",
-  pfa="type1 fonts",
   pfb="type1 fonts",
   afm="afm",
 }
@@ -5656,9 +5653,13 @@ function constructors.scale(tfmdata,specification)
     elseif autoitalicamount then 
       local vi=description.italic
       if not vi then
-        local vi=description.boundingbox[3]-description.width+autoitalicamount
-        if vi>0 then 
-          chr.italic=vi*hdelta
+        local bb=description.boundingbox
+        if bb then
+          local vi=bb[3]-description.width+autoitalicamount
+          if vi>0 then 
+            chr.italic=vi*hdelta
+          end
+        else
         end
       elseif vi~=0 then
         chr.italic=vi*hdelta
@@ -6408,6 +6409,7 @@ local formatters=string.formatters
 local trace_loading=false trackers.register("fonts.loading",function(v) trace_loading=v end)
 local trace_mapping=false trackers.register("fonts.mapping",function(v) trace_unimapping=v end)
 local report_fonts=logs.reporter("fonts","loading") 
+local force_ligatures=false directives.register("fonts.mapping.forceligatures",function(v) force_ligatures=v end)
 local fonts=fonts or {}
 local mappings=fonts.mappings or {}
 fonts.mappings=mappings
@@ -6676,7 +6678,7 @@ function mappings.addtounicode(data,filename,checklookups)
   local collected=false
   local unicoded=0
   for unicode,glyph in next,descriptions do
-    if not glyph.unicode and glyph.class=="ligature" then
+    if glyph.class=="ligature" and (force_ligatures or not glyph.unicode) then
       if not collected then
         collected=fonts.handlers.otf.readers.getcomponents(data)
         if not collected then
@@ -6927,7 +6929,7 @@ local function read_from_tfm(specification)
       end
     end
     properties.haskerns=true
-    properties.haslogatures=true
+    properties.hasligatures=true
     resources.unicodes={}
     resources.lookuptags={}
     depth[filename]=depth[filename]-1
@@ -6980,9 +6982,11 @@ if not modules then modules={} end modules ['font-afm']={
 }
 local fonts,logs,trackers,containers,resolvers=fonts,logs,trackers,containers,resolvers
 local next,type,tonumber=next,type,tonumber
-local format,match,gmatch,lower,gsub,strip=string.format,string.match,string.gmatch,string.lower,string.gsub,string.strip
+local match,gmatch,lower,gsub,strip,find=string.match,string.gmatch,string.lower,string.gsub,string.strip,string.find
+local char,byte,sub=string.char,string.byte,string.sub
 local abs=math.abs
-local P,S,C,R,lpegmatch,patterns=lpeg.P,lpeg.S,lpeg.C,lpeg.R,lpeg.match,lpeg.patterns
+local bxor,rshift=bit32.bxor,bit32.rshift
+local P,S,R,Cmt,C,Ct,Cs,lpegmatch,patterns=lpeg.P,lpeg.S,lpeg.R,lpeg.Cmt,lpeg.C,lpeg.Ct,lpeg.Cs,lpeg.match,lpeg.patterns
 local derivetable=table.derive
 local trace_features=false trackers.register("afm.features",function(v) trace_features=v end)
 local trace_indexing=false trackers.register("afm.indexing",function(v) trace_indexing=v end)
@@ -6994,15 +6998,11 @@ local findbinfile=resolvers.findbinfile
 local definers=fonts.definers
 local readers=fonts.readers
 local constructors=fonts.constructors
-local fontloader=fontloader
-local font_to_table=fontloader.to_table
-local open_font=fontloader.open
-local close_font=fontloader.close
 local afm=constructors.newhandler("afm")
 local pfb=constructors.newhandler("pfb")
 local afmfeatures=constructors.newfeatures("afm")
 local registerafmfeature=afmfeatures.register
-afm.version=1.500 
+afm.version=1.501 
 afm.cache=containers.define("fonts","afm",afm.version,true)
 afm.autoprefixed=true 
 afm.helpdata={} 
@@ -7111,39 +7111,140 @@ local function get_variables(data,fontmetrics)
     end
   end
 end
-local function get_indexes(data,pfbname)
-  data.resources.filename=resolvers.unresolve(pfbname) 
-  local pfbblob=open_font(pfbname)
-  if pfbblob then
-    local characters=data.characters
-    local pfbdata=font_to_table(pfbblob)
-    if pfbdata then
-      local glyphs=pfbdata.glyphs
-      if glyphs then
-        if trace_loading then
-          report_afm("getting index data from %a",pfbname)
-        end
-        for index,glyph in next,glyphs do
-          local name=glyph.name
-          if name then
-            local char=characters[name]
-            if char then
-              if trace_indexing then
-                report_afm("glyph %a has index %a",name,index)
+local get_indexes
+do
+  local fontloader=fontloader
+  if fontloader then
+    local font_to_table=fontloader.to_table
+    local open_font=fontloader.open
+    local close_font=fontloader.close
+    local function get_indexes_old(data,pfbname)
+      local pfbblob=open_font(pfbname)
+      if pfbblob then
+        local characters=data.characters
+        local pfbdata=font_to_table(pfbblob)
+        if pfbdata then
+          local glyphs=pfbdata.glyphs
+          if glyphs then
+            if trace_loading then
+              report_afm("getting index data from %a",pfbname)
+            end
+            for index,glyph in next,glyphs do
+              local name=glyph.name
+              if name then
+                local char=characters[name]
+                if char then
+                  if trace_indexing then
+                    report_afm("glyph %a has index %a",name,index)
+                  end
+                  char.index=index
+                end
               end
-              char.index=index
             end
+          elseif trace_loading then
+            report_afm("no glyph data in pfb file %a",pfbname)
           end
+        elseif trace_loading then
+          report_afm("no data in pfb file %a",pfbname)
         end
+        close_font(pfbblob)
       elseif trace_loading then
-        report_afm("no glyph data in pfb file %a",pfbname)
+        report_afm("invalid pfb file %a",pfbname)
+      end
+    end
+  end
+  local n,m
+  local progress=function(str,position,name,size)
+    local forward=position+tonumber(size)+3+2
+    n=n+1
+    if n>=m then
+      return #str,name
+    elseif forward<#str then
+      return forward,name
+    else
+      return #str,name
+    end
+  end
+  local initialize=function(str,position,size)
+    n=0
+    m=tonumber(size)
+    return position+1
+  end
+  local charstrings=P("/CharStrings")
+  local name=P("/")*C((R("az")+R("AZ")+R("09")+S("-_."))^1)
+  local size=C(R("09")^1)
+  local spaces=P(" ")^1
+  local p_filternames=Ct (
+    (1-charstrings)^0*charstrings*spaces*Cmt(size,initialize)*(Cmt(name*P(" ")^1*C(R("09")^1),progress)+P(1))^1
+  )
+  local decrypt
+  do
+    local r,c1,c2,n=0,0,0,0
+    local function step(c)
+      local cipher=byte(c)
+      local plain=bxor(cipher,rshift(r,8))
+      r=((cipher+r)*c1+c2)%65536
+      return char(plain)
+    end
+    decrypt=function(binary)
+      r,c1,c2,n=55665,52845,22719,4
+      binary=gsub(binary,".",step)
+      return sub(binary,n+1)
+    end
+  end
+  local function loadpfbvector(filename)
+    local data=io.loaddata(resolvers.findfile(filename))
+    if not find(data,"!PS%-AdobeFont%-") then
+      print("no font",filename)
+      return
+    end
+    if not data then
+      print("no data",filename)
+      return
+    end
+    local ascii,binary=match(data,"(.*)eexec%s+......(.*)")
+    if not binary then
+      print("no binary",filename)
+      return
+    end
+    binary=decrypt(binary,4)
+    local vector=lpegmatch(p_filternames,binary)
+    vector[0]=table.remove(vector,1)
+    if not vector then
+      print("no vector",filename)
+      return
+    end
+    return vector
+  end
+  get_indexes=function(data,pfbname)
+    local vector=loadpfbvector(pfbname)
+    if vector then
+      local characters=data.characters
+      if trace_loading then
+        report_afm("getting index data from %a",pfbname)
+      end
+      for index=1,#vector do
+        local name=vector[index]
+        local char=characters[name]
+        if char then
+          if trace_indexing then
+            report_afm("glyph %a has index %a",name,index)
+          end
+          char.index=index
+        end
+      end
+    end
+  end
+  if fontloader then
+    afm.use_new_indexer=true
+    get_indexes_new=get_indexes
+    get_indexes=function(data,pfbname)
+      if afm.use_new_indexer then
+        return get_indexes_new(data,pfbname)
+      else
+        return get_indexes_old(data,pfbname)
       end
-    elseif trace_loading then
-      report_afm("no data in pfb file %a",pfbname)
     end
-    close_font(pfbblob)
-  elseif trace_loading then
-    report_afm("invalid pfb file %a",pfbname)
   end
 end
 local function readafm(filename)
@@ -7222,6 +7323,7 @@ function afm.load(filename)
       data=readafm(filename)
       if data then
         if pfbname~="" then
+          data.resources.filename=resolvers.unresolve(pfbname)
           get_indexes(data,pfbname)
         elseif trace_loading then
           report_afm("no pfb file for %a",filename)
@@ -7265,7 +7367,8 @@ end
 local uparser=fonts.mappings.makenameparser()
 unify=function(data,filename)
   local unicodevector=fonts.encodings.agl.unicodes 
-  local unicodes,names={},{}
+  local unicodes={}
+  local names={}
   local private=constructors.privateoffset
   local descriptions=data.descriptions
   for name,blob in next,data.characters do
@@ -8142,15 +8245,11 @@ if not modules then modules={} end modules ['font-otr']={
   copyright="PRAGMA ADE / ConTeXt Development Team",
   license="see context related readme files"
 }
-if not characters then
-  require("char-def")
-  require("char-ini")
-end
 local next,type,unpack=next,type,unpack
 local byte,lower,char,strip,gsub=string.byte,string.lower,string.char,string.strip,string.gsub
 local bittest=bit32.btest
-local concat,remove,unpack=table.concat,table.remov,table.unpack
-local floor,mod,abs,sqrt,round=math.floor,math.mod,math.abs,math.sqrt,math.round
+local concat,remove,unpack,fastcopy=table.concat,table.remov,table.unpack,table.fastcopy
+local floor,abs,sqrt,round=math.floor,math.abs,math.sqrt,math.round
 local P,R,S,C,Cs,Cc,Ct,Carg,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.Carg,lpeg.Cmt
 local lpegmatch=lpeg.match
 local setmetatableindex=table.setmetatableindex
@@ -8701,18 +8800,17 @@ readers.hmtx=function(f,fontdata,specification)
       local nofmetrics=fontdata.horizontalheader.nofhmetrics
       local glyphs=fontdata.glyphs
       local nofglyphs=fontdata.nofglyphs
-      local nofrepeated=nofglyphs-nofmetrics
       local width=0 
       local leftsidebearing=0
       for i=0,nofmetrics-1 do
         local glyph=glyphs[i]
         width=readshort(f)
         leftsidebearing=readshort(f)
-        if advance~=0 then
+        if width~=0 then
           glyph.width=width
         end
       end
-      for i=nofmetrics,nofrepeated do
+      for i=nofmetrics,nofglyphs-1 do
         local glyph=glyphs[i]
         if width~=0 then
           glyph.width=width
@@ -8795,6 +8893,7 @@ local sequence={
   { 0,0,6 },
   { 3,0,6 },
   { 0,5,14 },
+  { 3,10,13 },
 }
 local supported={}
 for i=1,#sequence do
@@ -8853,10 +8952,10 @@ formatreaders[4]=function(f,fontdata,offset)
     elseif offset==0xFFFF then
     elseif offset==0 then
       if trace_cmap then
-        report("format 4.%i segment %2i from %C upto %C at index %H",1,segment,startchar,endchar,mod(startchar+delta,65536))
+        report("format 4.%i segment %2i from %C upto %C at index %H",1,segment,startchar,endchar,(startchar+delta)%65536)
       end
       for unicode=startchar,endchar do
-        local index=mod(unicode+delta,65536)
+        local index=(unicode+delta)%65536
         if index and index>0 then
           local glyph=glyphs[index]
           if glyph then
@@ -8885,13 +8984,13 @@ formatreaders[4]=function(f,fontdata,offset)
     else
       local shift=(segment-nofsegments+offset/2)-startchar
       if trace_cmap then
-        report("format 4.%i segment %2i from %C upto %C at index %H",0,segment,startchar,endchar,mod(startchar+delta,65536))
+        report("format 4.%i segment %2i from %C upto %C at index %H",0,segment,startchar,endchar,(startchar+delta)%65536)
       end
       for unicode=startchar,endchar do
         local slot=shift+unicode
         local index=indices[slot]
         if index and index>0 then
-          index=mod(index+delta,65536)
+          index=(index+delta)%65536
           local glyph=glyphs[index]
           if glyph then
             local gu=glyph.unicode
@@ -8966,7 +9065,7 @@ formatreaders[12]=function(f,fontdata,offset)
     local last=readulong(f)
     local index=readulong(f)
     if trace_cmap then
-      report("format 12 from %C to %C",first,last)
+      report("format 12 from %C to %C starts at index %i",first,last,index)
     end
     for unicode=first,last do
       local glyph=glyphs[index]
@@ -8992,6 +9091,49 @@ formatreaders[12]=function(f,fontdata,offset)
   end
   return nofdone
 end
+formatreaders[13]=function(f,fontdata,offset)
+  setposition(f,offset+2+2+4+4) 
+  local mapping=fontdata.mapping
+  local glyphs=fontdata.glyphs
+  local duplicates=fontdata.duplicates
+  local nofgroups=readulong(f)
+  local nofdone=0
+  for i=1,nofgroups do
+    local first=readulong(f)
+    local last=readulong(f)
+    local index=readulong(f)
+    if first<privateoffset then
+      if trace_cmap then
+        report("format 13 from %C to %C get index %i",first,last,index)
+      end
+      local glyph=glyphs[index]
+      local unicode=glyph.unicode
+      if not unicode then
+        unicode=first
+        glyph.unicode=unicode
+        first=first+1
+      end
+      local list=duplicates[unicode]
+      mapping[index]=unicode
+      if not list then
+        list={}
+        duplicates[unicode]=list
+      end
+      if last>=privateoffset then
+        local limit=privateoffset-1
+        report("format 13 from %C to %C pruned to %C",first,last,limit)
+        last=limit
+      end
+      for unicode=first,last do
+        list[unicode]=true
+      end
+      nofdone=nofdone+last-first+1
+    else
+      report("format 13 from %C to %C ignored",first,last)
+    end
+  end
+  return nofdone
+end
 formatreaders[14]=function(f,fontdata,offset)
   if offset and offset~=0 then
     setposition(f,offset)
@@ -13831,16 +13973,28 @@ local function copyduplicates(fontdata)
         local du=descriptions[u]
         if du then
           local t={ f_character_y(u),"@",f_index(du.index),"->" }
+          local n=0
+          local m=25
           for u in next,d do
             if descriptions[u] then
-              t[#t+1]=f_character_n(u)
+              if n<m then
+                t[n+4]=f_character_n(u)
+              end
             else
               local c=copy(du)
+              c.unicode=u 
               descriptions[u]=c
-              t[#t+1]=f_character_y(u)
+              if n<m then
+                t[n+4]=f_character_y(u)
+              end
             end
+            n=n+1
+          end
+          if n<=m then
+            report("duplicates: %i : % t",n,t)
+          else
+            report("duplicates: %i : % t ...",n,t)
           end
-          report("duplicates: % t",t)
         else
         end
       end
@@ -14025,13 +14179,12 @@ local function checklookups(fontdata,missing,nofmissing)
       if r then
         local name=descriptions[i].name or f_index(i)
         if not ignore[name] then
-          done[#done+1]=name
+          done[name]=true
         end
       end
     end
-    if #done>0 then
-      table.sort(done)
-      report("not unicoded: % t",done)
+    if next(done) then
+      report("not unicoded: % t",table.sortedkeys(done))
     end
   end
 end
@@ -15486,7 +15639,7 @@ local trace_defining=false registertracker("fonts.defining",function(v) trace_de
 local report_otf=logs.reporter("fonts","otf loading")
 local fonts=fonts
 local otf=fonts.handlers.otf
-otf.version=3.017 
+otf.version=3.018 
 otf.cache=containers.define("fonts","otl",otf.version,true)
 local otfreaders=otf.readers
 local hashes=fonts.hashes
@@ -16527,8 +16680,6 @@ local attributes,nodes,node=attributes,nodes,node
 fonts=fonts
 local hashes=fonts.hashes
 local fontdata=hashes.identifiers
-local parameters=fonts.hashes.parameters
-local resources=fonts.hashes.resources
 nodes.injections=nodes.injections or {}
 local injections=nodes.injections
 local tracers=nodes.tracers
@@ -17718,7 +17869,7 @@ local function injectspaces(head)
   local function updatefont(font,trig)
     leftkerns=trig.left
     rightkerns=trig.right
-    local par=parameters[font]
+    local par=fontdata[font].parameters 
     factor=par.factor
     threshold=par.spacing.width-1 
     lastfont=font
@@ -18566,7 +18717,21 @@ local function toligature(head,start,stop,char,dataset,sequence,markflag,discfou
       local pre,post,replace,pretail,posttail,replacetail=getdisc(discfound,true)
       if not replace then 
         local prev=getprev(base)
-        local copied=copy_node_list(comp)
+local current=comp
+local previous=nil
+local copied=nil
+while current do
+  if getid(current)==glyph_code then
+    local n=copy_node(current)
+    if copied then
+      setlink(previous,n)
+    else
+      copied=n
+    end
+    previous=n
+  end
+  current=getnext(current)
+end
         setprev(discnext,nil) 
         setnext(discprev,nil) 
         if pre then
@@ -19871,7 +20036,7 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
         else
           local discfound=nil
           local n=f+1
-          last=getnext(last)
+          last=getnext(last) 
           while n<=l do
             if not last and (sweeptype=="post" or sweeptype=="replace") then
               last=getnext(sweepnode)
@@ -19912,7 +20077,6 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
                   end
                   break
                 end
-                last=getnext(last)
               elseif char==false then
                 if discfound then
                   notmatchreplace[discfound]=true
@@ -19997,8 +20161,12 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
                       if trace_skips then
                         show_skip(dataset,sequence,char,ck,class)
                       end
+                      prev=getprev(prev) 
                     elseif seq[n][char] then
-                      n=n -1
+                      if n>1 then 
+                        prev=getprev(prev) 
+                      end
+                      n=n-1
                     else
                       if discfound then
                         notmatchreplace[discfound]=true
@@ -20017,7 +20185,6 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
                     end
                     break
                   end
-                  prev=getprev(prev)
                 elseif char==false then
                   if discfound then
                     notmatchreplace[discfound]=true
@@ -20079,19 +20246,19 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
                       if not match then
                         break
                       end
-                    else
                     end
-                  else
                   end
+                  prev=getprev(prev)
                 elseif seq[n][32] then
                   n=n-1
+                  prev=getprev(prev)
                 else
                   match=false
                   break
                 end
-                prev=getprev(prev)
               elseif seq[n][32] then 
                 n=n-1
+                prev=getprev(prev) 
               else
                 match=false
                 break
@@ -20126,7 +20293,11 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
                     if trace_skips then
                       show_skip(dataset,sequence,char,ck,class)
                     end
+                    current=getnext(current) 
                   elseif seq[n][char] then
+                    if n<s then 
+                      current=getnext(current) 
+                    end
                     n=n+1
                   else
                     if discfound then
@@ -20200,16 +20371,16 @@ local function handle_contextchain(head,start,dataset,sequence,contexts,rlmode)
                   end
                 else
                 end
+                current=getnext(current)
               elseif seq[n][32] then 
                 n=n+1
               else
                 match=false
                 break
               end
-              current=getnext(current)
             elseif seq[n][32] then
               n=n+1
-current=getnext(current)
+              current=getnext(current)
             else
               match=false
               break