Merge pull request #5 from phi-gamma/master

changes to font request and database code

1 files changed, 836 insertions, 20 deletions

diff --git a/luaotfload-merged.lua b/luaotfload-merged.lua
index 314305a..da5e35a 100644
--- a/luaotfload-merged.lua
+++ b/luaotfload-merged.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 04/20/13 13:33:53
+-- merge date  : 04/24/13 13:39:43
 
 do -- begin closure to overcome local limits and interference
 
@@ -2978,7 +2978,9 @@ function resolvers.unresolve(s)
   return s
 end
 caches={}
-local writable,readables=nil,{}
+local writable=nil
+local readables={}
+local usingjit=jit
 if not caches.namespace or caches.namespace=="" or caches.namespace=="context" then
   caches.namespace='generic'
 end
@@ -3038,7 +3040,7 @@ end
 local function makefullname(path,name)
   if path and path~="" then
     name="temp-"..name 
-    return file.addsuffix(file.join(path,name),"lua"),file.addsuffix(file.join(path,name),"luc")
+    return file.addsuffix(file.join(path,name),"lua"),file.addsuffix(file.join(path,name),usingjit and "lub" or "luc")
   end
 end
 function caches.is_writable(path,name)
@@ -3085,26 +3087,19 @@ function caches.savedata(path,name,data)
     end
   end
 end
-caches.compilemethod="both"
 function caches.compile(data,luaname,lucname)
-  local done=false
-  if caches.compilemethod=="luac" or caches.compilemethod=="both" then
-    done=os.spawn("texluac -o "..string.quoted(lucname).." -s "..string.quoted(luaname))==0
+  local d=io.loaddata(luaname)
+  if not d or d=="" then
+    d=table.serialize(data,true) 
   end
-  if not done and (caches.compilemethod=="dump" or caches.compilemethod=="both") then
-    local d=io.loaddata(luaname)
-    if not d or d=="" then
-      d=table.serialize(data,true) 
-    end
-    if d and d~="" then
-      local f=io.open(lucname,'w')
-      if f then
-        local s=loadstring(d)
-        if s then
-          f:write(string.dump(s,true))
-        end
-        f:close()
+  if d and d~="" then
+    local f=io.open(lucname,'w')
+    if f then
+      local s=loadstring(d)
+      if s then
+        f:write(string.dump(s,true))
       end
+      f:close()
     end
   end
 end
@@ -6103,6 +6098,14 @@ actions["reorganize lookups"]=function(data,filename,raw)
               local current=coverage.current
               if current then
                 current=t_uncover(splitter,t_u_cache,current)
+                local lookups=rule.lookups
+                if lookups then
+                  for i=1,#current do
+                    if not lookups[i] then
+                      lookups[i]="" 
+                    end
+                  end
+                end
                 rule.current=t_hashed(current,t_h_cache)
               end
               local after=coverage.after
@@ -10313,6 +10316,819 @@ end -- closure
 
 do -- begin closure to overcome local limits and interference
 
+if not modules then modules={} end modules ['font-otp']={
+  version=1.001,
+  comment="companion to font-otf.lua (packing)",
+  author="Hans Hagen, PRAGMA-ADE, Hasselt NL",
+  copyright="PRAGMA ADE / ConTeXt Development Team",
+  license="see context related readme files"
+}
+local next,type=next,type
+local sort,concat=table.sort,table.concat
+local sortedhash=table.sortedhash
+local trace_packing=false trackers.register("otf.packing",function(v) trace_packing=v end)
+local trace_loading=false trackers.register("otf.loading",function(v) trace_loading=v end)
+local report_otf=logs.reporter("fonts","otf loading")
+fonts=fonts or {}
+local handlers=fonts.handlers or {}
+fonts.handlers=handlers
+local otf=handlers.otf or {}
+handlers.otf=otf
+local enhancers=otf.enhancers or {}
+otf.enhancers=enhancers
+local glists=otf.glists or { "gsub","gpos" }
+otf.glists=glists
+local criterium=1
+local threshold=0
+local function tabstr_normal(t)
+  local s={}
+  local n=0
+  for k,v in next,t do
+    n=n+1
+    if type(v)=="table" then
+      s[n]=k..">"..tabstr_normal(v)
+    elseif v==true then
+      s[n]=k.."+" 
+    elseif v then
+      s[n]=k.."="..v
+    else
+      s[n]=k.."-" 
+    end
+  end
+  if n==0 then
+    return ""
+  elseif n==1 then
+    return s[1]
+  else
+    sort(s) 
+    return concat(s,",")
+  end
+end
+local function tabstr_flat(t)
+  local s={}
+  local n=0
+  for k,v in next,t do
+    n=n+1
+    s[n]=k.."="..v
+  end
+  if n==0 then
+    return ""
+  elseif n==1 then
+    return s[1]
+  else
+    sort(s) 
+    return concat(s,",")
+  end
+end
+local function tabstr_mixed(t) 
+  local s={}
+  local n=#t
+  if n==0 then
+    return ""
+  elseif n==1 then
+    local k=t[1]
+    if k==true then
+      return "++" 
+    elseif k==false then
+      return "--" 
+    else
+      return tostring(k) 
+    end
+  else
+    for i=1,n do
+      local k=t[i]
+      if k==true then
+        s[i]="++" 
+      elseif k==false then
+        s[i]="--" 
+      else
+        s[i]=k 
+      end
+    end
+    return concat(s,",")
+  end
+end
+local function tabstr_boolean(t)
+  local s={}
+  local n=0
+  for k,v in next,t do
+    n=n+1
+    if v then
+      s[n]=k.."+"
+    else
+      s[n]=k.."-"
+    end
+  end
+  if n==0 then
+    return ""
+  elseif n==1 then
+    return s[1]
+  else
+    sort(s) 
+    return concat(s,",")
+  end
+end
+local function packdata(data)
+  if data then
+    local h,t,c={},{},{}
+    local hh,tt,cc={},{},{}
+    local nt,ntt=0,0
+    local function pack_normal(v)
+      local tag=tabstr_normal(v)
+      local ht=h[tag]
+      if ht then
+        c[ht]=c[ht]+1
+        return ht
+      else
+        nt=nt+1
+        t[nt]=v
+        h[tag]=nt
+        c[nt]=1
+        return nt
+      end
+    end
+    local function pack_flat(v)
+      local tag=tabstr_flat(v)
+      local ht=h[tag]
+      if ht then
+        c[ht]=c[ht]+1
+        return ht
+      else
+        nt=nt+1
+        t[nt]=v
+        h[tag]=nt
+        c[nt]=1
+        return nt
+      end
+    end
+    local function pack_boolean(v)
+      local tag=tabstr_boolean(v)
+      local ht=h[tag]
+      if ht then
+        c[ht]=c[ht]+1
+        return ht
+      else
+        nt=nt+1
+        t[nt]=v
+        h[tag]=nt
+        c[nt]=1
+        return nt
+      end
+    end
+    local function pack_indexed(v)
+      local tag=concat(v," ")
+      local ht=h[tag]
+      if ht then
+        c[ht]=c[ht]+1
+        return ht
+      else
+        nt=nt+1
+        t[nt]=v
+        h[tag]=nt
+        c[nt]=1
+        return nt
+      end
+    end
+    local function pack_mixed(v)
+      local tag=tabstr_mixed(v)
+      local ht=h[tag]
+      if ht then
+        c[ht]=c[ht]+1
+        return ht
+      else
+        nt=nt+1
+        t[nt]=v
+        h[tag]=nt
+        c[nt]=1
+        return nt
+      end
+    end
+    local function pack_final(v)
+      if c[v]<=criterium then
+        return t[v]
+      else
+        local hv=hh[v]
+        if hv then
+          return hv
+        else
+          ntt=ntt+1
+          tt[ntt]=t[v]
+          hh[v]=ntt
+          cc[ntt]=c[v]
+          return ntt
+        end
+      end
+    end
+    local function success(stage,pass)
+      if nt==0 then
+        if trace_loading or trace_packing then
+          report_otf("pack quality: nothing to pack")
+        end
+        return false
+      elseif nt>=threshold then
+        local one,two,rest=0,0,0
+        if pass==1 then
+          for k,v in next,c do
+            if v==1 then
+              one=one+1
+            elseif v==2 then
+              two=two+1
+            else
+              rest=rest+1
+            end
+          end
+        else
+          for k,v in next,cc do
+            if v>20 then
+              rest=rest+1
+            elseif v>10 then
+              two=two+1
+            else
+              one=one+1
+            end
+          end
+          data.tables=tt
+        end
+        if trace_loading or trace_packing then
+          report_otf("pack quality: stage %s, pass %s, %s packed, 1-10:%s, 11-20:%s, rest:%s (criterium: %s)",stage,pass,one+two+rest,one,two,rest,criterium)
+        end
+        return true
+      else
+        if trace_loading or trace_packing then
+          report_otf("pack quality: stage %s, pass %s, %s packed, aborting pack (threshold: %s)",stage,pass,nt,threshold)
+        end
+        return false
+      end
+    end
+    local function packers(pass)
+      if pass==1 then
+        return pack_normal,pack_indexed,pack_flat,pack_boolean,pack_mixed
+      else
+        return pack_final,pack_final,pack_final,pack_final,pack_final
+      end
+    end
+    local resources=data.resources
+    local lookuptypes=resources.lookuptypes
+    for pass=1,2 do
+      if trace_packing then
+        report_otf("start packing: stage 1, pass %s",pass)
+      end
+      local pack_normal,pack_indexed,pack_flat,pack_boolean,pack_mixed=packers(pass)
+      for unicode,description in next,data.descriptions do
+        local boundingbox=description.boundingbox
+        if boundingbox then
+          description.boundingbox=pack_indexed(boundingbox)
+        end
+        local slookups=description.slookups
+        if slookups then
+          for tag,slookup in next,slookups do
+            local what=lookuptypes[tag]
+            if what=="pair" then
+              local t=slookup[2] if t then slookup[2]=pack_indexed(t) end
+              local t=slookup[3] if t then slookup[3]=pack_indexed(t) end
+            elseif what~="substitution" then
+              slookups[tag]=pack_indexed(slookup) 
+            end
+          end
+        end
+        local mlookups=description.mlookups
+        if mlookups then
+          for tag,mlookup in next,mlookups do
+            local what=lookuptypes[tag]
+            if what=="pair" then
+              for i=1,#mlookup do
+                local lookup=mlookup[i]
+                local t=lookup[2] if t then lookup[2]=pack_indexed(t) end
+                local t=lookup[3] if t then lookup[3]=pack_indexed(t) end
+              end
+            elseif what~="substitution" then
+              for i=1,#mlookup do
+                mlookup[i]=pack_indexed(mlookup[i]) 
+              end
+            end
+          end
+        end
+        local kerns=description.kerns
+        if kerns then
+          for tag,kern in next,kerns do
+            kerns[tag]=pack_flat(kern)
+          end
+        end
+        local math=description.math
+        if math then
+          local kerns=math.kerns
+          if kerns then
+            for tag,kern in next,kerns do
+              kerns[tag]=pack_normal(kern)
+            end
+          end
+        end
+        local anchors=description.anchors
+        if anchors then
+          for what,anchor in next,anchors do
+            if what=="baselig" then
+              for _,a in next,anchor do
+                for k=1,#a do
+                  a[k]=pack_indexed(a[k])
+                end
+              end
+            else
+              for k,v in next,anchor do
+                anchor[k]=pack_indexed(v)
+              end
+            end
+          end
+        end
+      end
+      local lookups=data.lookups
+      if lookups then
+        for _,lookup in next,lookups do
+          local rules=lookup.rules
+          if rules then
+            for i=1,#rules do
+              local rule=rules[i]
+              local r=rule.before    if r then for i=1,#r do r[i]=pack_boolean(r[i]) end end
+              local r=rule.after    if r then for i=1,#r do r[i]=pack_boolean(r[i]) end end
+              local r=rule.current   if r then for i=1,#r do r[i]=pack_boolean(r[i]) end end
+              local r=rule.replacements if r then rule.replacements=pack_flat  (r)  end 
+              local r=rule.lookups   if r then rule.lookups=pack_indexed(r)  end
+            end
+          end
+        end
+      end
+      local anchor_to_lookup=resources.anchor_to_lookup
+      if anchor_to_lookup then
+        for anchor,lookup in next,anchor_to_lookup do
+          anchor_to_lookup[anchor]=pack_normal(lookup)
+        end
+      end
+      local lookup_to_anchor=resources.lookup_to_anchor
+      if lookup_to_anchor then
+        for lookup,anchor in next,lookup_to_anchor do
+          lookup_to_anchor[lookup]=pack_normal(anchor)
+        end
+      end
+      local sequences=resources.sequences
+      if sequences then
+        for feature,sequence in next,sequences do
+          local flags=sequence.flags
+          if flags then
+            sequence.flags=pack_normal(flags)
+          end
+          local subtables=sequence.subtables
+          if subtables then
+            sequence.subtables=pack_normal(subtables)
+          end
+          local features=sequence.features
+          if features then
+            for script,feature in next,features do
+              features[script]=pack_normal(feature)
+            end
+          end
+        end
+      end
+      local lookups=resources.lookups
+      if lookups then
+        for name,lookup in next,lookups do
+          local flags=lookup.flags
+          if flags then
+            lookup.flags=pack_normal(flags)
+          end
+          local subtables=lookup.subtables
+          if subtables then
+            lookup.subtables=pack_normal(subtables)
+          end
+        end
+      end
+      local features=resources.features
+      if features then
+        for _,what in next,glists do
+          local list=features[what]
+          if list then
+            for feature,spec in next,list do
+              list[feature]=pack_normal(spec)
+            end
+          end
+        end
+      end
+      if not success(1,pass) then
+        return
+      end
+    end
+    if nt>0 then
+      for pass=1,2 do
+        if trace_packing then
+          report_otf("start packing: stage 2, pass %s",pass)
+        end
+        local pack_normal,pack_indexed,pack_flat,pack_boolean,pack_mixed=packers(pass)
+        for unicode,description in next,data.descriptions do
+          local kerns=description.kerns
+          if kerns then
+            description.kerns=pack_normal(kerns)
+          end
+          local math=description.math
+          if math then
+            local kerns=math.kerns
+            if kerns then
+              math.kerns=pack_normal(kerns)
+            end
+          end
+          local anchors=description.anchors
+          if anchors then
+            description.anchors=pack_normal(anchors)
+          end
+          local mlookups=description.mlookups
+          if mlookups then
+            for tag,mlookup in next,mlookups do
+              mlookups[tag]=pack_normal(mlookup)
+            end
+          end
+        end
+        local lookups=data.lookups
+        if lookups then
+          for _,lookup in next,lookups do
+            local rules=lookup.rules
+            if rules then
+              for i=1,#rules do 
+                local rule=rules[i]
+                local r=rule.before if r then rule.before=pack_normal(r) end
+                local r=rule.after  if r then rule.after=pack_normal(r) end
+                local r=rule.current if r then rule.current=pack_normal(r) end
+              end
+            end
+          end
+        end
+        local sequences=resources.sequences
+        if sequences then
+          for feature,sequence in next,sequences do
+            sequence.features=pack_normal(sequence.features)
+          end
+        end
+        if not success(2,pass) then
+        end
+      end
+      for pass=1,2 do
+        local pack_normal,pack_indexed,pack_flat,pack_boolean,pack_mixed=packers(pass)
+        for unicode,description in next,data.descriptions do
+          local slookups=description.slookups
+          if slookups then
+            description.slookups=pack_normal(slookups)
+          end
+          local mlookups=description.mlookups
+          if mlookups then
+            description.mlookups=pack_normal(mlookups)
+          end
+        end
+      end
+    end
+  end
+end
+local unpacked_mt={
+  __index=function(t,k)
+      t[k]=false
+      return k 
+    end
+}
+local function unpackdata(data)
+  if data then
+    local tables=data.tables
+    if tables then
+      local resources=data.resources
+      local lookuptypes=resources.lookuptypes
+      local unpacked={}
+      setmetatable(unpacked,unpacked_mt)
+      for unicode,description in next,data.descriptions do
+        local tv=tables[description.boundingbox]
+        if tv then
+          description.boundingbox=tv
+        end
+        local slookups=description.slookups
+        if slookups then
+          local tv=tables[slookups]
+          if tv then
+            description.slookups=tv
+            slookups=unpacked[tv]
+          end
+          if slookups then
+            for tag,lookup in next,slookups do
+              local what=lookuptypes[tag]
+              if what=="pair" then
+                local tv=tables[lookup[2]]
+                if tv then
+                  lookup[2]=tv
+                end
+                local tv=tables[lookup[3]]
+                if tv then
+                  lookup[3]=tv
+                end
+              elseif what~="substitution" then
+                local tv=tables[lookup]
+                if tv then
+                  slookups[tag]=tv
+                end
+              end
+            end
+          end
+        end
+        local mlookups=description.mlookups
+        if mlookups then
+          local tv=tables[mlookups]
+          if tv then
+            description.mlookups=tv
+            mlookups=unpacked[tv]
+          end
+          if mlookups then
+            for tag,list in next,mlookups do
+              local tv=tables[list]
+              if tv then
+                mlookups[tag]=tv
+                list=unpacked[tv]
+              end
+              if list then
+                local what=lookuptypes[tag]
+                if what=="pair" then
+                  for i=1,#list do
+                    local lookup=list[i]
+                    local tv=tables[lookup[2]]
+                    if tv then
+                      lookup[2]=tv
+                    end
+                    local tv=tables[lookup[3]]
+                    if tv then
+                      lookup[3]=tv
+                    end
+                  end
+                elseif what~="substitution" then
+                  for i=1,#list do
+                    local tv=tables[list[i]]
+                    if tv then
+                      list[i]=tv
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+        local kerns=description.kerns
+        if kerns then
+          local tm=tables[kerns]
+          if tm then
+            description.kerns=tm
+            kerns=unpacked[tm]
+          end
+          if kerns then
+            for k,kern in next,kerns do
+              local tv=tables[kern]
+              if tv then
+                kerns[k]=tv
+              end
+            end
+          end
+        end
+        local math=description.math
+        if math then
+          local kerns=math.kerns
+          if kerns then
+            local tm=tables[kerns]
+            if tm then
+              math.kerns=tm
+              kerns=unpacked[tm]
+            end
+            if kerns then
+              for k,kern in next,kerns do
+                local tv=tables[kern]
+                if tv then
+                  kerns[k]=tv
+                end
+              end
+            end
+          end
+        end
+        local anchors=description.anchors
+        if anchors then
+          local ta=tables[anchors]
+          if ta then
+            description.anchors=ta
+            anchors=unpacked[ta]
+          end
+          if anchors then
+            for tag,anchor in next,anchors do
+              if tag=="baselig" then
+                for _,list in next,anchor do
+                  for i=1,#list do
+                    local tv=tables[list[i]]
+                    if tv then
+                      list[i]=tv
+                    end
+                  end
+                end
+              else
+                for a,data in next,anchor do
+                  local tv=tables[data]
+                  if tv then
+                    anchor[a]=tv
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+      local lookups=data.lookups
+      if lookups then
+        for _,lookup in next,lookups do
+          local rules=lookup.rules
+          if rules then
+            for i=1,#rules do 
+              local rule=rules[i]
+              local before=rule.before
+              if before then
+                local tv=tables[before]
+                if tv then
+                  rule.before=tv
+                  before=unpacked[tv]
+                end
+                if before then
+                  for i=1,#before do
+                    local tv=tables[before[i]]
+                    if tv then
+                      before[i]=tv
+                    end
+                  end
+                end
+              end
+              local after=rule.after
+              if after then
+                local tv=tables[after]
+                if tv then
+                  rule.after=tv
+                  after=unpacked[tv]
+                end
+                if after then
+                  for i=1,#after do
+                    local tv=tables[after[i]]
+                    if tv then
+                      after[i]=tv
+                    end
+                  end
+                end
+              end
+              local current=rule.current
+              if current then
+                local tv=tables[current]
+                if tv then
+                  rule.current=tv
+                  current=unpacked[tv]
+                end
+                if current then
+                  for i=1,#current do
+                    local tv=tables[current[i]]
+                    if tv then
+                      current[i]=tv
+                    end
+                  end
+                end
+              end
+              local replacements=rule.replacements
+              if replacements then
+                local tv=tables[replacements]
+                if tv then
+                  rule.replacements=tv
+                end
+              end
+              local fore=rule.fore
+              if fore then
+                local tv=tables[fore]
+                if tv then
+                  rule.fore=tv
+                end
+              end
+              local back=rule.back
+              if back then
+                local tv=tables[back]
+                if tv then
+                  rule.back=tv
+                end
+              end
+              local names=rule.names
+              if names then
+                local tv=tables[names]
+                if tv then
+                  rule.names=tv
+                end
+              end
+              local lookups=rule.lookups
+              if lookups then
+                local tv=tables[lookups]
+                if tv then
+                  rule.lookups=tv
+                end
+              end
+            end
+          end
+        end
+      end
+      local anchor_to_lookup=resources.anchor_to_lookup
+      if anchor_to_lookup then
+        for anchor,lookup in next,anchor_to_lookup do
+          local tv=tables[lookup]
+          if tv then
+            anchor_to_lookup[anchor]=tv
+          end
+        end
+      end
+      local lookup_to_anchor=resources.lookup_to_anchor
+      if lookup_to_anchor then
+        for lookup,anchor in next,lookup_to_anchor do
+          local tv=tables[anchor]
+          if tv then
+            lookup_to_anchor[lookup]=tv
+          end
+        end
+      end
+      local ls=resources.sequences
+      if ls then
+        for _,feature in next,ls do
+          local flags=feature.flags
+          if flags then
+            local tv=tables[flags]
+            if tv then
+              feature.flags=tv
+            end
+          end
+          local subtables=feature.subtables
+          if subtables then
+            local tv=tables[subtables]
+            if tv then
+              feature.subtables=tv
+            end
+          end
+          local features=feature.features
+          if features then
+            local tv=tables[features]
+            if tv then
+              feature.features=tv
+              features=unpacked[tv]
+            end
+            if features then
+              for script,data in next,features do
+                local tv=tables[data]
+                if tv then
+                  features[script]=tv
+                end
+              end
+            end
+          end
+        end
+      end
+      local lookups=resources.lookups
+      if lookups then
+        for _,lookup in next,lookups do
+          local flags=lookup.flags
+          if flags then
+            local tv=tables[flags]
+            if tv then
+              lookup.flags=tv
+            end
+          end
+          local subtables=lookup.subtables
+          if subtables then
+            local tv=tables[subtables]
+            if tv then
+              lookup.subtables=tv
+            end
+          end
+        end
+      end
+      local features=resources.features
+      if features then
+        for _,what in next,glists do
+          local feature=features[what]
+          if feature then
+            for tag,spec in next,feature do
+              local tv=tables[spec]
+              if tv then
+                feature[tag]=tv
+              end
+            end
+          end
+        end
+      end
+      data.tables=nil
+    end
+  end
+end
+if otf.enhancers.register then
+  otf.enhancers.register("pack",packdata)
+  otf.enhancers.register("unpack",unpackdata)
+end
+otf.enhancers.unpack=unpackdata 
+
+end -- closure
+
+do -- begin closure to overcome local limits and interference
+
 if not modules then modules={} end modules ['luatex-fonts-lua']={
   version=1.001,
   comment="companion to luatex-*.tex",