[fontloader] sync with Context as of 2016-07-26

8 files changed, 181 insertions, 34 deletions

diff --git a/src/fontloader/misc/fontloader-basics-nod.lua b/src/fontloader/misc/fontloader-basics-nod.lua
index e7b5ab2..42a7a2e 100644
--- a/src/fontloader/misc/fontloader-basics-nod.lua
+++ b/src/fontloader/misc/fontloader-basics-nod.lua
@@ -71,7 +71,7 @@ nodes.nodecodes    = nodecodes
 nodes.glyphcodes   = glyphcodes
 nodes.disccodes    = disccodes
 
-local free_node    = node.free
+local flush_node   = node.flush_node
 local remove_node  = node.remove
 local new_node     = node.new
 local traverse_id  = node.traverse_id
@@ -95,7 +95,7 @@ function nodes.remove(head, current, free_too)
    head, current = remove_node(head,current)
    if t then
         if free_too then
-            free_node(t)
+            flush_node(t)
             t = nil
         else
             t.next, t.prev = nil, nil
@@ -128,12 +128,14 @@ nodes.setattr  = setfield
 
 nodes.tostring             = node.tostring or tostring
 nodes.copy                 = node.copy
+nodes.copy_node            = node.copy
 nodes.copy_list            = node.copy_list
 nodes.delete               = node.delete
 nodes.dimensions           = node.dimensions
 nodes.end_of_math          = node.end_of_math
 nodes.flush_list           = node.flush_list
 nodes.flush_node           = node.flush_node
+nodes.flush                = node.flush_node
 nodes.free                 = node.free
 nodes.insert_after         = node.insert_after
 nodes.insert_before        = node.insert_before
@@ -149,7 +151,6 @@ nodes.first_glyph          = node.first_glyph
 nodes.has_glyph            = node.has_glyph or node.first_glyph
 
 nodes.current_attr         = node.current_attr
-nodes.do_ligature_n        = node.do_ligature_n
 nodes.has_field            = node.has_field
 nodes.last_node            = node.last_node
 nodes.usedlist             = node.usedlist
@@ -253,9 +254,12 @@ nuts.insert_before       = direct.insert_before
 nuts.insert_after        = direct.insert_after
 nuts.delete              = direct.delete
 nuts.copy                = direct.copy
+nuts.copy_node           = direct.copy
 nuts.copy_list           = direct.copy_list
 nuts.tail                = direct.tail
 nuts.flush_list          = direct.flush_list
+nuts.flush_node          = direct.flush_node
+nuts.flush               = direct.flush
 nuts.free                = direct.free
 nuts.remove              = direct.remove
 nuts.is_node             = direct.is_node
diff --git a/src/fontloader/misc/fontloader-font-gbn.lua b/src/fontloader/misc/fontloader-font-gbn.lua
index 1ae817d..1f8df64 100644
--- a/src/fontloader/misc/fontloader-font-gbn.lua
+++ b/src/fontloader/misc/fontloader-font-gbn.lua
@@ -19,7 +19,7 @@ local nodes = nodes
 local nuts        = nodes.nuts -- context abstraction of direct nodes
 
 local traverse_id = nuts.traverse_id
-local free_node   = nuts.free
+local flush_node  = nuts.flush_node
 
 local glyph_code  = nodes.nodecodes.glyph
 local disc_code   = nodes.nodecodes.disc
@@ -159,7 +159,7 @@ function nodes.handlers.nodepass(head)
                         end
                     end
                 end
-                free_node(r)
+                flush_node(r)
             end
         end
         for d in traverse_id(disc_code,nuthead) do
diff --git a/src/fontloader/misc/fontloader-font-osd.lua b/src/fontloader/misc/fontloader-font-osd.lua
index a3dda67..26af691 100644
--- a/src/fontloader/misc/fontloader-font-osd.lua
+++ b/src/fontloader/misc/fontloader-font-osd.lua
@@ -107,9 +107,9 @@ local ischar             = nuts.is_char
 
 local insert_node_after  = nuts.insert_after
 local copy_node          = nuts.copy
-local free_node          = nuts.free
 local remove_node        = nuts.remove
 local flush_list         = nuts.flush_list
+local flush_node         = nuts.flush_node
 
 local copyinjection      = nodes.injections.copy -- KE: is this necessary? HH: probably not as positioning comes later and we rawget/set
 
@@ -792,7 +792,7 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
         if current == stop then
             stop = getprev(stop)
             head = remove_node(head,current)
-            free_node(current)
+            flush_node(current)
             return head, stop, nbspaces
         else
             nbspaces  = nbspaces + 1
@@ -830,7 +830,7 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
                                 setchar(current,getchar(tempcurrent)) -- we assumes that the result of blwf consists of one node
                                 local freenode = getnext(current)
                                 setlink(current,tmp)
-                                free_node(freenode)
+                                flush_node(freenode)
                                 flush_list(tempcurrent)
                                 if changestop then
                                     stop = current
@@ -1093,7 +1093,7 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
     if getchar(base) == c_nbsp then
         nbspaces = nbspaces - 1
         head = remove_node(head,base)
-        free_node(base)
+        flush_node(base)
     end
 
     return head, stop, nbspaces
@@ -1571,7 +1571,7 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces) -- maybe do a pa
         if current == stop then
             stop = getprev(stop)
             head = remove_node(head,current)
-            free_node(current)
+            flush_node(current)
             return head, stop, nbspaces
         else
             nbspaces = nbspaces + 1
@@ -1768,7 +1768,7 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces) -- maybe do a pa
     if getchar(base) == c_nbsp then
         nbspaces = nbspaces - 1
         head = remove_node(head, base)
-        free_node(base)
+        flush_node(base)
     end
 
     return head, stop, nbspaces
diff --git a/src/fontloader/misc/fontloader-font-otl.lua b/src/fontloader/misc/fontloader-font-otl.lua
index a35db5b..bdce80d 100644
--- a/src/fontloader/misc/fontloader-font-otl.lua
+++ b/src/fontloader/misc/fontloader-font-otl.lua
@@ -574,10 +574,48 @@ local function copytotfm(data,cache_id)
     end
 end
 
+-- These woff files are a kind of joke in a tex environment because one can simply convert
+-- them to ttf/otf and use them as such (after all, we cache them too). The successor format
+-- woff2 is more complex so there we can as well call an external converter which in the end
+-- makes this code kind of obsolete before it's even used. Although ... it might become a
+-- more general conversion plug in.
+
+local converters = {
+    woff = {
+        cachename = "webfonts",
+        action    = otf.readers.woff2otf,
+    }
+}
+
+local function checkconversion(specification)
+    local filename  = specification.filename
+    local converter = converters[lower(file.suffix(filename))]
+    if converter then
+        local base = file.basename(filename)
+        local name = file.removesuffix(base)
+        local attr = lfs.attributes(filename)
+        local size = attr and attr.size or 0
+        local time = attr and attr.modification or 0
+        if size > 0 then
+            local cleanname = containers.cleanname(name)
+            local cachename = caches.setfirstwritablefile(cleanname,converter.cachename)
+            if not io.exists(cachename) or (time ~= lfs.attributes(cachename).modification) then
+                report_otf("caching font %a in %a",filename,cachename)
+                converter.action(filename,cachename) -- todo infoonly
+                lfs.touch(cachename,time,time)
+            end
+            specification.filename = cachename
+        end
+    end
+end
+
 local function otftotfm(specification)
     local cache_id = specification.hash
     local tfmdata  = containers.read(constructors.cache,cache_id)
     if not tfmdata then
+
+        checkconversion(specification) -- for the moment here
+
         local name     = specification.name
         local sub      = specification.sub
         local subindex = specification.subindex
@@ -811,9 +849,14 @@ end
 
 readers.opentype = opentypereader -- kind of useless and obsolete
 
-function readers.otf  (specification) return opentypereader(specification,"otf") end
-function readers.ttf  (specification) return opentypereader(specification,"ttf") end
-function readers.ttc  (specification) return opentypereader(specification,"ttf") end
+function readers.otf(specification) return opentypereader(specification,"otf") end
+function readers.ttf(specification) return opentypereader(specification,"ttf") end
+function readers.ttc(specification) return opentypereader(specification,"ttf") end
+
+function readers.woff(specification)
+    checkconversion(specification)
+    opentypereader(specification,"")
+end
 
 -- this will be overloaded
 
diff --git a/src/fontloader/misc/fontloader-font-otr.lua b/src/fontloader/misc/fontloader-font-otr.lua
index 7d0bf04..7c81285 100644
--- a/src/fontloader/misc/fontloader-font-otr.lua
+++ b/src/fontloader/misc/fontloader-font-otr.lua
@@ -94,8 +94,10 @@ otf.readers             = readers
 
 ----- streamreader      = utilities.streams -- faster on big files
 local streamreader      = utilities.files   -- faster on identify
+local streamwriter      = utilities.files
 
 readers.streamreader    = streamreader
+readers.streamwriter    = streamwriter
 
 local openfile          = streamreader.open
 local closefile         = streamreader.close
diff --git a/src/fontloader/misc/fontloader-font-ots.lua b/src/fontloader/misc/fontloader-font-ots.lua
index 0f38508..10f0518 100644
--- a/src/fontloader/misc/fontloader-font-ots.lua
+++ b/src/fontloader/misc/fontloader-font-ots.lua
@@ -184,7 +184,7 @@ local copy_node          = nuts.copy
 local copy_node_list     = nuts.copy_list
 local find_node_tail     = nuts.tail
 local flush_node_list    = nuts.flush_list
-local free_node          = nuts.free
+local flush_node         = nuts.flush_node
 local end_of_math        = nuts.end_of_math
 local traverse_nodes     = nuts.traverse
 local traverse_id        = nuts.traverse_id
@@ -366,7 +366,7 @@ end
 local function flattendisk(head,disc)
     local _, _, replace, _, _, replacetail = getdisc(disc,true)
     setfield(disc,"replace",nil)
-    free_node(disc)
+    flush_node(disc)
     if head == disc then
         local next = getnext(disc)
         if replace then
@@ -2744,6 +2744,7 @@ local function kernrun(disc,k_run,font,attr,...)
                 done = true
             end
             setprev(pre,nest)
+-- setprev(pre,nil)
             setnext(prev,disc)
         end
     end
@@ -2773,6 +2774,7 @@ local function kernrun(disc,k_run,font,attr,...)
                 done = true
             end
             setprev(replace,nest)
+-- setprev(replace,nil)
             setnext(prev,disc)
         end
         if next then
diff --git a/src/fontloader/misc/fontloader-util-fil.lua b/src/fontloader/misc/fontloader-util-fil.lua
index 47d9d03..eeb6856 100644
--- a/src/fontloader/misc/fontloader-util-fil.lua
+++ b/src/fontloader/misc/fontloader-util-fil.lua
@@ -6,8 +6,10 @@ if not modules then modules = { } end modules ['util-fil'] = {
     license   = "see context related readme files"
 }
 
-local byte = string.byte
+local byte    = string.byte
+local char    = string.char
 local extract = bit32.extract
+local floor   = math.floor
 
 -- Here are a few helpers (the starting point were old ones I used for parsing
 -- flac files). In Lua 5.3 we can probably do this better. Some code will move
@@ -36,6 +38,8 @@ function files.size(f)
     return f:seek("end")
 end
 
+files.getsize = files.size
+
 function files.setposition(f,n)
     if zerobased[f] then
         f:seek("set",n)
@@ -180,3 +184,32 @@ end
 function files.skiplong(f,n)
     f:read(4*(n or 1))
 end
+
+-- writers (kind of slow)
+
+function files.writecardinal2(f,n)
+    local a = char(n % 256)
+    n = floor(n/256)
+    local b = char(n % 256)
+    f:write(b,a)
+end
+
+function files.writecardinal4(f,n)
+    local a = char(n % 256)
+    n = floor(n/256)
+    local b = char(n % 256)
+    n = floor(n/256)
+    local c = char(n % 256)
+    n = floor(n/256)
+    local d = char(n % 256)
+    f:write(d,c,b,a)
+end
+
+function files.writestring(f,s)
+    f:write(char(byte(s,1,#s)))
+end
+
+function files.writebyte(f,b)
+    f:write(char(b))
+end
+
diff --git a/src/fontloader/runtime/fontloader-reference.lua b/src/fontloader/runtime/fontloader-reference.lua
index e6738ea..13c45aa 100644
--- a/src/fontloader/runtime/fontloader-reference.lua
+++ b/src/fontloader/runtime/fontloader-reference.lua
@@ -1,6 +1,6 @@
 -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua
 -- parent file : c:/data/develop/context/sources/luatex-fonts.lua
--- merge date  : 07/13/16 15:09:54
+-- merge date  : 07/25/16 21:49:08
 
 do -- begin closure to overcome local limits and interference
 
@@ -4103,7 +4103,9 @@ if not modules then modules={} end modules ['util-fil']={
   license="see context related readme files"
 }
 local byte=string.byte
+local char=string.char
 local extract=bit32.extract
+local floor=math.floor
 utilities=utilities or {}
 local files={}
 utilities.files=files
@@ -4122,6 +4124,7 @@ end
 function files.size(f)
   return f:seek("end")
 end
+files.getsize=files.size
 function files.setposition(f,n)
   if zerobased[f] then
     f:seek("set",n)
@@ -4242,6 +4245,28 @@ end
 function files.skiplong(f,n)
   f:read(4*(n or 1))
 end
+function files.writecardinal2(f,n)
+  local a=char(n%256)
+  n=floor(n/256)
+  local b=char(n%256)
+  f:write(b,a)
+end
+function files.writecardinal4(f,n)
+  local a=char(n%256)
+  n=floor(n/256)
+  local b=char(n%256)
+  n=floor(n/256)
+  local c=char(n%256)
+  n=floor(n/256)
+  local d=char(n%256)
+  f:write(d,c,b,a)
+end
+function files.writestring(f,s)
+  f:write(char(byte(s,1,#s)))
+end
+function files.writebyte(f,b)
+  f:write(char(b))
+end
 
 end -- closure
 
@@ -4704,7 +4729,7 @@ end
 nodes.nodecodes=nodecodes
 nodes.glyphcodes=glyphcodes
 nodes.disccodes=disccodes
-local free_node=node.free
+local flush_node=node.flush_node
 local remove_node=node.remove
 local new_node=node.new
 local traverse_id=node.traverse_id
@@ -4724,7 +4749,7 @@ function nodes.remove(head,current,free_too)
   head,current=remove_node(head,current)
   if t then
     if free_too then
-      free_node(t)
+      flush_node(t)
       t=nil
     else
       t.next,t.prev=nil,nil
@@ -4748,12 +4773,14 @@ nodes.getattr=getfield
 nodes.setattr=setfield
 nodes.tostring=node.tostring or tostring
 nodes.copy=node.copy
+nodes.copy_node=node.copy
 nodes.copy_list=node.copy_list
 nodes.delete=node.delete
 nodes.dimensions=node.dimensions
 nodes.end_of_math=node.end_of_math
 nodes.flush_list=node.flush_list
 nodes.flush_node=node.flush_node
+nodes.flush=node.flush_node
 nodes.free=node.free
 nodes.insert_after=node.insert_after
 nodes.insert_before=node.insert_before
@@ -4767,7 +4794,6 @@ nodes.vpack=node.vpack
 nodes.first_glyph=node.first_glyph
 nodes.has_glyph=node.has_glyph or node.first_glyph
 nodes.current_attr=node.current_attr
-nodes.do_ligature_n=node.do_ligature_n
 nodes.has_field=node.has_field
 nodes.last_node=node.last_node
 nodes.usedlist=node.usedlist
@@ -4854,9 +4880,12 @@ nuts.insert_before=direct.insert_before
 nuts.insert_after=direct.insert_after
 nuts.delete=direct.delete
 nuts.copy=direct.copy
+nuts.copy_node=direct.copy
 nuts.copy_list=direct.copy_list
 nuts.tail=direct.tail
 nuts.flush_list=direct.flush_list
+nuts.flush_node=direct.flush_node
+nuts.flush=direct.flush
 nuts.free=direct.free
 nuts.remove=direct.remove
 nuts.is_node=direct.is_node
@@ -7608,7 +7637,9 @@ handlers.otf=otf
 local readers=otf.readers or {}
 otf.readers=readers
 local streamreader=utilities.files  
+local streamwriter=utilities.files
 readers.streamreader=streamreader
+readers.streamwriter=streamwriter
 local openfile=streamreader.open
 local closefile=streamreader.close
 local setposition=streamreader.setposition
@@ -15613,10 +15644,38 @@ local function copytotfm(data,cache_id)
     }
   end
 end
+local converters={
+  woff={
+    cachename="webfonts",
+    action=otf.readers.woff2otf,
+  }
+}
+local function checkconversion(specification)
+  local filename=specification.filename
+  local converter=converters[lower(file.suffix(filename))]
+  if converter then
+    local base=file.basename(filename)
+    local name=file.removesuffix(base)
+    local attr=lfs.attributes(filename)
+    local size=attr and attr.size or 0
+    local time=attr and attr.modification or 0
+    if size>0 then
+      local cleanname=containers.cleanname(name)
+      local cachename=caches.setfirstwritablefile(cleanname,converter.cachename)
+      if not io.exists(cachename) or (time~=lfs.attributes(cachename).modification) then
+        report_otf("caching font %a in %a",filename,cachename)
+        converter.action(filename,cachename) 
+        lfs.touch(cachename,time,time)
+      end
+      specification.filename=cachename
+    end
+  end
+end
 local function otftotfm(specification)
   local cache_id=specification.hash
   local tfmdata=containers.read(constructors.cache,cache_id)
   if not tfmdata then
+    checkconversion(specification) 
     local name=specification.name
     local sub=specification.sub
     local subindex=specification.subindex
@@ -15824,9 +15883,13 @@ local function opentypereader(specification,suffix)
   end
 end
 readers.opentype=opentypereader 
-function readers.otf (specification) return opentypereader(specification,"otf") end
-function readers.ttf (specification) return opentypereader(specification,"ttf") end
-function readers.ttc (specification) return opentypereader(specification,"ttf") end
+function readers.otf(specification) return opentypereader(specification,"otf") end
+function readers.ttf(specification) return opentypereader(specification,"ttf") end
+function readers.ttc(specification) return opentypereader(specification,"ttf") end
+function readers.woff(specification)
+  checkconversion(specification)
+  opentypereader(specification,"")
+end
 function otf.scriptandlanguage(tfmdata,attr)
   local properties=tfmdata.properties
   return properties.script or "dflt",properties.language or "dflt"
@@ -18042,7 +18105,7 @@ local copy_node=nuts.copy
 local copy_node_list=nuts.copy_list
 local find_node_tail=nuts.tail
 local flush_node_list=nuts.flush_list
-local free_node=nuts.free
+local flush_node=nuts.flush_node
 local end_of_math=nuts.end_of_math
 local traverse_nodes=nuts.traverse
 local traverse_id=nuts.traverse_id
@@ -18176,7 +18239,7 @@ end
 local function flattendisk(head,disc)
   local _,_,replace,_,_,replacetail=getdisc(disc,true)
   setfield(disc,"replace",nil)
-  free_node(disc)
+  flush_node(disc)
   if head==disc then
     local next=getnext(disc)
     if replace then
@@ -21087,9 +21150,9 @@ local setprop=nuts.setprop
 local ischar=nuts.is_char
 local insert_node_after=nuts.insert_after
 local copy_node=nuts.copy
-local free_node=nuts.free
 local remove_node=nuts.remove
 local flush_list=nuts.flush_list
+local flush_node=nuts.flush_node
 local copyinjection=nodes.injections.copy 
 local unsetvalue=attributes.unsetvalue
 local fontdata=fonts.hashes.identifiers
@@ -21599,7 +21662,7 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
     if current==stop then
       stop=getprev(stop)
       head=remove_node(head,current)
-      free_node(current)
+      flush_node(current)
       return head,stop,nbspaces
     else
       nbspaces=nbspaces+1
@@ -21637,7 +21700,7 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
                 setchar(current,getchar(tempcurrent)) 
                 local freenode=getnext(current)
                 setlink(current,tmp)
-                free_node(freenode)
+                flush_node(freenode)
                 flush_list(tempcurrent)
                 if changestop then
                   stop=current
@@ -21874,7 +21937,7 @@ local function deva_reorder(head,start,stop,font,attr,nbspaces)
   if getchar(base)==c_nbsp then
     nbspaces=nbspaces-1
     head=remove_node(head,base)
-    free_node(base)
+    flush_node(base)
   end
   return head,stop,nbspaces
 end
@@ -22242,7 +22305,7 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces)
     if current==stop then
       stop=getprev(stop)
       head=remove_node(head,current)
-      free_node(current)
+      flush_node(current)
       return head,stop,nbspaces
     else
       nbspaces=nbspaces+1
@@ -22427,7 +22490,7 @@ local function dev2_reorder(head,start,stop,font,attr,nbspaces)
   if getchar(base)==c_nbsp then
     nbspaces=nbspaces-1
     head=remove_node(head,base)
-    free_node(base)
+    flush_node(base)
   end
   return head,stop,nbspaces
 end
@@ -25658,7 +25721,7 @@ local fonts=fonts
 local nodes=nodes
 local nuts=nodes.nuts 
 local traverse_id=nuts.traverse_id
-local free_node=nuts.free
+local flush_node=nuts.flush_node
 local glyph_code=nodes.nodecodes.glyph
 local disc_code=nodes.nodecodes.disc
 local tonode=nuts.tonode
@@ -25782,7 +25845,7 @@ function nodes.handlers.nodepass(head)
             end
           end
         end
-        free_node(r)
+        flush_node(r)
       end
     end
     for d in traverse_id(disc_code,nuthead) do