Merge pull request #150 from phi-gamma/master

tweak type1 classification; enable compression; sync

15 files changed, 136 insertions, 66 deletions

diff --git a/NEWS b/NEWS
index c72b943..3825605 100644
--- a/NEWS
+++ b/NEWS
@@ -9,8 +9,9 @@ Change History
     * Test runner (script mktests)
     * New luaotfload-tool option: ``--no-reload``
     * ``luaotfload-tool --find`` now understands request syntax
-    * option ``--compress`` filters text (Lua script) version of the font
-      index through gzip
+    * text (Lua script) version of the font index is filtered through gzip by
+      default, the ``--no-compress`` option of luaotfload-tool will prevent
+      this
     * rename ``luaotfload-merged.lua`` (the fontloader package from Context)
       to ``luaotfload-fontloader.lua``
 
diff --git a/filegraph.dot b/filegraph.dot
index f70ef87..0c64e58 100644
--- a/filegraph.dot
+++ b/filegraph.dot
@@ -182,7 +182,7 @@ strict digraph luaotfload_files { //looks weird with circo ...
                 fillcolor  = "#01012222",
                 color      = grey40,
                 style      = "filled,dotted,rounded",
-                label      = "luaotfload-names.lua\nluaotfload-names.luc"]
+                label      = "luaotfload-names.lua.gz\nluaotfload-names.luc"]
 
     status [style      = "filled,dashed",
             shape      = rect,
diff --git a/luaotfload-auxiliary.lua b/luaotfload-auxiliary.lua
index 7daf367..d3de731 100644
--- a/luaotfload-auxiliary.lua
+++ b/luaotfload-auxiliary.lua
@@ -2,9 +2,9 @@
 -----------------------------------------------------------------------
 --         FILE:  luaotfload-auxiliary.lua
 --  DESCRIPTION:  part of luaotfload
--- REQUIREMENTS:  luaotfload 2.3
+-- REQUIREMENTS:  luaotfload 2.4
 --       AUTHOR:  Khaled Hosny, Élie Roux, Philipp Gesang
---      VERSION:  2.3a
+--      VERSION:  2.4
 --      CREATED:  2013-05-01 14:40:50+0200
 -----------------------------------------------------------------------
 --
diff --git a/luaotfload-colors.lua b/luaotfload-colors.lua
index ac9291d..b5c1fb9 100644
--- a/luaotfload-colors.lua
+++ b/luaotfload-colors.lua
@@ -1,5 +1,5 @@
 if not modules then modules = { } end modules ['luaotfload-colors'] = {
-    version   = "2.3a",
+    version   = "2.4",
     comment   = "companion to luaotfload.lua (font color)",
     author    = "Khaled Hosny, Elie Roux, Philipp Gesang",
     copyright = "Luaotfload Development Team",
diff --git a/luaotfload-database.lua b/luaotfload-database.lua
index 5198b16..fba01cc 100644
--- a/luaotfload-database.lua
+++ b/luaotfload-database.lua
@@ -1,15 +1,40 @@
 if not modules then modules = { } end modules ['luaotfload-database'] = {
-    version   = "2.3b",
+    version   = "2.4",
     comment   = "companion to luaotfload.lua",
     author    = "Khaled Hosny, Elie Roux, Philipp Gesang",
     copyright = "Luaotfload Development Team",
     license   = "GNU GPL v2"
 }
 
---- TODO: if the specification is an absolute filename with a font not in the
---- database, add the font to the database and load it. There is a small
---- difficulty with the filenames of the TEXMF tree that are referenced as
---- relative paths...
+--[[doc--
+
+    Some statistics:
+
+        a) TL 2012,     mkluatexfontdb --force
+        b) v2.4,        luaotfload-tool --update --force
+        c) v2.4,        luaotfload-tool --update --force --formats=+afm,pfa,pfb
+        d) Context,     mtxrun --script fonts --reload --force
+
+    (Keep in mind that Context does index fewer fonts since it
+    considers only the contents of the minimals tree, not the
+    tex live one!)
+
+                time (m:s)       peak VmSize (kB)
+            a     1:19              386 018
+            b     0:37              715 797
+            c     2:27            1 017 674
+            d     0:44            1 082 313
+
+    Most of the increase in memory consumption from version 1.x to 2.2+
+    can be attributed to the move from single-pass to a multi-pass
+    approach to building the index: Information is first gathered from
+    all reachable fonts and only afterwards processed, classified and
+    discarded.  Also, there is a good deal of additional stuff kept in
+    the database now: two extra tables for file names and font families
+    have been added, making font lookups more efficient while improving
+    maintainability of the code.
+
+--doc]]--
 
 local lpeg = require "lpeg"
 
@@ -99,11 +124,10 @@ luaotfloadconfig.resolver      = luaotfloadconfig.resolver or "normal"
 luaotfloadconfig.formats       = luaotfloadconfig.formats  or "otf,ttf,ttc,dfont"
 luaotfloadconfig.strip         = luaotfloadconfig.strip == true
 
-if luaotfloadconfig.update_live ~= false then
-    --- this option allows for disabling updates
-    --- during a TeX run
-    luaotfloadconfig.update_live = true
-end
+--- this option allows for disabling updates
+--- during a TeX run
+luaotfloadconfig.update_live   = luaotfloadconfig.update_live ~= false
+luaotfloadconfig.compress      = luaotfloadconfig.compress ~= false
 
 local names                    = fonts.names
 local name_index               = nil --> upvalue for names.data
@@ -509,6 +533,7 @@ local set_font_filter
 
 --- state of the database
 local fonts_reloaded = false
+local fonts_read     = 0
 
 --- limit output when approximate font matching (luaotfload-tool -F)
 local fuzzy_limit = 1 --- display closest only
@@ -1481,7 +1506,7 @@ ot_fullinfo = function (filename,
                                               english_names,
                                               info)
 
-    return {
+    local res = {
         file            = { base        = basename,
                             full        = filename,
                             subfont     = subfont,
@@ -1491,6 +1516,10 @@ ot_fullinfo = function (filename,
         style           = style,
         version         = metadata.version,
     }
+    --- Closing the file manually is a tad faster and more memory
+    --- efficient than having it closed by the gc
+    fontloaderclose (metadata)
+    return res
 end
 
 --[[doc--
@@ -1507,13 +1536,14 @@ end
 
 t1_fullinfo = function (filename, _subfont, location, basename, format)
     local sanitized
-    local metadata  = load_font_file (filename)
-
+    local metadata      = load_font_file (filename)
     local fontname      = metadata.fontname
     local fullname      = metadata.fullname
     local familyname    = metadata.familyname
     local italicangle   = metadata.italicangle
-    local weight        = metadata.weight --- string identifier
+    local splitstyle    = split_fontname (fontname)
+    local style         = ""
+    local weight
 
     sanitized = sanitize_fontnames ({
         fontname        = fontname,
@@ -1521,10 +1551,20 @@ t1_fullinfo = function (filename, _subfont, location, basename, format)
         pfullname       = fullname,
         metafamily      = family,
         familyname      = familyname,
-        subfamily       = weight,
+        weight          = metadata.weight, --- string identifier
         prefmodifiers   = style,
     })
 
+    weight = sanitized.weight
+
+    if weight == "bold" then
+        style = weight
+    end
+
+    if italicangle ~= 0 then
+        style = style .. "italic"
+    end
+
     return {
         basename         = basename,
         fullpath         = filename,
@@ -1538,10 +1578,10 @@ t1_fullinfo = function (filename, _subfont, location, basename, format)
         psname           = sanitized.fontname,
         version          = metadata.version,
         size             = false,
-        splitstyle       = split_fontname (fontname),
-        fontstyle_name   = sanitized.subfamily,
+        splitstyle       = splitstyle,
+        fontstyle_name   = style ~= "" and style or weight,
         weight           = { metadata.pfminfo.weight,
-                             sanitized.subfamily },
+                             weight },
         italicangle      = italicangle,
     }
 end
@@ -2077,21 +2117,29 @@ local scan_dir = function (dirname, currentnames, targetnames,
     end
     report ("both", 4, "db", "Scanning directory %s", dirname)
 
-    local n_new = 0   --- total of fonts collected
-    local n_found = #found
+    local n_new         = 0   --- total of fonts collected
+    local n_found       = #found
+    local max_fonts     = luaotfloadconfig.max_fonts
+
     report ("both", 4, "db", "%d font files detected", n_found)
     for j=1, n_found do
+        if max_fonts and fonts_read >= max_fonts then
+            break
+        end
+
         local fullname = found[j]
         fullname = path_normalize(fullname)
         local new
+
         if dry_run == true then
             report_status ("both", "db",
-                           "Would have been loading %q", fullname)
+                        "Would have been loading %q", fullname)
         else
             report_status ("both", "db", "Loading font %q", fullname)
             local new = read_font_names (fullname, currentnames,
-                                         targetnames, texmf)
+                                        targetnames, texmf)
             if new == true then
+                fonts_read = fonts_read + 1
                 n_new = n_new + 1
             end
         end
diff --git a/luaotfload-diagnostics.lua b/luaotfload-diagnostics.lua
index a11f9ea..f191913 100644
--- a/luaotfload-diagnostics.lua
+++ b/luaotfload-diagnostics.lua
@@ -4,7 +4,7 @@
 --  DESCRIPTION:  functionality accessible by the --diagnose option
 -- REQUIREMENTS:  luaotfload-tool.lua
 --       AUTHOR:  Philipp Gesang (Phg), <phg42.2a@gmail.com>
---      VERSION:  1.0
+--      VERSION:  2.4
 --      CREATED:  2013-07-28 10:01:18+0200
 -----------------------------------------------------------------------
 --
diff --git a/luaotfload-extralibs.lua b/luaotfload-extralibs.lua
index d04ba5b..6d486b2 100644
--- a/luaotfload-extralibs.lua
+++ b/luaotfload-extralibs.lua
@@ -1,5 +1,5 @@
 if not modules then modules = { } end modules ["extralibs"] = {
-    version   = "2.3a",
+    version   = "2.4",
     comment   = "companion to luaotfload.lua",
     author    = "Hans Hagen, Philipp Gesang",
     copyright = "PRAGMA ADE / ConTeXt Development Team",
diff --git a/luaotfload-features.lua b/luaotfload-features.lua
index 83f5a99..4b2f206 100644
--- a/luaotfload-features.lua
+++ b/luaotfload-features.lua
@@ -1,5 +1,5 @@
 if not modules then modules = { } end modules ["features"] = {
-    version   = "2.3a",
+    version   = "2.4",
     comment   = "companion to luaotfload.lua",
     author    = "Hans Hagen, Khaled Hosny, Elie Roux, Philipp Gesang",
     copyright = "PRAGMA ADE / ConTeXt Development Team",
diff --git a/luaotfload-fontloader.lua b/luaotfload-fontloader.lua
index 2bfa775..e187979 100644
--- a/luaotfload-fontloader.lua
+++ b/luaotfload-fontloader.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 11/09/13 20:21:53
+-- merge date  : 11/14/13 13:48:28
 
 do -- begin closure to overcome local limits and interference
 
@@ -6354,7 +6354,7 @@ local utfbyte=utf.byte
 local format,gmatch,gsub,find,match,lower,strip=string.format,string.gmatch,string.gsub,string.find,string.match,string.lower,string.strip
 local type,next,tonumber,tostring=type,next,tonumber,tostring
 local abs=math.abs
-local getn=table.getn
+local insert=table.insert
 local lpegmatch=lpeg.match
 local reversed,concat,remove=table.reversed,table.concat,table.remove
 local ioflush=io.flush
@@ -6378,7 +6378,7 @@ local report_otf=logs.reporter("fonts","otf loading")
 local fonts=fonts
 local otf=fonts.handlers.otf
 otf.glists={ "gsub","gpos" }
-otf.version=2.747 
+otf.version=2.748 
 otf.cache=containers.define("fonts","otf",otf.version,true)
 local fontdata=fonts.hashes.identifiers
 local chardata=characters and characters.data 
@@ -7032,20 +7032,20 @@ actions["check encoding"]=function(data,filename,raw)
           if parentdescription then
             local altuni=parentdescription.altuni
             if not altuni then
-              altuni={ { unicode=parent } }
+              altuni={ { unicode=unicode } }
               parentdescription.altuni=altuni
               duplicates[parent]={ unicode }
             else
               local done=false
               for i=1,#altuni do
-                if altuni[i].unicode==parent then
+                if altuni[i].unicode==unicode then
                   done=true
                   break
                 end
               end
               if not done then
-                altuni[#altuni+1]={ unicode=parent }
-                table.insert(duplicates[parent],unicode)
+                insert(altuni,{ unicode=unicode })
+                insert(duplicates[parent],unicode)
               end
             end
             if trace_loading then
diff --git a/luaotfload-override.lua b/luaotfload-override.lua
index ea6af9a..0e17f05 100644
--- a/luaotfload-override.lua
+++ b/luaotfload-override.lua
@@ -1,5 +1,5 @@
 if not modules then modules = { } end modules ['luat-ovr'] = {
-    version   = "2.3a",
+    version   = "2.4",
     comment   = "companion to luatex-*.tex",
     author    = "Khaled Hosny, Elie Roux, Philipp Gesang",
     copyright = "Luaotfload Development Team",
diff --git a/luaotfload-tool.lua b/luaotfload-tool.lua
index 0cc26d2..a945ee2 100755
--- a/luaotfload-tool.lua
+++ b/luaotfload-tool.lua
@@ -183,6 +183,7 @@ local sanitize_fontname              = names.sanitize_fontname
 
 local pathdata      = names.path
 local names_plain   = pathdata.index.lua
+local names_gzip    = names_plain .. ".gz"
 local names_bin     = pathdata.index.luc
 
 local help_messages = {
@@ -219,11 +220,12 @@ This tool is part of the luaotfload package. Valid options are:
   -n --no-reload               suppress db update
   --no-strip                   keep redundant information in db
   -f --force                   force re-indexing all fonts
-  -c --compress                gzip index file (text version only)
+  -c --no-compress             do not gzip index file (text version only)
   -l --flush-lookups           empty lookup cache of font requests
   -D --dry-run                 skip loading of fonts, just scan
   --formats=[+|-]EXTENSIONS    set, add, or subtract formats to index
   -p --prefer-texmf            prefer fonts in the TEXMF over system fonts
+  --max-fonts=N                process at most N font files
 
   --find="font name"           query the database for a font name
   -F --fuzzy                   look for approximate matches if --find fails
@@ -293,7 +295,8 @@ local help_msg = function (version)
     local template = help_messages[version]
     iowrite(stringformat(template,
                          luaotfloadconfig.self,
-                         names_plain,
+--                         names_plain,
+                         names_gzip,
                          names_bin,
                          caches.getwritablepath (
                          luaotfloadconfig.cache_dir)))
@@ -1097,7 +1100,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
     local long_options = {
         alias              = 1,
         cache              = 1,
-        compress           = "c",
+        ["no-compress"]    = "c",
         diagnose           = 1,
         ["dry-run"]        = "D",
         ["flush-lookups"]  = "l",
@@ -1112,6 +1115,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
         limit              = 1,
         list               = 1,
         log                = 1,
+        ["max-fonts"]      = 1,
         ["no-reload"]      = "n",
         ["no-strip"]       = 0,
         ["skip-read"]      = "R",
@@ -1214,9 +1218,17 @@ local process_cmdline = function ( ) -- unit -> jobspec
             ---  dev only, undocumented
             luaotfloadconfig.skip_read = true
         elseif v == "c" then
-            luaotfloadconfig.compress = true
+            luaotfloadconfig.compress = false
         elseif v == "no-strip" then
             luaotfloadconfig.strip = false
+        elseif v == "max-fonts" then
+            local n = optarg[n]
+            if n then
+                n = tonumber(n)
+                if n and n > 0 then
+                    luaotfloadconfig.max_fonts = n
+                end
+            end
         end
     end
 
diff --git a/luaotfload-tool.rst b/luaotfload-tool.rst
index 37ef779..957ec85 100644
--- a/luaotfload-tool.rst
+++ b/luaotfload-tool.rst
@@ -20,7 +20,7 @@ SYNOPSIS
 **luaotfload-tool** --update [ --force ] [ --quiet ] [ --verbose ]
                              [ --prefer-texmf ] [ --dry-run ]
                              [ --formats=[+|-]EXTENSIONS ]
-                             [ --compress ] [ --no-strip ]
+                             [ --no-compress ] [ --no-strip ]
 
 **luaotfload-tool** --find=FONTNAME [ --fuzzy ] [ --info ] [ --inspect ]
                                     [ --no-reload ]
@@ -67,12 +67,15 @@ update mode
                         building the database. Warning: this will
                         inflate the index to about two to three times
                         the normal size.
---compress              Filter plain text version of font index through
-                        gzip.
+--no-compress, -c       Do not filter the plain text version of the
+                        font index through gzip. Useful for debugging
+                        if your editor is built without zlib.
 
 --prefer-texmf, -p      Organize the file name database in a way so
                         that it prefer fonts in the *TEXMF* tree over
                         system fonts if they are installed in both.
+--max-fonts=N           Process at most *N* font files, including fonts
+                        already indexed in the count.
 --formats=EXTENSIONS    Extensions of the font files to index.
                         Where *EXTENSIONS* is a comma-separated list of
                         supported file extensions (otf, ttf, ttc,
@@ -220,11 +223,17 @@ FILES
 
 The font name database is usually located in the directory
 ``texmf-var/luatex-cache/generic/names/`` (``$TEXMFCACHE`` as set in
-``texmf.cnf``) of your *TeX Live* distribution as
-``luaotfload-names.lua``.  The experimental lookup cache will be
-created as ``luaotfload-lookup-cache.lua`` in the same directory.
-Both files are safe to delete, at the cost of regenerating them with
-the next run of *LuaTeX*.
+``texmf.cnf``) of your *TeX Live* distribution as a zlib-compressed
+file ``luaotfload-names.lua.gz``.
+The experimental lookup cache will be created as
+``luaotfload-lookup-cache.lua`` in the same directory.
+These Lua tables are not used directly by Luaotfload, though.
+Instead, they are compiled to Lua bytecode which is written to
+corresponding files with the extension ``.luc`` in the same directory.
+When modifying the files by hand keep in mind that only if the bytecode
+files are missing will Luaotfload use the plain version instead.
+Both kinds of files are safe to delete, at the cost of regenerating
+them with the next run of *LuaTeX*.
 
 SEE ALSO
 =======================================================================
diff --git a/mkcharacters b/mkcharacters
index 69201fb..5d4a2f4 100755
--- a/mkcharacters
+++ b/mkcharacters
@@ -5,7 +5,7 @@
 --  DESCRIPTION:  import parts of char-def.lua
 -- REQUIREMENTS:  lua, ConTeXt, the lualibs package
 --       AUTHOR:  Philipp Gesang (Phg), <phg42.2a@gmail.com>
---      VERSION:  1.1
+--      VERSION:  2.4
 --      CREATED:  2013-05-17 12:41:39+0200
 -----------------------------------------------------------------------
 -- we create a stripped-down version of char-def.lua
diff --git a/mkglyphlist b/mkglyphlist
index e476eca..f7a1cb9 100755
--- a/mkglyphlist
+++ b/mkglyphlist
@@ -5,7 +5,7 @@
 --  DESCRIPTION:  part of the luaotfload package
 -- REQUIREMENTS:  lua, lpeg, luasocket, the lualibs package
 --       AUTHOR:  Philipp Gesang (Phg), <phg42.2a@gmail.com>
---      VERSION:  1.0
+--      VERSION:  2.4
 --      CREATED:  04/23/2013 12:42:17 PM CEST
 -----------------------------------------------------------------------
 -- interesting thread on the Context list:
@@ -96,7 +96,7 @@ end
 
 local file_header = [==[
 if not modules then modules = { } end modules ["font-age"] = {
-  version     = 2.300,
+  version     = 2.400,
   comment     = "part of the luaotfload package",
   author      = "luaotfload team / mkglyphlist",
   copyright   = "derived from %s",
diff --git a/mktests b/mktests
index eaf931f..d496896 100755
--- a/mktests
+++ b/mktests
@@ -119,18 +119,18 @@ local choose_style = {
 --- this needs a database built with --formats=+pfa,pfb,afm
 
 local resolve_t1_font = {
-  { { name = "URW Gothic L",          style = "regular"    }, "uagk8a.pfb"   },
-  { { name = "URW Gothic L",          style = "italic"     }, "uagko8a.pfb"  },
-  { { name = "URW Gothic L",          style = "bold"       }, "uagd8a.pfb"   },
-  { { name = "URW Gothic L",          style = "bolditalic" }, "uagdo8a.pfb"  },
-  { { name = "Century Schoolbook L",  style = "regular"    }, "uncr8a.pfb"   },
-  { { name = "Century Schoolbook L",  style = "italic"     }, "uncri8a.pfb"  },
-  { { name = "Century Schoolbook L",  style = "bold"       }, "uncb8a.pfb"   },
-  { { name = "Century Schoolbook L",  style = "bolditalic" }, "uncbi8a.pfb"  },
-  { { name = "Nimbus Roman No9 L",    style = "regular"    }, "utmr8a.pfb"   },
-  { { name = "Nimbus Roman No9 L",    style = "italic"     }, "utmri8a.pfb"  },
-  { { name = "Nimbus Roman No9 L",    style = "bold"       }, "utmb8a.pfb"   }, --- medium, actually
-  { { name = "Nimbus Roman No9 L",    style = "bolditalic" }, "utmbi8a.pfb"  },
+  { { name = "URW Gothic L",          style = "regular"    }, "a010013l.pfb"  }, --> “book”
+--  { { name = "URW Gothic L",          style = "italic"     }, "a010033l.pfb"  }, --> “book oblique”
+--  { { name = "URW Gothic L",          style = "bold"       }, "a010015l.pfb"  }, --> “demi”
+--  { { name = "URW Gothic L",          style = "bolditalic" }, "a010035l.pfb"  }, --> “demi oblique”
+  { { name = "Century Schoolbook L",  style = "regular"    }, "c059013l.pfb"  },
+  { { name = "Century Schoolbook L",  style = "italic"     }, "c059033l.pfb"  },
+  { { name = "Century Schoolbook L",  style = "bold"       }, "c059016l.pfb"  },
+  { { name = "Century Schoolbook L",  style = "bolditalic" }, "c059036l.pfb"  },
+  { { name = "Nimbus Roman No9 L",    style = "regular"    }, "n021003l.pfb"  },
+  { { name = "Nimbus Roman No9 L",    style = "italic"     }, "n021023l.pfb"  },
+  { { name = "Nimbus Roman No9 L",    style = "bold"       }, "n021004l.pfb"  }, --- medium, actually
+  { { name = "Nimbus Roman No9 L",    style = "bolditalic" }, "n021024l.pfb"  },
 }
 
 local translate_style = {