Merge pull request #198 from phi-gamma/texlive2014

fix Makefile, add support for scanning local font files

4 files changed, 209 insertions, 78 deletions

diff --git a/Makefile b/Makefile
index aa5e580..d00b237 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,8 @@ BUILDDIR	= ./build
 MISCDIR		= ./misc
 
 SRC		= $(wildcard $(SRCSRCDIR)/luaotfload-*.lua)
-SRC		= $(MISCDIR)/luaotfload-blacklist.cnf
+SRC		+= $(SRCSRCDIR)/luaotfload.sty
+SRC		+= $(MISCDIR)/luaotfload-blacklist.cnf
 
 GLYPHSCRIPT	= $(SCRIPTSRCDIR)/mkglyphlist
 CHARSCRIPT	= $(SCRIPTSRCDIR)/mkcharacters
@@ -76,6 +77,8 @@ DO_GLYPHS	= $(LUA) $(GLYPHSCRIPT) > /dev/null
 DO_CHARS	= $(LUA) $(CHARSCRIPT)  > /dev/null
 DO_STATUS	= $(LUA) $(STATUSSCRIPT)  > /dev/null
 
+show: showtargets
+
 all: $(GENERATED)
 builddir: $(BUILDDIR)
 resources: $(RESOURCES)
@@ -130,7 +133,7 @@ endef
 
 define run-install
 @mkdir -p $(SCRIPTDIR) && cp -- $(SCRIPTSTATUS) $(SCRIPTDIR)
-@mkdir -p $(RUNDIR)    && cp -- $(RUNSTATUS) $(RUNDIR)
+@mkdir -p $(RUNDIR)    && cp -- $(RESOURCES) $(RUNSTATUS) $(RUNDIR)
 endef
 
 $(TDS_ZIP): TEXMFROOT=./tmp-texmf
@@ -142,7 +145,7 @@ $(TDS_ZIP): $(DOCS) $(ALL_STATUS)
 	@cd $(TEXMFROOT) && zip -9 ../$@ -r . >/dev/null
 	@$(RM) -r -- $(TEXMFROOT)
 
-.PHONY: install manifest clean mrproper
+.PHONY: install manifest clean mrproper show showtargets
 
 install: $(ALL_STATUS)
 	@echo "Installing in '$(TEXMFROOT)'."
@@ -169,4 +172,24 @@ mrproper: clean
 	@$(RM) -- $(GENERATED) $(ZIPS) $(GLYPHSOURCE)
 	@$(RM) -r -- $(BUILDDIR)
 
+###############################################################################
+showtargets:
+	@echo "Available targets:"
+	@echo
+	@echo "       all         build everything: documentation, resources,"
+	@echo "       world       build everything and package zipballs"
+	@echo "       doc         compile PDF documentation"
+	@echo "       resources   generate resource files (chars, glyphs)"
+	@echo
+	@echo "       pdf         build luaotfload.pdf"
+	@echo "       manual      crate manpage for luaotfload-tool (requires Docutils)"
+	@echo "       graph       generate file graph (requires GraphViz)"
+	@echo
+	@echo "       chars       import char-def.lua as luaotfload-characters.lua"
+	@echo "       status      create repository info (luaotfload-status.lua)"
+	@echo
+	@echo "       tds         package a zipball according to the TDS"
+	@echo "       ctan        package a zipball for uploading to CTAN"
+	@echo
+
 # vim:noexpandtab:tabstop=8:shiftwidth=2
diff --git a/src/luaotfload-database.lua b/src/luaotfload-database.lua
index 4b2d201..06a3dea 100644
--- a/src/luaotfload-database.lua
+++ b/src/luaotfload-database.lua
@@ -140,7 +140,7 @@ luaotfloadconfig.compress      = luaotfloadconfig.compress ~= false
 local names                    = fonts.names
 local name_index               = nil --> upvalue for names.data
 local lookup_cache             = nil --> for names.lookups
-names.version                  = 2.5
+names.version                  = 2.51
 names.data                     = nil      --- contains the loaded database
 names.lookups                  = nil      --- contains the lookup cache
 
@@ -356,6 +356,7 @@ This is a sketch of the luaotfload db:
         optical : (int, int) list;  // design size -> index entry
     }
     and metadata = {
+        local       : bool;        (* set if local fonts were added to the db *)
         formats     : string list; // { "otf", "ttf", "ttc", "dfont" }
         statistics  : TODO;
         version     : float;
@@ -439,7 +440,9 @@ mtx-fonts has in names.tma:
 
 --doc]]--
 
-local initialize_namedata = function (formats) --- returns dbobj
+--- string list -> dbobj
+
+local initialize_namedata = function (formats)
     return {
         --families        = { },
         status          = { }, -- was: status; map abspath -> mapping
@@ -447,6 +450,7 @@ local initialize_namedata = function (formats) --- returns dbobj
         names           = { },
 --      files           = { }, -- created later
         meta            = {
+            ["local"]  = false,
             formats    = formats,
             statistics = { },
             version    = names.version,
@@ -549,6 +553,10 @@ local save_lookups
 local update_names
 local get_font_filter
 local set_font_filter
+local generate_filedata
+local collect_families
+local group_modifiers
+local order_design_sizes
 
 --- state of the database
 local fonts_reloaded = false
@@ -568,7 +576,7 @@ load_names = function (dry_run)
         report ("info", 3, "db", "Loading took %0.f ms.",
                 1000 * (osgettimeofday () - starttime))
 
-        local db_version, nms_version
+        local db_version, names_version
         if data.meta then
             db_version = data.meta.version
         else
@@ -577,11 +585,11 @@ load_names = function (dry_run)
             --- an earlier index version broke.
             db_version = data.version or -42 --- invalid
         end
-        nms_version = names.version
-        if db_version ~= nms_version then
+        names_version = names.version
+        if db_version ~= names_version then
             report ("both", 0, "db",
                     [[Version mismatch; expected %4.3f, got %4.3f.]],
-                    nms_version, db_version)
+                    names_version, db_version)
             if not fonts_reloaded then
                 report ("both", 0, "db", [[Force rebuild.]])
                 data = update_names ({ }, true, false)
@@ -2207,7 +2215,7 @@ local scan_dir = function (dirname, currentnames, targetnames,
         --- ignore
         return 0, 0
     end
-    local found = find_font_files (dirname, location ~= "texmf")
+    local found = find_font_files (dirname, location ~= "texmf" and location ~= "local")
     if not found then
         report ("both", 4, "db",
                 "No such directory: %q; skipping.", dirname)
@@ -2380,17 +2388,49 @@ local scan_os_fonts = function (currentnames,
     return n_scanned, n_new
 end
 
---- unit -> (bool, lookup_cache)
+--- unit -> bool
 flush_lookup_cache = function ()
     lookup_cache = { }
     collectgarbage "collect"
-    return true, lookup_cache
+    return true
+end
+
+--[[doc--
+
+    scan_local_fonts() -- Scan font files in $PWD (during a TeX run)
+    and add them to the database.
+
+    This sets the “local” flag in the subtable “meta” to prevent the
+    merged table from being saved to disk.
+    
+    TODO the local tree could be cached in $PWD.
+
+--doc]]--
+
+local scan_local_fonts = function ()
+    local n_scanned, n_new  = 0, 0
+    local pwd               = lfscurrentdir ()
+    local name_index        = name_index
+    report ("both", 1, "db", "Scanning fonts in $PWD (%q) ...", pwd)
+
+    n_scanned, n_new = scan_dir (pwd, name_index, name_index, false, "local")
+    if n_new > 0 then
+        name_index.files       = generate_filedata (name_index.mappings)
+        name_index.families    = collect_families  (name_index.mappings)
+        name_index.families    = group_modifiers (name_index.mappings,
+                                                  name_index.families)
+        name_index.families    = order_design_sizes (name_index.families)
+        name_index.meta["local"] = true --- prevent saving to disk
+    end
+
+    return n_scanned, n_new
 end
 
+--- dbobj -> dbobj -> int * int
 
 --- fontentry list -> filemap
 
-local generate_filedata = function (mappings)
+generate_filedata = function (mappings)
 
     report ("both", 2, "db", "Creating filename map.")
 
@@ -2668,7 +2708,7 @@ local get_subtable = function (families, entry)
     return subtable
 end
 
-local collect_families = function (mappings)
+collect_families = function (mappings)
 
     report ("info", 2, "db", "Analyzing families.")
 
@@ -2765,7 +2805,7 @@ local bold_weight        = 700
 local style_categories   = { "r", "b", "i", "bi" }
 local bold_categories    = {      "b",      "bi" }
 
-local group_modifiers = function (mappings, families)
+group_modifiers = function (mappings, families)
     report ("info", 2, "db", "Analyzing shapes, weights, and styles.")
     for location, location_data in next, families do
         for format, format_data in next, location_data do
@@ -2863,7 +2903,7 @@ local cmp_sizes = function (a, b)
     return a [1] < b [1]
 end
 
-local order_design_sizes = function (families)
+order_design_sizes = function (families)
 
     report ("info", 2, "db", "Ordering design sizes.")
 
@@ -3123,11 +3163,11 @@ update_names = function (currentnames, force, dry_run)
 
         read_blacklist ()
 
-        local n_raw, n_new= retrieve_namedata (currentnames,
-                                               targetnames,
-                                               dry_run,
-                                               n_rawnames,
-                                               n_newnames)
+        local n_raw, n_new = retrieve_namedata (currentnames,
+                                                targetnames,
+                                                dry_run,
+                                                n_rawnames,
+                                                n_newnames)
         report ("info", 3, "db",
                 "Scanned %d font files; %d new entries.",
                 n_rawnames, n_newnames)
@@ -3164,16 +3204,16 @@ update_names = function (currentnames, force, dry_run)
 
     if dry_run ~= true then
 
-        save_names ()
+        local success, reason = save_names ()
+        if not success then
+            report ("both", 0, "db",
+                    "Failed to save database to disk: %s",
+                    reason)
+        end
 
-        local success, _lookups = flush_lookup_cache ()
-        if success then
-            local success = save_lookups ()
-            if success then
-                report ("info", 2, "cache",
-                                   "Lookup cache emptied.")
-                return targetnames
-            end
+        if flush_lookup_cache () and save_lookups () then
+            report ("both", 2, "cache", "Lookup cache emptied.")
+            return targetnames
         end
     end
     return targetnames
@@ -3206,11 +3246,16 @@ save_lookups = function ( )
 end
 
 --- save_names() is usually called without the argument
---- dbobj? -> bool
+--- dbobj? -> bool * string option
 save_names = function (currentnames)
     if not currentnames then
         currentnames = name_index
     end
+    if not currentnames or type (currentnames) ~= "table" then
+        return false, "invalid names table"
+    elseif currentnames.meta and currentnames.meta["local"] then
+        return false, "table contains local entries"
+    end
     local path = names.path.index
     local luaname, lucname = path.lua, path.luc
     if fileiswritable (luaname) and fileiswritable (lucname) then
@@ -3421,6 +3466,7 @@ names.read_blacklist              = read_blacklist
 names.sanitize_fontname           = sanitize_fontname
 names.getfilename                 = resolve_fullpath
 names.set_location_precedence     = set_location_precedence
+names.scan_local_fonts            = scan_local_fonts
 
 --- font cache
 names.purge_cache    = purge_cache
diff --git a/src/luaotfload-fontloader.lua b/src/luaotfload-fontloader.lua
index 8c31750..3f408b9 100644
--- a/src/luaotfload-fontloader.lua
+++ b/src/luaotfload-fontloader.lua
@@ -1,6 +1,6 @@
 -- merged file : luatex-fonts-merged.lua
 -- parent file : luatex-fonts.lua
--- merge date  : 02/07/14 00:57:35
+-- merge date  : 02/14/14 17:07:59
 
 do -- begin closure to overcome local limits and interference
 
@@ -82,6 +82,9 @@ function optionalrequire(...)
     return result
   end
 end
+if lua then
+  lua.mask=load([[τεχ = 1]]) and "utf" or "ascii"
+end
 
 end -- closure
 
@@ -172,9 +175,11 @@ patterns.spacer=spacer
 patterns.whitespace=whitespace
 patterns.nonspacer=nonspacer
 patterns.nonwhitespace=nonwhitespace
-local stripper=spacer^0*C((spacer^0*nonspacer^1)^0)
+local stripper=spacer^0*C((spacer^0*nonspacer^1)^0)   
+local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0)
 local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0))
 patterns.stripper=stripper
+patterns.fullstripper=fullstripper
 patterns.collapser=collapser
 patterns.lowercase=lowercase
 patterns.uppercase=uppercase
@@ -754,11 +759,15 @@ function string.limit(str,n,sentinel)
   end
 end
 local stripper=patterns.stripper
+local fullstripper=patterns.fullstripper
 local collapser=patterns.collapser
 local longtostring=patterns.longtostring
 function string.strip(str)
   return lpegmatch(stripper,str) or ""
 end
+function string.fullstrip(str)
+  return lpegmatch(fullstripper,str) or ""
+end
 function string.collapsespaces(str)
   return lpegmatch(collapser,str) or ""
 end
@@ -2516,11 +2525,18 @@ local unpack,concat=table.unpack,table.concat
 local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc
 local patterns,lpegmatch=lpeg.patterns,lpeg.match
 local utfchar,utfbyte=utf.char,utf.byte
-local loadstripped=function(str,shortcuts)
-  if shortcuts then
-    return load(dump(load(str),true),nil,nil,shortcuts)
-  else
-    return load(dump(load(str),true))
+local loadstripped=nil
+if _LUAVERSION<5.2 then
+  loadstripped=function(str,shortcuts)
+    return load(str)
+  end
+else
+  loadstripped=function(str,shortcuts)
+    if shortcuts then
+      return load(dump(load(str),true),nil,nil,shortcuts)
+    else
+      return load(dump(load(str),true))
+    end
   end
 end
 if not number then number={} end 
@@ -2676,29 +2692,53 @@ local template=[[
 %s
 return function(%s) return %s end
 ]]
-local environment={
-  global=global or _G,
-  lpeg=lpeg,
-  type=type,
-  tostring=tostring,
-  tonumber=tonumber,
-  format=string.format,
-  concat=table.concat,
-  signed=number.signed,
-  points=number.points,
-  basepoints=number.basepoints,
-  utfchar=utf.char,
-  utfbyte=utf.byte,
-  lpegmatch=lpeg.match,
-  nspaces=string.nspaces,
-  tracedchar=string.tracedchar,
-  autosingle=string.autosingle,
-  autodouble=string.autodouble,
-  sequenced=table.sequenced,
-  formattednumber=number.formatted,
-  sparseexponent=number.sparseexponent,
-}
-local preamble=""
+local preamble,environment="",{}
+if _LUAVERSION<5.2 then
+  preamble=[[
+local lpeg=lpeg
+local type=type
+local tostring=tostring
+local tonumber=tonumber
+local format=string.format
+local concat=table.concat
+local signed=number.signed
+local points=number.points
+local basepoints= number.basepoints
+local utfchar=utf.char
+local utfbyte=utf.byte
+local lpegmatch=lpeg.match
+local nspaces=string.nspaces
+local tracedchar=string.tracedchar
+local autosingle=string.autosingle
+local autodouble=string.autodouble
+local sequenced=table.sequenced
+local formattednumber=number.formatted
+local sparseexponent=number.sparseexponent
+    ]]
+else
+  environment={
+    global=global or _G,
+    lpeg=lpeg,
+    type=type,
+    tostring=tostring,
+    tonumber=tonumber,
+    format=string.format,
+    concat=table.concat,
+    signed=number.signed,
+    points=number.points,
+    basepoints=number.basepoints,
+    utfchar=utf.char,
+    utfbyte=utf.byte,
+    lpegmatch=lpeg.match,
+    nspaces=string.nspaces,
+    tracedchar=string.tracedchar,
+    autosingle=string.autosingle,
+    autodouble=string.autodouble,
+    sequenced=table.sequenced,
+    formattednumber=number.formatted,
+    sparseexponent=number.sparseexponent,
+  }
+end
 local arguments={ "a1" } 
 setmetatable(arguments,{ __index=function(t,k)
     local v=t[k-1]..",a"..k
@@ -3045,14 +3085,22 @@ local function use(t,fmt,...)
   return t[fmt](...)
 end
 strings.formatters={}
-function strings.formatters.new()
-  local e={} 
-  for k,v in next,environment do
-    e[k]=v
+if _LUAVERSION<5.2 then
+  function strings.formatters.new()
+    local t={ _extensions_={},_preamble_=preamble,_environment_={},_type_="formatter" }
+    setmetatable(t,{ __index=make,__call=use })
+    return t
+  end
+else
+  function strings.formatters.new()
+    local e={} 
+    for k,v in next,environment do
+      e[k]=v
+    end
+    local t={ _extensions_={},_preamble_="",_environment_=e,_type_="formatter" }
+    setmetatable(t,{ __index=make,__call=use })
+    return t
   end
-  local t={ _extensions_={},_preamble_="",_environment_=e,_type_="formatter" }
-  setmetatable(t,{ __index=make,__call=use })
-  return t
 end
 local formatters=strings.formatters.new() 
 string.formatters=formatters 
@@ -3074,9 +3122,15 @@ patterns.xmlescape=Cs((P("<")/"&lt;"+P(">")/"&gt;"+P("&")/"&amp;"+P('"')/"&quot;
 patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+P(1))^0)
 patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) 
 patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"'))
-add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape })
-add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape })
-add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape })
+if _LUAVERSION<5.2 then
+  add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],"local xmlescape = lpeg.patterns.xmlescape")
+  add(formatters,"tex",[[lpegmatch(texescape,%s)]],"local texescape = lpeg.patterns.texescape")
+  add(formatters,"lua",[[lpegmatch(luaescape,%s)]],"local luaescape = lpeg.patterns.luaescape")
+else
+  add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape })
+  add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape })
+  add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape })
+end
 
 end -- closure
 
diff --git a/src/luaotfload-tool.lua b/src/luaotfload-tool.lua
index 35765b5..9f00956 100755
--- a/src/luaotfload-tool.lua
+++ b/src/luaotfload-tool.lua
@@ -723,9 +723,9 @@ set.
 --]]--
 
 local action_sequence = {
-    "loglevel",  "help",  "version", "diagnose",
-    "blacklist", "cache", "flush",   "generate",
-    "list",      "query",
+    "loglevel",     "help",  "version", "diagnose",
+    "blacklist",    "cache", "flush",   "generate",
+    "scan_local",   "list",  "query",
 }
 
 local action_pending  = tabletohash(action_sequence, false)
@@ -762,8 +762,7 @@ actions.blacklist = function (job)
 end
 
 actions.generate = function (job)
-    local fontnames, savedname
-    fontnames = names.update(fontnames, job.force_reload, job.dry_run)
+    local fontnames = names.update(fontnames, job.force_reload, job.dry_run)
     report ("info", 2, "db", "Fonts in the database: %i", #fontnames.mappings)
     if names.data() then
         return true, true
@@ -771,8 +770,13 @@ actions.generate = function (job)
     return false, false
 end
 
+actions.scan_local = function (job)
+    names.scan_local_fonts ()
+    return true, true
+end
+
 actions.flush = function (job)
-    local success, lookups = names.flush_lookup_cache()
+    local success = names.flush_lookup_cache()
     if success then
         local success = names.save_lookups()
         if success then
@@ -1094,6 +1098,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
         inspect            = "I",
         limit              = 1,
         list               = 1,
+        ["local"]          = "L",
         log                = 1,
         ["max-fonts"]      = 1,
         ["no-reload"]      = "n",
@@ -1109,7 +1114,7 @@ local process_cmdline = function ( ) -- unit -> jobspec
         warnings           = "w",
     }
 
-    local short_options = "bcDfFiIlnpqRSuvVhw"
+    local short_options = "bcDfFiIlLnpqRSuvVhw"
 
     local options, _, optarg =
         alt_getopt.get_ordered_opts (arg, short_options, long_options)
@@ -1167,6 +1172,9 @@ local process_cmdline = function ( ) -- unit -> jobspec
             result.full_info = true
         elseif v == "l" then
             action_pending["flush"] = true
+        elseif v == "L" then
+            action_pending["generate"] = true
+            action_pending["scan_local"] = true
         elseif v == "list" then
             action_pending["list"] = true
             result.criterion = optarg[n]