Merge pull request #77 from phi-gamma/master

address Robert’s feature request concerning ``\letterspacefont``

4 files changed, 339 insertions, 16 deletions

diff --git a/Makefile b/Makefile
index fce2e70..10b93ad 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,9 @@ OTFL         = $(wildcard luaotfload-*.lua) luaotfload-blacklist.cnf
 
 GLYPHSCRIPT  = mkglyphlist
 GLYPHSOURCE  = glyphlist.txt
+CHARSCRIPT   = mkcharacters
+
+RESOURCESCRIPTS = $(GLYPHSCRIPT) $(CHARSCRIPT)
 
 SCRIPTNAME   = luaotfload-tool
 SCRIPT       = $(SCRIPTNAME).lua
@@ -20,12 +23,13 @@ DOT    		 = $(GRAPH).dot
 
 # Files grouped by generation mode
 GLYPHS      = luaotfload-glyphlist.lua
+CHARS       = luaotfload-characters.lua
 GRAPHED     = $(DOTPDF)
 MAN			= $(MANPAGE)
 COMPILED    = $(DOC)
 UNPACKED    = luaotfload.sty luaotfload.lua
-GENERATED   = $(GRAPHED) $(UNPACKED) $(COMPILED) $(GLYPHS) $(MAN)
-SOURCE 		= $(DTX) $(MANSOURCE) $(OTFL) README Makefile NEWS $(GLYPHSCRIPT)
+GENERATED   = $(GRAPHED) $(UNPACKED) $(COMPILED) $(GLYPHS) $(CHARS) $(MAN)
+SOURCE 		= $(DTX) $(MANSOURCE) $(OTFL) README Makefile NEWS $(RESOURCESCRIPTS)
 
 # test files
 TESTDIR 		= tests
@@ -34,7 +38,7 @@ TESTFILES_SYS 	= $(TESTDIR)/systemfonts.tex $(TESTDIR)/fontconfig_conf_reading.t
 TESTFILES_TL 	= $(filter-out $(TESTFILES_SYS), $(TESTFILES))
 
 # Files grouped by installation location
-SCRIPTFILES = $(SCRIPT) $(OLDSCRIPT) $(GLYPHSCRIPT)
+SCRIPTFILES = $(SCRIPT) $(OLDSCRIPT) $(RESOURCESCRIPTS)
 RUNFILES    = $(UNPACKED) $(filter-out $(SCRIPTFILES),$(OTFL))
 DOCFILES    = $(DOC) $(DOTPDF) README NEWS
 MANFILES	= $(MANPAGE)
@@ -57,11 +61,14 @@ CTAN_ZIP = $(NAME).zip
 TDS_ZIP  = $(NAME).tds.zip
 ZIPS 	 = $(CTAN_ZIP) $(TDS_ZIP)
 
+LUA	= texlua
+
 DO_TEX 		  	= luatex --interaction=batchmode $< >/dev/null
 # (with the next version of latexmk: -pdf -pdflatex=lualatex)
 DO_LATEX 	  	= latexmk -pdf -e '$$pdflatex = q(lualatex %O %S)' -silent $< >/dev/null
 DO_GRAPHVIZ 	= dot -Tpdf -o $@ $< > /dev/null
-DO_GLYPHLIST 	= texlua ./mkglyphlist > /dev/null
+DO_GLYPHS 		= $(LUA) $(GLYPHSCRIPT) > /dev/null
+DO_CHARS 		= $(LUA) $(CHARSCRIPT)  > /dev/null
 DO_DOCUTILS 	= rst2man $< >$@ 2>/dev/null
 
 all: $(GENERATED)
@@ -69,13 +76,17 @@ graph: $(GRAPHED)
 doc: $(GRAPHED) $(COMPILED) $(MAN)
 manual: $(MAN)
 unpack: $(UNPACKED)
-glyphs: $(GLYPHS)
+resources: $(GLYPHS) $(CHARS)
+chars: $(CHARS)
 ctan: $(CTAN_ZIP)
 tds: $(TDS_ZIP)
 world: all ctan
 
 $(GLYPHS): /dev/null
-	$(DO_GLYPHLIST)
+	$(DO_GLYPHS)
+
+$(CHARS): /dev/null
+	$(DO_CHARS)
 
 $(GRAPHED): $(DOT)
 	$(DO_GRAPHVIZ)
diff --git a/NEWS b/NEWS
index 2b7d4a0..9cb5ad9 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,7 @@
 Change History
 --------------
 
-2013/05/11, luaotfload v2.2:
+2013/05/17, luaotfload v2.2:
     * There is now a central, non-personal dev repo on github:
       https://github.com/lualatex/luaotfload
     * Synchronisation with ConTeXt from TeXLive 2013, inducing
@@ -30,6 +30,7 @@ Change History
     * Unsupported v1.3 codebase is included in the luaotfload-*lua files. Crude,
       but effective way to support older Luatex binaries
     * implemented basic cache control/stats option for luaotfload-tool
+    * Character list is generated from char-def.lua (mkcharacters)
 
 2013/04/27, luaotfload v1.3:
     *  blacklisting lingoes.ttf (segfaults)
diff --git a/luaotfload-auxiliary.lua b/luaotfload-auxiliary.lua
index dde5686..2bfcbf0 100644
--- a/luaotfload-auxiliary.lua
+++ b/luaotfload-auxiliary.lua
@@ -21,11 +21,15 @@ config.luaotfload           = config.luaotfload or { }
 local aux           = luaotfload.aux
 local log           = luaotfload.log
 local warning       = luaotfload.log
-local identifiers   = fonts.hashes.identifiers
+local fonthashes    = fonts.hashes
+local identifiers   = fonthashes.identifiers
 
 local fontid        = font.id
 local texsprint     = tex.sprint
 
+local dofile        = dofile
+local getmetatable  = getmetatable
+local setmetatable  = setmetatable
 local utf8          = unicode.utf8
 local stringlower   = string.lower
 local stringformat  = string.format
@@ -227,9 +231,11 @@ luatexbase.add_to_callback(
   "luaotfload.aux.set_capheight")
 
 -----------------------------------------------------------------------
----                             glyphs
+---                      glyphs and characters
 -----------------------------------------------------------------------
 
+local agl = fonts.encodings.agl
+
 --- int -> int -> bool
 local font_has_glyph = function (font_id, codepoint)
   local fontdata = fonts.hashes.identifiers[font_id]
@@ -241,6 +247,20 @@ end
 
 aux.font_has_glyph = font_has_glyph
 
+--- undocumented
+
+local raw_slot_of_name = function (font_id, glyphname)
+  local fontdata = font.fonts[font_id]
+  if fontdata.type == "virtual" then --- get base font for glyph idx
+    local codepoint  = agl.unicodes[glyphname]
+    local glyph      = fontdata.characters[codepoint]
+    if fontdata.characters[codepoint] then
+      return codepoint
+    end
+  end
+  return false
+end
+
 --[[doc--
 
   This one is approximately “name_to_slot” from the microtype package;
@@ -249,18 +269,28 @@ aux.font_has_glyph = font_has_glyph
 
   http://www.adobe.com/devnet/opentype/archives/glyph.html
 
+  The “unsafe” switch triggers a fallback lookup in the raw fonts
+  table. As some of the information is stored as references, this may
+  have unpredictable side-effects.
+
 --doc]]--
 
---- int -> string -> (int | false)
-local slot_of_name = function (font_id, glyphname)
+--- int -> string -> bool -> (int | false)
+local slot_of_name = function (font_id, glyphname, unsafe)
   local fontdata = identifiers[font_id]
   if fontdata then
     local unicode = fontdata.resources.unicodes[glyphname]
-    if unicode and type(unicode) == "number" then
-      return unicode
-    else
-      return unicode[1] --- for multiple components
+    if unicode then
+      if type(unicode) == "number" then
+        return unicode
+      else
+        return unicode[1] --- for multiple components
+      end
+--  else
+--    --- missing
     end
+  elseif unsafe == true then -- for Robert
+    return raw_slot_of_name(font_id, glyphname)
   end
   return false
 end
@@ -283,7 +313,7 @@ local indices
 --- int -> (string | false)
 local name_of_slot = function (codepoint)
   if not indices then --- this will load the glyph list
-    local unicodes = fonts.encodings.agl.unicodes
+    local unicodes = agl.unicodes
     indices = table.swapped(unicodes)
   end
   local glyphname = indices[codepoint]
@@ -295,6 +325,43 @@ end
 
 aux.name_of_slot      = name_of_slot
 
+--[[doc--
+
+  In Context, characters.data is where the data from char-def.lua
+  resides. The file is huge (>3.7 MB as of 2013) and not part of the
+  isolated font loader. Nevertheless, we include a partial version
+  generated by the mkcharacters script that contains only the
+  “direction” and “mirror” fields of each character defined.
+
+--doc]]--
+
+characters      = characters or { } --- should be created in basics-gen
+characters.data = { }
+local chardef   = "luaotfload-characters"
+
+do
+  local chardata
+  local index = function (t, k)
+    if chardata == nil then
+      log("Loading character metadata from %s.", chardef)
+      chardata = dofile(kpse.find_file("luaotfload-characters.lua"))
+      if chardata == nil then
+        warning("Could not load %s; continuing with empty character table.",
+                chardef)
+        chardata = { }
+      end
+    end
+    return chardata[k]
+  end
+
+  local mt = getmetatable(characters.data)
+  if mt then
+    mt.__index = index
+  else
+    setmetatable(characters.data, { __index = index })
+  end
+end
+
 -----------------------------------------------------------------------
 ---                 features / scripts / languages
 -----------------------------------------------------------------------
@@ -484,6 +551,8 @@ aux.sprint_math_dimension = sprint_math_dimension
 local namesresolve      = fonts.names.resolve
 local namesscan_dir     = fonts.names.scan_dir
 
+--- local directories -------------------------------------------------
+
 --- migrated from luaotfload-database.lua
 --- https://github.com/lualatex/luaotfload/pull/61#issuecomment-17776975
 
@@ -506,6 +575,8 @@ end
 
 aux.scan_external_dir = scan_external_dir
 
+--- db queries --------------------------------------------------------
+
 --- https://github.com/lualatex/luaotfload/issues/74
 --- string -> (string * int)
 local resolve_fontname = function (name)
@@ -542,4 +613,121 @@ end
 
 aux.resolve_fontlist = resolve_fontlist
 
+--- loaded fonts ------------------------------------------------------
+
+--- just a proof of concept
+
+--- fontobj -> string list -> (string list) list
+local get_font_data get_font_data = function (tfmdata, keys, acc, n)
+  if not acc then
+    return get_font_data(tfmdata, keys, {}, 1)
+  end
+  local key = keys[n]
+  if key then
+    local val = tfmdata[key]
+    if val then
+      acc[#acc+1] = val
+    else
+      acc[#acc+1] = false
+    end
+    return get_font_data(tfmdata, keys, acc, n+1)
+  end
+  return acc
+end
+
+--[[doc--
+
+    The next one operates on the fonts.hashes.identifiers table.
+    It returns a list containing tuples of font ids and the
+    contents of the fields specified in the first argument.
+    Font table entries that were created indirectly -- e.g. by
+    \letterspacefont or during font expansion -- will not be
+    listed.
+
+--doc]]--
+
+local default_keys = { "fullname" }
+
+--- string list -> (int * string list) list
+local get_loaded_fonts get_loaded_fonts = function (keys, acc, lastid)
+  if not acc then
+    if not keys then
+      keys = default_keys
+    end
+    return get_loaded_fonts(keys, {}, lastid)
+  end
+  local id, tfmdata = next(identifiers, lastid)
+  if id then
+    local data = get_font_data(tfmdata, keys)
+    acc[#acc+1] = { id, data }
+    return get_loaded_fonts (keys, acc, id)
+  end
+  return acc
+end
+
+aux.get_loaded_fonts = get_loaded_fonts
+
+--- Raw access to the font.* namespace is unsafe so no documentation on
+--- this one.
+local get_raw_fonts = function ( )
+  local res = { }
+  for i, v in font.each() do
+    if v.filename then
+      res[#res+1] = { i, v }
+    end
+  end
+  return res
+end
+
+aux.get_raw_fonts = get_raw_fonts
+
+-----------------------------------------------------------------------
+---                         font parameters
+-----------------------------------------------------------------------
+--- analogy of font-hsh
+
+fonthashes.parameters    = fonthashes.parameters or { }
+fonthashes.quads         = fonthashes.quads or { }
+
+local parameters         = fonthashes.parameters or { }
+local quads              = fonthashes.quads or { }
+
+setmetatable(parameters, { __index = function (t, font_id)
+  local tfmdata = identifiers[font_id]
+  if not tfmdata then --- unsafe; avoid
+    tfmdata = font.fonts[font_id]
+  end
+  if tfmdata and type(tfmdata) == "table" then
+    local fontparameters = tfmdata.parameters
+    t[font_id] = fontparameters
+    return fontparameters
+  end
+  return nil
+end})
+
+--[[doc--
+
+  Note that the reason as to why we prefer functions over table indices
+  is that functions are much safer against unintended manipulation.
+  This justifies the overhead they cost.
+
+--doc]]--
+
+--- int -> (number | false)
+local get_quad = function (font_id)
+  local quad = quads[font_id]
+  if quad then
+    return quad
+  end
+  local fontparameters = parameters[font_id]
+  if fontparameters then
+    local quad     = fontparameters.quad or 0
+    quads[font_id] = quad
+    return quad
+  end
+  return false
+end
+
+aux.get_quad = get_quad
+
 -- vim:tw=71:sw=2:ts=2:expandtab
diff --git a/mkcharacters b/mkcharacters
new file mode 100755
index 0000000..a627ea9
--- /dev/null
+++ b/mkcharacters
@@ -0,0 +1,123 @@
+#!/usr/bin/env texlua
+-----------------------------------------------------------------------
+--         FILE:  mkcharacters.lua
+--        USAGE:  ./mkcharacters.lua 
+--  DESCRIPTION:  import parts of char-def.lua
+-- REQUIREMENTS:  lua, ConTeXt, the lualibs package
+--       AUTHOR:  Philipp Gesang (Phg), <phg42.2a@gmail.com>
+--      VERSION:  1.0
+--      CREATED:  2013-05-17 12:41:39+0200
+-----------------------------------------------------------------------
+-- we create a stripped-down version of char-def.lua
+-----------------------------------------------------------------------
+
+-----------------------------------------------------------------------
+--                              config
+-----------------------------------------------------------------------
+local charfile      = "./luaotfload-characters.lua"
+local chardef       = "~phg/base/char-def.lua"
+local import        = { "direction", "mirror", } --> πολυγλωσσία/uax9
+
+-----------------------------------------------------------------------
+--                             includes
+-----------------------------------------------------------------------
+
+kpse.set_program_name"luatex"
+
+for _, lib in next, { "lualibs-lua.lua",
+                      "lualibs-lpeg.lua",
+                      "lualibs-table.lua", } do
+  local found = assert(kpse.find_file(lib, "lua"),
+                       "Could not locate " .. lib .. ".\n"
+                       .. "Please install the lualibs package.")
+  require(found)
+end
+
+if not (chardef and lfs.isfile(chardef)) then
+  --- we could grab the file from contextgarden but as Context is part
+  --- of TL it’s not worth bothering 
+  chardef = assert(kpse.find_file("char-def.lua", "lua"),
+                   "Could not find ConTeXt.")
+end
+
+-----------------------------------------------------------------------
+--                           functionality
+-----------------------------------------------------------------------
+
+local get_characters = function ( )
+  local data
+  local inchan = io.open(chardef, "r")
+  if not inchan then
+    io.write("Could not open file for reading: "..chardef.."\n.")
+    goto fail
+  end
+  data = inchan:read "*all"
+  inchan:close()
+  data = loadstring(data)
+  if data then
+    data() --> characters.data
+    data = nil
+    collectgarbage "collect"
+    if characters.data and next(characters.data) then
+      return characters.data
+    end
+    io.write "Character table empty.\n"
+    goto fail
+  end
+  io.write(chardef .. " is not a valid Lua file.\n")
+  ::fail::
+  io.write "Emergency exit.\n"
+  os.exit(1)
+end
+
+local extract_fields_indeed
+extract_fields_indeed = function (data, acc, lastidx)
+  local idx, char = next(data, lastidx)
+  if idx then
+    local imported = { }
+    for i=1, #import do
+      local field = import[i]
+      imported[field] = char[field]
+    end
+    acc[idx] = imported
+    return extract_fields_indeed(data, acc, idx)
+  end
+  return acc
+end
+
+local extract_fields = function (data)
+  return extract_fields_indeed(data, {}, nil)
+end
+
+local writedata = function (data)
+  local outchan = io.open(charfile, "w")
+  if not outchan then
+    io.write("Could not open "..charfile.." for writing.\n")
+    return false
+  end
+  outchan:write(data)
+  outchan:close()
+  return true
+end
+
+do
+  local chardata    = get_characters()
+  local stripped    = extract_fields(chardata)
+  local serialized  = table.serialize(stripped, true, {
+    compact   = true,
+    noquotes  = true,
+    hexify    = true, --- for consistency with char-def
+  })
+  if writedata(serialized) then
+    goto done
+  end
+  goto fail
+end
+
+::done::
+  os.exit(0)
+
+::fail::
+  io.write "Emergency exit.\n"
+  os.exit(1)
+