Merge pull request #6 from lualatex/master

Sync with latest lualatex

6 files changed, 308 insertions, 45 deletions

diff --git a/TODO b/TODO
index 6205464..65e80cc 100644
--- a/TODO
+++ b/TODO
@@ -13,7 +13,6 @@ all
     or generalise use of qstest
 - add debug messages to be used in the test suite?
 - mark all occurences of version-dependent stuff in the code
-- quit using module() in favor of locals
 - support the syntax foo = require"foo" in modutils? How?
 
 compat
diff --git a/luatexbase-attr.dtx b/luatexbase-attr.dtx
index ded08b3..aa5e987 100644
--- a/luatexbase-attr.dtx
+++ b/luatexbase-attr.dtx
@@ -116,7 +116,7 @@ See the aforementioned source file(s) for copyright and licensing information.
 % \begin{abstract}
 % In addition to the registers existing in \tex and \etex, \luatex introduces
 % a new concept: attributes. This package takes care of attribute allocation
-% just like Plain TeX and LaTeX do for other registers, and also provides a
+% just like Plain \tex and \latex do for other registers, and also provides a
 % Lua interface.
 % \end{abstract}
 %
@@ -283,7 +283,7 @@ See the aforementioned source file(s) for copyright and licensing information.
 %    \subsubsection{Load supporting Lua module}
 %
 %    First load \pk{luatexbase-loader} (hence \pk{luatexbase-compat}), then
-%    the supporting Lua module. We make sure luatex.sty is loaded.
+%    the supporting Lua module. We make sure \verb|luatex.sty| is loaded.
 %
 %    \begin{macrocode}
 \begingroup\expandafter\expandafter\expandafter\endgroup
@@ -299,7 +299,7 @@ See the aforementioned source file(s) for copyright and licensing information.
 %
 %    \subsection{User macros}
 %
-%    The allocaton macro is merely a wrapper around the Lua function, but
+%    The allocation macro is merely a wrapper around the Lua function, but
 %    handles error and logging in \tex, for consistency with other allocation
 %    macros.
 %
@@ -344,22 +344,63 @@ See the aforementioned source file(s) for copyright and licensing information.
 %
 %    \begin{macrocode}
 %<*luamodule>
-module('luatexbase', package.seeall)
+--- locals
+local copynode          = node.copy
+local newnode           = node.new
+local nodesubtype       = node.subtype
+local nodetype          = node.id
+local stringfind        = string.find
+local stringformat      = string.format
+local tableunpack       = unpack or table.unpack
+local texiowrite_nl     = texio.write_nl
+local texiowrite        = texio.write
+--- luatex internal types
+local whatsit_t         = nodetype"whatsit"
+local user_defined_t    = nodesubtype"user_defined"
+local unassociated      = "__unassociated"
+luatexbase              = luatexbase or { }
+local luatexbase        = luatexbase
+%    \end{macrocode}
+%
+%    We improvise a basic logging facility.
+%
+%    \begin{macrocode}
+local reporter = function (log, category, ...)
+    if log == true then
+        texiowrite_nl("log", "("..category..") ")
+        texiowrite("log", stringformat(...))
+    else
+        texiowrite_nl("("..category..") ")
+        texiowrite(stringformat(...))
+    end
+end
+local warning = function (...) reporter (false, "warning", ...) end
+----- info    = function (...) reporter (false, "info",    ...) end
+local log     = function (...) reporter (true,  "log",     ...) end
 %    \end{macrocode}
 %
 %    This table holds the values of the allocated attributes, indexed by name.
 %
 %    \begin{macrocode}
-attributes = {}
+luatexbase.attributes   = luatexbase.attributes or { }
+local attributes        = luatexbase.attributes
+%    \end{macrocode}
+%
+%    Scoping: we use locals for the attribute functions.
+%
+%    \begin{macrocode}
+local new_attribute
+local unset_attribute
 %    \end{macrocode}
 %
-%    There are currently two functions that create a new attribute.One is in
-%    |oberdiek| bundle, the other is this one. We will hack a little in order
-%    to make them compatible. The other function uses |LuT@AllocAttribute| as
-%    attribute counter, we will keep it in sync with ours. A possible problem
-%    might also appear: the other function starts attribute allocation at 0,
-%    which might break luaotfload. We output an error if a new attribute has
-%    already been allocated with number 0.
+%    In the \luatex ecosystem there are currently two functions that create a
+%    new attribute.
+%    One is in |oberdiek| bundle, the other is this one. We will hack a little
+%    in order to make them compatible. The other function uses
+%    |LuT@AllocAttribute| as attribute counter, we will keep it in sync with
+%    ours. A possible problem might also appear: the other function starts
+%    attribute allocation at 0, which will break luaotfload. We output an
+%    error if a new attribute has already been allocated with number 0.
 %
 %    \begin{macrocode}
 local luatex_sty_counter = 'LuT@AllocAttribute'
@@ -367,17 +408,17 @@ if tex.count[luatex_sty_counter] then
   if tex.count[luatex_sty_counter] > -1 then
     error("luatexbase error: attribute 0 has already been set by \newattribute"
         .."macro from luatex.sty, not belonging to this package, this makes"
-        .."luaotfload unuseable. Please report to the maintainer of luatex.sty")
+        .."luaotfload unusable. Please report to the maintainer of luatex.sty")
   else
     tex.count[luatex_sty_counter] = 0
   end
 end
 %    \end{macrocode}
 %
-%    The allocaton function. Unlike other registers, allocate starting from 1.
-%    Some code (eg, font handling coming from Con\tex{}t) behaves strangely
-%    with \verb+\attribute0+ and since there is plenty of room here, it
-%    doesn't seem bad to ``loose'' one item in order to avoid this problem.
+%    The allocation function. Unlike other registers, allocate starting from 1.
+%    Some code (e.~g., font handling coming from Con\tex{}t) behaves strangely
+%    with \verb+\attribute0+ set, and since there is plenty of room here, it
+%    doesn't seem bad to ``lose'' one item in order to avoid this problem.
 %
 %    \begin{macrocode}
 local last_alloc = 0
@@ -400,11 +441,11 @@ function new_attribute(name, silent)
     attributes[name] = last_alloc
     unset_attribute(name)
     if not silent then
-        texio.write_nl('log', string.format(
-            'luatexbase.attributes[%q] = %d', name, last_alloc))
+        log('luatexbase.attributes[%q] = %d', name, last_alloc)
     end
     return last_alloc
 end
+luatexbase.new_attribute = new_attribute
 %    \end{macrocode}
 %
 %    Unset an attribute the correct way depending on \luatex's version.
@@ -414,6 +455,197 @@ local unset_value = (luatexbase.luatexversion < 37) and -1 or -2147483647
 function unset_attribute(name)
     tex.setattribute(attributes[name], unset_value)
 end
+luatexbase.unset_attribute = unset_attribute
+%    \end{macrocode}
+%
+%    User whatsit allocation (experimental).
+%
+%    \begin{macrocode}
+--- cf. luatexref-t.pdf, sect. 8.1.4.25
+local user_whatsits       = {    --- (package, (name, id hash)) hash
+    __unassociated = { },        --- those without package name
+}
+local whatsit_ids         = { }  --- (id, (name * package)) hash
+local whatsit_cap         = 2^53 --- Lua numbers are doubles
+local current_whatsit     = 0
+local anonymous_whatsits  = 0
+local anonymous_prefix    = "anon"
+%    \end{macrocode}
+%
+%    The whatsit allocation is split into two functions:
+%    \verb|new_user_whatsit_id| registers a new id (an integer)
+%    and returns it. It is up to the user what he actually does
+%    with the return value.
+%
+%    Registering whatsits without a name, though supported, is
+%    not exactly good style. In these cases we generate a name
+%    from a counter.
+%
+%    In addition to the whatsit name, it is possible and even
+%    encouraged to specify the name of the package that will be
+%    using the whatsit as the second argument.
+%
+%    \begin{macrocode}
+--- string -> string -> int
+local new_user_whatsit_id = function (name, package)
+    if name then
+        if not package then
+            package = unassociated
+        end
+    else -- anonymous
+        anonymous_whatsits = anonymous_whatsits + 1
+        warning("defining anonymous user whatsit no. %d", anonymous_whatsits)
+        warning("dear package authors, please name your whatsits!")
+        package = unassociated
+        name    = anonymous_prefix .. tostring(anonymous_whatsits)
+    end
+
+    local whatsitdata = user_whatsits[package]
+    if not whatsitdata then
+        whatsitdata             = { }
+        user_whatsits[package]  = whatsitdata
+    end
+
+    local id = whatsitdata[name]
+    if id then --- warning
+        warning("replacing whatsit %s:%s (%d)", package, name, id)
+    else --- new id
+        current_whatsit     = current_whatsit + 1
+        if current_whatsit >= whatsit_cap then
+            warning("maximum of %d integral user whatsit ids reached",
+                whatsit_cap)
+            warning("further whatsit allocation may be inconsistent")
+        end
+        id                  = current_whatsit
+        whatsitdata[name]   = id
+        whatsit_ids[id]     = { name, package }
+    end
+    log("new user-defined whatsit %d (%s:%s)", id, package, name)
+    return id
+end
+luatexbase.new_user_whatsit_id = new_user_whatsit_id
+%    \end{macrocode}
+%
+%    \verb|new_user_whatsit| first registers a new id and then also
+%    creates the corresponding whatsit of subtype “user defined”.
+%    We return a nullary function that delivers copies of the whatsit.
+%
+%    \begin{macrocode}
+--- string -> string -> (unit -> node_t, int)
+local new_user_whatsit = function (name, package)
+    local id         = new_user_whatsit_id(name, package)
+    local whatsit    = newnode(whatsit_t, user_defined_t)
+    whatsit.user_id  = id
+    --- unit -> node_t
+    return function ( ) return copynode(whatsit) end, id
+end
+luatexbase.new_user_whatsit         = new_user_whatsit
+luatexbase.new_user_whatsit_factory = new_user_whatsit --- for Stephan
+%    \end{macrocode}
+%
+%    If one knows the name of a whatsit, its corresponding id
+%    can be retrieved by means of \verb|get_user_whatsit_id|.
+%
+%    \begin{macrocode}
+--- string -> string -> int
+local get_user_whatsit_id = function (name, package)
+    if not package then
+        package = unassociated
+    end
+    return user_whatsits[package][name]
+end
+luatexbase.get_user_whatsit_id = get_user_whatsit_id
+%    \end{macrocode}
+%
+%    The inverse lookup is also possible via \verb|get_user_whatsit_name|.
+%    Here it finally becomes obvious why it is beneficial to supply a package
+%    name -- it adds information about who created and might be relying on the
+%    whatsit in question. First return value is the whatsit name, the second
+%    the package identifier it was registered with.
+%
+%    We issue a warning and return empty strings in case the asked whatsit is
+%    unregistered.
+%
+%    \begin{macrocode}
+--- int | fun | node -> (string, string)
+local get_user_whatsit_name = function (asked)
+    local id
+    if type(asked) == "number" then
+        id = asked
+    elseif type(asked) == "function" then
+        --- node generator
+        local n = asked()
+        id = n.user_id
+    else --- node
+        id = asked.user_id
+    end
+    local metadata = whatsit_ids[id]
+    if not metadata then -- unknown
+        warning("whatsit id %d unregistered; inconsistencies may arise", id)
+        return "", ""
+    end
+    return tableunpack(metadata)
+end
+luatexbase.get_user_whatsit_name = get_user_whatsit_name
+%    \end{macrocode}
+%
+%    For the curious as well as the cautious who are interesting in
+%    what they are dealing with, we add a function that outputs the
+%    current allocation status to the terminal.
+%
+%    \begin{macrocode}
+--- string -> unit
+local dump_registered_whatsits = function (asked_package)
+    local whatsit_list = { }
+    if asked_package then
+        local whatsitdata = user_whatsits[asked_package]
+        if not whatsitdata then
+            error("(no user whatsits registered for package %s)",
+                  asked_package)
+            return
+        end
+        texiowrite_nl("(user whatsit allocation stats for " .. asked_package)
+        for name, id in next, whatsitdata do
+            whatsit_list[#whatsit_list+1] =
+                stringformat("(%s:%s %d)", asked_package, name, id)
+        end
+    else
+        texiowrite_nl("(user whatsit allocation stats")
+        texiowrite_nl(stringformat(" ((total %d)\n  (anonymous %d))",
+            current_whatsit, anonymous_whatsits))
+        for package, whatsitdata in next, user_whatsits do
+            for name, id in next, whatsitdata do
+                whatsit_list[#whatsit_list+1] =
+                    stringformat("(%s:%s %d)", package, name, id)
+            end
+        end
+    end
+
+    texiowrite_nl" ("
+    --- in an attempt to be clever the texio.write* functions
+    --- mess up line breaking, so concatenation is unusable ...
+    local first = true
+    for i=1, #whatsit_list do
+        if first then 
+            first = false
+        else -- indent
+            texiowrite_nl"  "
+        end
+        texiowrite(whatsit_list[i])
+    end
+    texiowrite"))\n"
+end
+luatexbase.dump_registered_whatsits = dump_registered_whatsits
+%    \end{macrocode}
+%    Lastly, we define a couple synonyms for convenience.
+%    \begin{macrocode}
+luatexbase.newattribute            = new_attribute
+luatexbase.newuserwhatsit          = new_user_whatsit
+luatexbase.newuserwhatsitfactory   = new_user_whatsit_factory
+luatexbase.newuserwhatsitid        = new_user_whatsit_id
+luatexbase.getuserwhatsitid        = get_user_whatsit_id
+luatexbase.getuserwhatsitname      = get_user_whatsit_name
+luatexbase.dumpregisteredwhatsits  = dump_registered_whatsits
 %    \end{macrocode}
 %
 %    \begin{macrocode}
diff --git a/luatexbase-cctb.dtx b/luatexbase-cctb.dtx
index f869c5b..1e95379 100644
--- a/luatexbase-cctb.dtx
+++ b/luatexbase-cctb.dtx
@@ -649,17 +649,20 @@ See the aforementioned source file(s) for copyright and licensing information.
 %
 %    \begin{macrocode}
 %<*luamodule>
-module('luatexbase', package.seeall)
+luatexbase                  = luatexbase or { }
+local luatexbase            = luatexbase
 %    \end{macrocode}
 %
 %    The number associated to a CS name is remembered in the |catcodetables|
 %    table.
 %
 %    \begin{macrocode}
-catcodetables = {}
-function catcodetabledef_from_tex(name, number)
+luatexbase.catcodetables    = luatexbase.catcodetables or { }
+local catcodetables         = luatexbase.catcodetables
+local function catcodetabledef_from_tex(name, number)
     catcodetables[name] = tonumber(number)
 end
+luatexbase.catcodetabledef_from_tex = catcodetabledef_from_tex
 %    \end{macrocode}
 %
 %    The next function creates some shortcuts for better readability in lua
@@ -667,7 +670,7 @@ end
 %    |luatexbase.catcodetables.CatcodeTableLaTeX|.
 %
 %    \begin{macrocode}
-function catcodetable_do_shortcuts()
+local function catcodetable_do_shortcuts()
     local cat = catcodetables
     cat['latex']                = cat.CatcodeTableLaTeX
     cat['latex-package']        = cat.CatcodeTableLaTeXAtLetter
@@ -678,6 +681,7 @@ function catcodetable_do_shortcuts()
     cat['string']               = cat.CatcodeTableString
     cat['other']                = cat.CatcodeTableOther
 end
+luatexbase.catcodetable_do_shortcuts = catcodetable_do_shortcuts
 %    \end{macrocode}
 %
 %    \begin{macrocode}
diff --git a/luatexbase-loader.dtx b/luatexbase-loader.dtx
index b8e5098..11918a3 100644
--- a/luatexbase-loader.dtx
+++ b/luatexbase-loader.dtx
@@ -274,7 +274,8 @@ See the aforementioned source file(s) for copyright and licensing information.
 %
 %    \begin{macrocode}
 %<*luamodule>
-module('luatexbase', package.seeall)
+luatexbase          = luatexbase or { }
+local luatexbase    = luatexbase
 %    \end{macrocode}
 %
 %    Just in case it's called from a \TeX Lua script...
diff --git a/luatexbase-mcb.dtx b/luatexbase-mcb.dtx
index f4d6448..0c04d4d 100644
--- a/luatexbase-mcb.dtx
+++ b/luatexbase-mcb.dtx
@@ -392,7 +392,8 @@ See the aforementioned source file(s) for copyright and licensing information.
 %    \subsubsection{Module identification}
 %
 %    \begin{macrocode}
-module('luatexbase', package.seeall)
+luatexbase          = luatexbase or { }
+local luatexbase    = luatexbase
 local err, warning, info, log = luatexbase.provides_module({
     name          = "luatexbase-mcb",
     version       = 0.5,
@@ -404,6 +405,17 @@ local err, warning, info, log = luatexbase.provides_module({
 })
 %    \end{macrocode}
 %
+%    First we declare the function references for the entire scope.
+%
+%    \begin{macrocode}
+local add_to_callback
+local call_callback
+local create_callback
+local priority_in_callback
+local remove_from_callback
+local reset_callback
+%    \end{macrocode}
+%
 %    \subsubsection{Housekeeping}
 %
 %    The main table: keys are callback names, and values are the associated
@@ -690,6 +702,7 @@ function add_to_callback (name,func,description,priority)
     log("inserting '%s'\nat position %s in '%s'",
       description, priority, name)
 end
+luatexbase.add_to_callback = add_to_callback
 %    \end{macrocode}
 %
 %    Remove a function from a callback. First check arguments.
@@ -742,6 +755,7 @@ function remove_from_callback (name, description)
     end
     return
 end
+luatexbase.remove_from_callback = remove_from_callback
 %    \end{macrocode}
 %
 %    Remove all the functions registered in a callback. Unregisters the
@@ -767,6 +781,7 @@ function reset_callback (name, make_false)
         end
     end
 end
+luatexbase.reset_callback = reset_callback
 %    \end{macrocode}
 %
 %    Get a function's priority in a callback list, or false if the function is
@@ -788,6 +803,7 @@ function priority_in_callback (name, description)
     end
     return false
 end
+luatexbase.priority_in_callback = priority_in_callback
 %    \end{macrocode}
 %
 %    \subsubsection{Public functions for user-defined callbacks}
@@ -827,6 +843,7 @@ function create_callback(name, ctype, default)
     lua_callbacks_defaults[name] = default
     callbacktypes[name] = ctype
 end
+luatexbase.create_callback = create_callback
 %    \end{macrocode}
 %
 %    This function calls a callback. It can only call a callback created by
@@ -855,6 +872,7 @@ function call_callback(name, ...)
     end
     return f(...)
 end
+luatexbase.call_callback = call_callback
 %    \end{macrocode}
 %
 %    That's all folks!
diff --git a/luatexbase-modutils.dtx b/luatexbase-modutils.dtx
index 6f500da..0645e1b 100644
--- a/luatexbase-modutils.dtx
+++ b/luatexbase-modutils.dtx
@@ -237,22 +237,23 @@ See the aforementioned source file(s) for copyright and licensing information.
 % they just don't do the same thing (declaring information vs changing the
 % current name space).
 %
-% Now, here is how you module may begin:
+% Now, here is a module header template showing all the recommended elements:
 % \begin{verbatim}
 % local err, warn, info, log = luatexbase.provides_module({
 %     -- required
-%     name        = 'mymodule',
+%     name          = 'mymodule',
 %     -- recommended
-%     date        = '1970/01/01',
-%     version     = 0.0,            -- or version = '0.0a',
-%     description = 'a Lua module template',
+%     date          = '1970/01/01',
+%     version       = 0.0,            -- or version = '0.0a',
+%     description   = 'a Lua module template',
 %     -- optional and ignored
-%     author      = 'A. U. Thor',
-%     licence     = 'LPPL v1.3+',
+%     author        = 'A. U. Thor',
+%     licence       = 'LPPL v1.3+',
 % })
 %
-% module('mynamespace', package.seeall)
-% -- or any other method (see chapter 15 of PIL for examples)
+% mynamespace           = mynamespace or { }
+% local mynamespace     = mynamespace
+%
 % \end{verbatim}
 %
 % Alternatively, if you don't want to assume \pk{luatexbase-modutils} is
@@ -276,8 +277,8 @@ See the aforementioned source file(s) for copyright and licensing information.
 %     })
 % end
 %
-% module('mynamespace', package.seeall)
-% -- or any other method (see chapter 15 of PIL for examples)
+% mynamespace           = mynamespace or { }
+% local mynamespace     = mynamespace
 %
 % local function err(msg)
 %    -- etc.
@@ -457,7 +458,8 @@ See the aforementioned source file(s) for copyright and licensing information.
 %
 %    \begin{macrocode}
 %<*luamodule>
-module("luatexbase", package.seeall)
+luatexbase          = luatexbase or { }
+local luatexbase    = luatexbase
 %    \end{macrocode}
 %
 %    \subsection{Internal functions and data}
@@ -495,44 +497,49 @@ end
 local function module_error_int(mod, ...)
   error(msg_format('error', mod, ...), 3)
 end
-function module_error(mod, ...)
+local function module_error(mod, ...)
   module_error_int(mod, ...)
 end
+luatexbase.module_error = module_error
 %    \end{macrocode}
 %
 %    Split the lines explicitly in order not to depend on the value of
 %    |\newlinechar|.
 %
 %    \begin{macrocode}
-function module_warning(mod, ...)
+local function module_warning(mod, ...)
   for _, line in ipairs(msg_format('warning', mod, ...):explode('\n')) do
     texio.write_nl(line)
   end
 end
-function module_info(mod, ...)
+luatexbase.module_warning = module_warning
+local function module_info(mod, ...)
   for _, line in ipairs(msg_format('info', mod, ...):explode('\n')) do
     texio.write_nl(line)
   end
 end
+luatexbase.module_info = module_info
 %    \end{macrocode}
 %
 %    No line splitting or advanced formating here.
 %
 %    \begin{macrocode}
-function module_log(mod, msg, ...)
+local function module_log(mod, msg, ...)
   texio.write_nl('log', mod..': '..msg:format(...))
 end
+luatexbase.module_log = module_log
 %    \end{macrocode}
 %
 %    Produce custom versions of the reporting functions.
 %
 %    \begin{macrocode}
-function errwarinf(name)
+local function errwarinf(name)
   return function(...) module_error_int(name, ...) end,
     function(...) module_warning(name, ...) end,
     function(...) module_info(name, ...) end,
     function(...) module_log(name, ...) end
 end
+luatexbase.errwarinf = errwarinf
 %    \end{macrocode}
 %
 %    For our own convenience, local functions for warning and errors in the
@@ -547,7 +554,7 @@ local err, warn = errwarinf('luatexbase.modutils')
 %    Load a module with mandatory name checking and optional version checking.
 %
 %    \begin{macrocode}
-function require_module(name, req_date)
+local function require_module(name, req_date)
     require(name)
     local info = modules[name]
     if not info then
@@ -560,6 +567,7 @@ function require_module(name, req_date)
         end
     end
 end
+luatexbase.require_module = require_module
 %    \end{macrocode}
 %
 %    Provide identification information for a module. As a bonus, custom
@@ -567,7 +575,7 @@ end
 %    everything done in |require_module()|.
 %
 %    \begin{macrocode}
-function provides_module(info)
+local function provides_module(info)
     if not (info and info.name) then
         err('provides_module: missing information')
     end
@@ -576,6 +584,7 @@ function provides_module(info)
     modules[info.name] = info
     return errwarinf(info.name)
 end
+luatexbase.provides_module = provides_module
 %    \end{macrocode}
 %
 %    \begin{macrocode}