update util-lua

1 files changed, 288 insertions, 171 deletions

diff --git a/lualibs-util-lua.lua b/lualibs-util-lua.lua
index 2baeaa8..f3be9dc 100644
--- a/lualibs-util-lua.lua
+++ b/lualibs-util-lua.lua
@@ -7,14 +7,16 @@ if not modules then modules = { } end modules ['util-lua'] = {
     license   = "see context related readme files"
 }
 
+-- we will remove the 5.1 code some day soon
+
 local rep, sub, byte, dump, format = string.rep, string.sub, string.byte, string.dump, string.format
-local loadstring, loadfile, type = loadstring, loadfile, type
+local load, loadfile, type = load, loadfile, type
 
 utilities          = utilities or {}
 utilities.lua      = utilities.lua or { }
 local luautilities = utilities.lua
 
-utilities.report   = logs and logs.reporter("system") or print -- can be overloaded later
+local report_lua = logs.reporter("system","lua")
 
 local tracestripping           = false
 local forcestupidcompile       = true  -- use internal bytecode compiler
@@ -22,213 +24,328 @@ luautilities.stripcode         = true  -- support stripping when asked for
 luautilities.alwaysstripcode   = false -- saves 1 meg on 7 meg compressed format file (2012.08.12)
 luautilities.nofstrippedchunks = 0
 luautilities.nofstrippedbytes  = 0
+local strippedchunks           = { } -- allocate()
+luautilities.strippedchunks    = strippedchunks
 
--- The next function was posted by Peter Cawley on the lua list and strips line
--- number information etc. from the bytecode data blob. We only apply this trick
--- when we store data tables. Stripping makes the compressed format file about
--- 1MB smaller (and uncompressed we save at least 6MB).
---
--- You can consider this feature an experiment, so it might disappear. There is
--- no noticeable gain in runtime although the memory footprint should be somewhat
--- smaller (and the file system has a bit less to deal with).
---
--- Begin of borrowed code ... works for Lua 5.1 which LuaTeX currently uses ...
-
-local function strip_code_pc(dump,name)
-    local before = #dump
-    local version, format, endian, int, size, ins, num = byte(dump,5,11)
-    local subint
-    if endian == 1 then
-        subint = function(dump, i, l)
-            local val = 0
-            for n = l, 1, -1 do
-                val = val * 256 + byte(dump,i + n - 1)
-            end
-            return val, i + l
+luautilities.suffixes = {
+    tma = "tma",
+    tmc = jit and "tmb" or "tmc",
+    lua = "lua",
+    luc = jit and "lub" or "luc",
+    lui = "lui",
+    luv = "luv",
+    luj = "luj",
+    tua = "tua",
+    tuc = "tuc",
+}
+
+-- environment.loadpreprocessedfile can be set to a preprocessor
+
+if jit or status.luatex_version >= 74 then
+
+    local function register(name)
+        if tracestripping then
+            report_lua("stripped bytecode from %a",name or "unknown")
         end
-    else
-        subint = function(dump, i, l)
-            local val = 0
-            for n = 1, l, 1 do
-                val = val * 256 + byte(dump,i + n - 1)
+        strippedchunks[#strippedchunks+1] = name
+        luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1
+    end
+
+    local function stupidcompile(luafile,lucfile,strip)
+        local code = io.loaddata(luafile)
+        if code and code ~= "" then
+            code = load(code)
+            if code then
+                code = dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode)
+                if code and code ~= "" then
+                    register(name)
+                    io.savedata(lucfile,code)
+                    return true, 0
+                end
+            else
+                report_lua("fatal error %a in file %a",1,luafile)
             end
-            return val, i + l
+        else
+            report_lua("fatal error %a in file %a",2,luafile)
         end
+        return false, 0
     end
-    local strip_function
-    strip_function = function(dump)
-        local count, offset = subint(dump, 1, size)
-        local stripped, dirty = rep("\0", size), offset + count
-        offset = offset + count + int * 2 + 4
-        offset = offset + int + subint(dump, offset, int) * ins
-        count, offset = subint(dump, offset, int)
-        for n = 1, count do
-            local t
-            t, offset = subint(dump, offset, 1)
-            if t == 1 then
-                offset = offset + 1
-            elseif t == 4 then
-                offset = offset + size + subint(dump, offset, size)
-            elseif t == 3 then
-                offset = offset + num
-            end
+
+    -- quite subtle ... doing this wrong incidentally can give more bytes
+
+    function luautilities.loadedluacode(fullname,forcestrip,name)
+        -- quite subtle ... doing this wrong incidentally can give more bytes
+        name = name or fullname
+        local code = environment.loadpreprocessedfile and environment.loadpreprocessedfile(fullname) or loadfile(fullname)
+        if code then
+            code()
         end
-        count, offset = subint(dump, offset, int)
-        stripped = stripped .. sub(dump,dirty, offset - 1)
-        for n = 1, count do
-            local proto, off = strip_function(sub(dump,offset, -1))
-            stripped, offset = stripped .. proto, offset + off - 1
+        if forcestrip and luautilities.stripcode then
+            if type(forcestrip) == "function" then
+                forcestrip = forcestrip(fullname)
+            end
+            if forcestrip or luautilities.alwaysstripcode then
+                register(name)
+                return load(dump(code,true)), 0
+            else
+                return code, 0
+            end
+        elseif luautilities.alwaysstripcode then
+            register(name)
+            return load(dump(code,true)), 0
+        else
+            return code, 0
         end
-        offset = offset + subint(dump, offset, int) * int + int
-        count, offset = subint(dump, offset, int)
-        for n = 1, count do
-            offset = offset + subint(dump, offset, size) + size + int * 2
+    end
+
+    function luautilities.strippedloadstring(code,forcestrip,name) -- not executed
+        if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then
+            code = load(code)
+            if not code then
+                report_lua("fatal error %a in file %a",3,name)
+            end
+            register(name)
+            code = dump(code,true)
         end
-        count, offset = subint(dump, offset, int)
-        for n = 1, count do
-            offset = offset + subint(dump, offset, size) + size
+        return load(code), 0
+    end
+
+    function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true
+        report_lua("compiling %a into %a",luafile,lucfile)
+        os.remove(lucfile)
+        local done = stupidcompile(luafile,lucfile,strip ~= false)
+        if done then
+            report_lua("dumping %a into %a stripped",luafile,lucfile)
+            if cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+                report_lua("removing %a",luafile)
+                os.remove(luafile)
+            end
         end
-        stripped = stripped .. rep("\0", int * 3)
-        return stripped, offset
+        return done
     end
-    dump = sub(dump,1,12) .. strip_function(sub(dump,13,-1))
-    local after = #dump
-    local delta = before-after
-    if tracestripping then
-        utilities.report("stripped bytecode: %s, before %s, after %s, delta %s",name or "unknown",before,after,delta)
+
+    function luautilities.loadstripped(...)
+        local l = load(...)
+        if l then
+            return load(dump(l,true))
+        end
     end
-    luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1
-    luautilities.nofstrippedbytes  = luautilities.nofstrippedbytes  + delta
-    return dump, delta
-end
 
--- ... end of borrowed code.
+else
 
-local function strippedbytecode(code,forcestrip,name)
-    if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then
-        return strip_code_pc(code,name)
-    else
-        return code, 0
+    -- The next function was posted by Peter Cawley on the lua list and strips line
+    -- number information etc. from the bytecode data blob. We only apply this trick
+    -- when we store data tables. Stripping makes the compressed format file about
+    -- 1MB smaller (and uncompressed we save at least 6MB).
+    --
+    -- You can consider this feature an experiment, so it might disappear. There is
+    -- no noticeable gain in runtime although the memory footprint should be somewhat
+    -- smaller (and the file system has a bit less to deal with).
+    --
+    -- Begin of borrowed code ... works for Lua 5.1 which LuaTeX currently uses ...
+
+    local function register(name,before,after)
+        local delta = before - after
+        if tracestripping then
+            report_lua("bytecodes stripped from %a, # before %s, # after %s, delta %s",name,before,after,delta)
+        end
+        strippedchunks[#strippedchunks+1] = name
+        luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1
+        luautilities.nofstrippedbytes  = luautilities.nofstrippedbytes  + delta
+        return delta
     end
-end
 
-luautilities.stripbytecode    = strip_code_pc
-luautilities.strippedbytecode = strippedbytecode
+    local strip_code_pc
 
-local function fatalerror(name)
-    utilities.report(format("fatal error in %q",name or "unknown"))
-end
+    if _MAJORVERSION == 5 and _MINORVERSION == 1 then
+
+        strip_code_pc = function(dump,name)
+            local before = #dump
+            local version, format, endian, int, size, ins, num = byte(dump,5,11)
+            local subint
+            if endian == 1 then
+                subint = function(dump, i, l)
+                    local val = 0
+                    for n = l, 1, -1 do
+                        val = val * 256 + byte(dump,i + n - 1)
+                    end
+                    return val, i + l
+                end
+            else
+                subint = function(dump, i, l)
+                    local val = 0
+                    for n = 1, l, 1 do
+                        val = val * 256 + byte(dump,i + n - 1)
+                    end
+                    return val, i + l
+                end
+            end
+            local strip_function
+            strip_function = function(dump)
+                local count, offset = subint(dump, 1, size)
+                local stripped, dirty = rep("\0", size), offset + count
+                offset = offset + count + int * 2 + 4
+                offset = offset + int + subint(dump, offset, int) * ins
+                count, offset = subint(dump, offset, int)
+                for n = 1, count do
+                    local t
+                    t, offset = subint(dump, offset, 1)
+                    if t == 1 then
+                        offset = offset + 1
+                    elseif t == 4 then
+                        offset = offset + size + subint(dump, offset, size)
+                    elseif t == 3 then
+                        offset = offset + num
+                    end
+                end
+                count, offset = subint(dump, offset, int)
+                stripped = stripped .. sub(dump,dirty, offset - 1)
+                for n = 1, count do
+                    local proto, off = strip_function(sub(dump,offset, -1))
+                    stripped, offset = stripped .. proto, offset + off - 1
+                end
+                offset = offset + subint(dump, offset, int) * int + int
+                count, offset = subint(dump, offset, int)
+                for n = 1, count do
+                    offset = offset + subint(dump, offset, size) + size + int * 2
+                end
+                count, offset = subint(dump, offset, int)
+                for n = 1, count do
+                    offset = offset + subint(dump, offset, size) + size
+                end
+                stripped = stripped .. rep("\0", int * 3)
+                return stripped, offset
+            end
+            dump = sub(dump,1,12) .. strip_function(sub(dump,13,-1))
+            local after = #dump
+            local delta = register(name,before,after)
+            return dump, delta
+        end
 
--- quite subtle ... doing this wrong incidentally can give more bytes
+    else
 
+        strip_code_pc = function(dump,name)
+            return dump, 0
+        end
 
-function luautilities.loadedluacode(fullname,forcestrip,name)
-    -- quite subtle ... doing this wrong incidentally can give more bytes
-    name = name or fullname
-    local code = loadfile(fullname)
-    if code then
-        code()
     end
-    if forcestrip and luautilities.stripcode then
-        if type(forcestrip) == "function" then
-            forcestrip = forcestrip(fullname)
+
+    -- ... end of borrowed code.
+
+    -- quite subtle ... doing this wrong incidentally can give more bytes
+
+    function luautilities.loadedluacode(fullname,forcestrip,name)
+        -- quite subtle ... doing this wrong incidentally can give more bytes
+        local code = environment.loadpreprocessedfile and environment.preprocessedloadfile(fullname) or loadfile(fullname)
+        if code then
+            code()
         end
-        if forcestrip then
-            local code, n = strip_code_pc(dump(code,name))
-            return loadstring(code), n
+        if forcestrip and luautilities.stripcode then
+            if type(forcestrip) == "function" then
+                forcestrip = forcestrip(fullname)
+            end
+            if forcestrip then
+                local code, n = strip_code_pc(dump(code),name)
+                return load(code), n
+            elseif luautilities.alwaysstripcode then
+                return load(strip_code_pc(dump(code),name))
+            else
+                return code, 0
+            end
         elseif luautilities.alwaysstripcode then
-            return loadstring(strip_code_pc(dump(code),name))
+            return load(strip_code_pc(dump(code),name))
         else
             return code, 0
         end
-    elseif luautilities.alwaysstripcode then
-        return loadstring(strip_code_pc(dump(code),name))
-    else
-        return code, 0
     end
-end
 
-function luautilities.strippedloadstring(code,forcestrip,name) -- not executed
-    local n = 0
-    if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then
-        code = loadstring(code)
-        if not code then
-            fatalerror(name)
+    function luautilities.strippedloadstring(code,forcestrip,name) -- not executed
+        local n = 0
+        if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then
+            code = load(code)
+            if not code then
+                report_lua("fatal error in file %a",name)
+            end
+            code, n = strip_code_pc(dump(code),name)
         end
-        code, n = strip_code_pc(dump(code),name)
+        return load(code), n
     end
-    return loadstring(code), n
-end
 
-local function stupidcompile(luafile,lucfile,strip)
-    local code = io.loaddata(luafile)
-    local n = 0
-    if code and code ~= "" then
-        code = loadstring(code)
-        if not code then
-            fatalerror()
-        end
-        code = dump(code)
-        if strip then
-            code, n = strippedbytecode(code,true,luafile) -- last one is reported
-        end
+    local function stupidcompile(luafile,lucfile,strip)
+        local code = io.loaddata(luafile)
+        local n = 0
         if code and code ~= "" then
-            io.savedata(lucfile,code)
+            code = load(code)
+            if not code then
+                report_lua("fatal error in file %a",luafile)
+            end
+            code = dump(code)
+            if strip then
+                code, n = strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) -- last one is reported
+            end
+            if code and code ~= "" then
+                io.savedata(lucfile,code)
+            end
         end
+        return n
     end
-    return n
-end
 
-local luac_normal = "texluac -o %q %q"
-local luac_strip  = "texluac -s -o %q %q"
+    local luac_normal = "texluac -o %q %q"
+    local luac_strip  = "texluac -s -o %q %q"
 
-function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true
-    utilities.report("lua: compiling %s into %s",luafile,lucfile)
-    os.remove(lucfile)
-    local done = false
-    if strip ~= false then
-        strip = true
-    end
-    if forcestupidcompile then
-        fallback = true
-    elseif strip then
-        done = os.spawn(format(luac_strip, lucfile,luafile)) == 0
-    else
-        done = os.spawn(format(luac_normal,lucfile,luafile)) == 0
-    end
-    if not done and fallback then
-        local n = stupidcompile(luafile,lucfile,strip)
-        if n > 0 then
-            utilities.report("lua: %s dumped into %s (%i bytes stripped)",luafile,lucfile,n)
+    function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true
+        report_lua("compiling %a into %a",luafile,lucfile)
+        os.remove(lucfile)
+        local done = false
+        if strip ~= false then
+            strip = true
+        end
+        if forcestupidcompile then
+            fallback = true
+        elseif strip then
+            done = os.spawn(format(luac_strip, lucfile,luafile)) == 0
         else
-            utilities.report("lua: %s dumped into %s (unstripped)",luafile,lucfile)
+            done = os.spawn(format(luac_normal,lucfile,luafile)) == 0
         end
-        cleanup = false -- better see how bad it is
-    end
-    if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
-        utilities.report("lua: removing %s",luafile)
-        os.remove(luafile)
+        if not done and fallback then
+            local n = stupidcompile(luafile,lucfile,strip)
+            if n > 0 then
+                report_lua("%a dumped into %a (%i bytes stripped)",luafile,lucfile,n)
+            else
+                report_lua("%a dumped into %a (unstripped)",luafile,lucfile)
+            end
+            cleanup = false -- better see how bad it is
+            done = true -- hm
+        end
+        if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then
+            report_lua("removing %a",luafile)
+            os.remove(luafile)
+        end
+        return done
     end
-    return done
-end
---~ local getmetatable, type = getmetatable, type
-
---~ local types = { }
-
---~ function luautilities.registerdatatype(d,name)
---~     types[getmetatable(d)] = name
---~ end
 
---~ function luautilities.datatype(d)
---~     local t = type(d)
---~     if t == "userdata" then
---~         local m = getmetatable(d)
---~         return m and types[m] or "userdata"
---~     else
---~         return t
---~     end
---~ end
+    luautilities.loadstripped = loadstring
 
---~ luautilities.registerdatatype(lpeg.P("!"),"lpeg")
+end
 
---~ print(luautilities.datatype(lpeg.P("oeps")))
+-- local getmetatable, type = getmetatable, type
+--
+-- local types = { }
+--
+-- function luautilities.registerdatatype(d,name)
+--     types[getmetatable(d)] = name
+-- end
+--
+-- function luautilities.datatype(d)
+--     local t = type(d)
+--     if t == "userdata" then
+--         local m = getmetatable(d)
+--         return m and types[m] or "userdata"
+--     else
+--         return t
+--     end
+-- end
+--
+-- luautilities.registerdatatype(lpeg.P("!"),"lpeg")
+--
+-- print(luautilities.datatype(lpeg.P("oeps")))