Merge pull request #7 from phi-gamma/master

file integrity test

5 files changed, 148 insertions, 289 deletions

diff --git a/Makefile b/Makefile
index 4c9edc3..4a08cf0 100644
--- a/Makefile
+++ b/Makefile
@@ -6,20 +6,21 @@ DOC_DTX = $(patsubst %.dtx, %.pdf, $(DTX))
 MODULES = $(wildcard lualibs-*.lua)
 
 # Files grouped by generation mode
-UNPACKED= lualibs.lua
-COMPILED = $(DOC_DTX)
-GENERATED = $(UNPACKED) $(DOC_DTX) $(MERGED)
-SOURCE = $(DTX) $(MODULES) LICENSE README Makefile NEWS
-MERGED = lualibs-basic-merged.lua lualibs-extended-merged.lua
+TESTSCRIPT 		= test-lualibs.lua
+UNPACKED		= lualibs.lua lualibs-extended.lua lualibs-basic.lua
+COMPILED 		= $(DOC_DTX)
+GENERATED 		= $(UNPACKED) $(DOC_DTX) $(MERGED)
+SOURCE 			= $(DTX) $(MODULES) $(TESTSCRIPT) LICENSE README Makefile NEWS
+MERGED 			= lualibs-basic-merged.lua lualibs-extended-merged.lua
 
 # Files grouped by installation location
 RUNFILES = $(UNPACKED) $(MODULES)
 DOCFILES = $(DOC_DTX) LICENSE README NEWS
-SRCFILES = $(DTX) $(SRC_TEX) Makefile
+SRCFILES = $(DTX) $(SRC_TEX) Makefile $(TESTSCRIPT)
 
 # The following definitions should be equivalent
 # ALL_FILES = $(RUNFILES) $(DOCFILES) $(SRCFILES)
-ALL_FILES = $(GENERATED) $(SOURCE) $(MERGED)
+ALL_FILES = $(GENERATED) $(SOURCE)
 
 # Installation locations
 FORMAT = luatex
@@ -37,13 +38,16 @@ DO_PDFLATEX 	= latexmk -pdf -e '$$pdflatex = q(lualatex %O %S)' -silent $< >/dev
 DO_MAKEINDEX 	= makeindex  -s gind.ist $(subst .dtx,,$<) >/dev/null 2>&1
 DO_PACKAGE 		= mtxrun     --script package --merge $< >/dev/null
 
-all: $(GENERATED) $(DOC_TEX) $(MERGED)
+all: $(GENERATED) $(DOC_TEX)
 doc: $(COMPILED)
 unpack: $(UNPACKED)
-ctan: $(CTAN_ZIP)
+ctan: check $(CTAN_ZIP)
 tds: $(TDS_ZIP)
 world: all ctan
 
+check: $(TESTSCRIPT)
+	@texlua $(TESTSCRIPT)
+
 .PHONY: all doc unpack ctan tds world
 
 %.pdf: %.dtx
diff --git a/README b/README
index daf0c8e..499ceb8 100644
--- a/README
+++ b/README
@@ -33,52 +33,54 @@ The methods "commented out" are currently not available.
 
 Manifest
 --------------------------------------------------------------------------------
+            file                            TDS location
 
 Source files:
-    lualibs.dtx
-    lualibs-boolean.lua
-    lualibs-compat.lua
-    lualibs-dir.lua
-    lualibs-file.lua
-    lualibs-function.lua
-    lualibs-io.lua
-    lualibs-lpeg.lua
-    lualibs-lua.lua
-    lualibs-math.lua
-    lualibs-md5.lua
-    lualibs-number.lua
-    lualibs-os.lua
-    lualibs-package.lua
-    lualibs-set.lua
-    lualibs-string.lua
-    lualibs-table.lua
-    lualibs-trac-inf.lua
-    lualibs-unicode.lua
-    lualibs-url.lua
-    lualibs-util-deb.lua
-    lualibs-util-dim.lua
-    lualibs-util-env.lua
-    lualibs-util-jsn.lua
-    lualibs-util-lua.lua
-    lualibs-util-mrg.lua
-    lualibs-util-prs.lua
-    lualibs-util-sta.lua
-    lualibs-util-sto.lua
-    lualibs-util-str.lua
-    lualibs-util-tab.lua
-    lualibs-util-tpl.lua
-    LICENSE
-    Makefile
-    NEWS
-    README
+    lualibs.dtx                     source/luatex/lualibs/lualibs.dtx
+    lualibs-boolean.lua             tex/luatex/lualibs/lualibs-boolean.lua
+    lualibs-compat.lua              tex/luatex/lualibs/lualibs-compat.lua
+    lualibs-dir.lua                 tex/luatex/lualibs/lualibs-dir.lua
+    lualibs-file.lua                tex/luatex/lualibs/lualibs-file.lua
+    lualibs-function.lua            tex/luatex/lualibs/lualibs-function.lua
+    lualibs-io.lua                  tex/luatex/lualibs/lualibs-io.lua
+    lualibs-lpeg.lua                tex/luatex/lualibs/lualibs-lpeg.lua
+    lualibs-lua.lua                 tex/luatex/lualibs/lualibs-lua.lua
+    lualibs-math.lua                tex/luatex/lualibs/lualibs-math.lua
+    lualibs-md5.lua                 tex/luatex/lualibs/lualibs-md5.lua
+    lualibs-number.lua              tex/luatex/lualibs/lualibs-number.lua
+    lualibs-os.lua                  tex/luatex/lualibs/lualibs-os.lua
+    lualibs-package.lua             tex/luatex/lualibs/lualibs-package.lua
+    lualibs-set.lua                 tex/luatex/lualibs/lualibs-set.lua
+    lualibs-string.lua              tex/luatex/lualibs/lualibs-string.lua
+    lualibs-table.lua               tex/luatex/lualibs/lualibs-table.lua
+    lualibs-trac-inf.lua            tex/luatex/lualibs/lualibs-trac-inf.lua
+    lualibs-unicode.lua             tex/luatex/lualibs/lualibs-unicode.lua
+    lualibs-url.lua                 tex/luatex/lualibs/lualibs-url.lua
+    lualibs-util-deb.lua            tex/luatex/lualibs/lualibs-util-deb.lua
+    lualibs-util-dim.lua            tex/luatex/lualibs/lualibs-util-dim.lua
+    lualibs-util-env.lua            tex/luatex/lualibs/lualibs-util-env.lua
+    lualibs-util-jsn.lua            tex/luatex/lualibs/lualibs-util-jsn.lua
+    lualibs-util-lua.lua            tex/luatex/lualibs/lualibs-util-lua.lua
+    lualibs-util-prs.lua            tex/luatex/lualibs/lualibs-util-prs.lua
+    lualibs-util-sta.lua            tex/luatex/lualibs/lualibs-util-sta.lua
+    lualibs-util-sto.lua            tex/luatex/lualibs/lualibs-util-sto.lua
+    lualibs-util-str.lua            tex/luatex/lualibs/lualibs-util-str.lua
+    lualibs-util-tab.lua            tex/luatex/lualibs/lualibs-util-tab.lua
+    lualibs-util-fmt.lua            tex/luatex/lualibs/lualibs-util-fmt.lua
+    lualibs-util-tpl.lua            tex/luatex/lualibs/lualibs-util-tpl.lua
+    LICENSE                         doc/luatex/lualibs/LICENSE
+    Makefile                        source/luatex/lualibs/Makefile
+    NEWS                            doc/luatex/lualibs/NEWS
+    README                          doc/luatex/lualibs/README
+    test-lualibs.lua                source/luatex/lualibs/test-lualibs.lua
 
 Derived files:
-    lualibs.lua
-    lualibs-basic.lua
-    lualibs-basic-merged.lua
-    lualibs-extended.lua
-    lualibs-extended-merged.lua
-    lualibs.pdf
+    lualibs.lua                     tex/luatex/lualibs/lualibs.lua
+    lualibs-basic.lua               tex/luatex/lualibs/lualibs-basic.lua
+    lualibs-basic-merged.lua        tex/luatex/lualibs/lualibs-basic-merged.lua
+    lualibs-extended.lua            tex/luatex/lualibs/lualibs-extended.lua
+    lualibs-extended-merged.lua     tex/luatex/lualibs/lualibs-extended-merged.lua
+    lualibs.pdf                     doc/luatex/lualibs/lualibs.pdf
 
 License and Copyright
 --------------------------------------------------------------------------------
diff --git a/lualibs-util-mrg.lua b/lualibs-util-mrg.lua
deleted file mode 100644
index 690188e..0000000
--- a/lualibs-util-mrg.lua
+++ /dev/null
@@ -1,228 +0,0 @@
-if not modules then modules = { } end modules ['util-mrg'] = {
-    version   = 1.001,
-    comment   = "companion to luat-lib.mkiv",
-    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
-    copyright = "PRAGMA ADE / ConTeXt Development Team",
-    license   = "see context related readme files"
-}
-
--- hm, quite unreadable
-
-local gsub, format = string.gsub, string.format
-local concat = table.concat
-local type, next = type, next
-
-local P, R, S, V, Ct, C, Cs, Cc, Cp, Cmt, Cb, Cg = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Cp, lpeg.Cmt, lpeg.Cb, lpeg.Cg
-local lpegmatch, patterns = lpeg.match, lpeg.patterns
-
-utilities             = utilities or { }
-local merger          = utilities.merger or { }
-utilities.merger      = merger
-merger.strip_comment  = true
-
-local report          = logs.reporter("system","merge")
-utilities.report      = report
-
-local m_begin_merge   = "begin library merge"
-local m_end_merge     = "end library merge"
-local m_begin_closure = "do -- create closure to overcome 200 locals limit"
-local m_end_closure   = "end -- of closure"
-
-local m_pattern =
-    "%c+" ..
-    "%-%-%s+" .. m_begin_merge ..
-    "%c+(.-)%c+" ..
-    "%-%-%s+" .. m_end_merge ..
-    "%c+"
-
-local m_format =
-    "\n\n-- " .. m_begin_merge ..
-    "\n%s\n" ..
-    "-- " .. m_end_merge .. "\n\n"
-
-local m_faked =
-    "-- " .. "created merged file" .. "\n\n" ..
-    "-- " .. m_begin_merge .. "\n\n" ..
-    "-- " .. m_end_merge .. "\n\n"
-
-local m_report = [[
--- used libraries    : %s
--- skipped libraries : %s
--- original bytes    : %s
--- stripped bytes    : %s
-]]
-
-local m_preloaded = [[package.loaded[%q] = package.loaded[%q] or true]]
-
-local function self_fake()
-    return m_faked
-end
-
-local function self_nothing()
-    return ""
-end
-
-local function self_load(name)
-    local data = io.loaddata(name) or ""
-    if data == "" then
-        report("unknown file %a",name)
-    else
-        report("inserting file %a",name)
-    end
-    return data or ""
-end
-
--- -- saves some 20K .. scite comments
--- data = gsub(data,"%-%-~[^\n\r]*[\r\n]","")
--- -- saves some 20K .. ldx comments
--- data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","")
-
-local space           = patterns.space
-local eol             = patterns.newline
-local equals          = P("=")^0
-local open            = P("[") * Cg(equals,"init") * P("[") * P("\n")^-1
-local close           = P("]") * C(equals) * P("]")
-local closeeq         = Cmt(close * Cb("init"), function(s,i,a,b) return a == b end)
-local longstring      = open * (1 - closeeq)^0 * close
-
-local quoted          = patterns.quoted
-local digit           = patterns.digit
-local emptyline       = space^0 * eol
-local operator1       = P("<=") + P(">=") + P("~=") + P("..") + S("/^<>=*+%%")
-local operator2       = S("*+/")
-local operator3       = S("-")
-local operator4       = P("..")
-local separator       = S(",;")
-
-local ignore          = (P("]") * space^1 * P("=") * space^1 * P("]")) / "]=[" +
-                        (P("=") * space^1 * P("{")) / "={" +
-                        (P("(") * space^1) / "(" +
-                        (P("{") * (space+eol)^1 * P("}")) / "{}"
-local strings         = quoted --  / function (s) print("<<"..s..">>") return s end
-local longcmt         = (emptyline^0 * P("--") * longstring * emptyline^0) / ""
-local longstr         = longstring
-local comment         = emptyline^0 * P("--") * P("-")^0 * (1-eol)^0 * emptyline^1 / "\n"
-local optionalspaces  = space^0 / ""
-local mandatespaces   = space^1 / ""
-local optionalspacing = (eol+space)^0 / ""
-local mandatespacing  = (eol+space)^1 / ""
-local pack            = digit * space^1 * operator4 * optionalspacing +
-                        optionalspacing * operator1 * optionalspacing +
-                        optionalspacing * operator2 * optionalspaces  +
-                        mandatespacing  * operator3 * mandatespaces   +
-                        optionalspaces  * separator * optionalspaces
-local lines           = emptyline^2 / "\n"
-local spaces          = (space * space) / " "
------ spaces          = ((space+eol)^1 ) / " "
-
-local compact = Cs ( (
-    ignore  +
-    strings +
-    longcmt +
-    longstr +
-    comment +
-    pack    +
-    lines   +
-    spaces  +
-    1
-)^1 )
-
-local strip       = Cs((emptyline^2/"\n" + 1)^0)
-local stripreturn = Cs((1-P("return") * space^1 * P(1-space-eol)^1 * (space+eol)^0 * P(-1))^1)
-
-function merger.compact(data)
-    return lpegmatch(strip,lpegmatch(compact,data))
-end
-
-local function self_compact(data)
-    local delta = 0
-    if merger.strip_comment then
-        local before = #data
-        data = lpegmatch(compact,data)
-        data = lpegmatch(strip,data) -- also strips in longstrings ... alas
-     -- data = string.strip(data)
-        local after = #data
-        delta = before - after
-        report("original size %s, compacted to %s, stripped %s",before,after,delta)
-        data = format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data)
-    end
-    return lpegmatch(stripreturn,data) or data, delta
-end
-
-local function self_save(name, data)
-    if data ~= "" then
-        io.savedata(name,data)
-        report("saving %s with size %s",name,#data)
-    end
-end
-
-local function self_swap(data,code)
-    return data ~= "" and (gsub(data,m_pattern, function() return format(m_format,code) end, 1)) or ""
-end
-
-local function self_libs(libs,list)
-    local result, f, frozen, foundpath = { }, nil, false, nil
-    result[#result+1] = "\n"
-    if type(libs) == 'string' then libs = { libs } end
-    if type(list) == 'string' then list = { list } end
-    for i=1,#libs do
-        local lib = libs[i]
-        for j=1,#list do
-            local pth = gsub(list[j],"\\","/") -- file.clean_path
-            report("checking library path %a",pth)
-            local name = pth .. "/" .. lib
-            if lfs.isfile(name) then
-                foundpath = pth
-            end
-        end
-        if foundpath then break end
-    end
-    if foundpath then
-        report("using library path %a",foundpath)
-        local right, wrong, original, stripped = { }, { }, 0, 0
-        for i=1,#libs do
-            local lib = libs[i]
-            local fullname = foundpath .. "/" .. lib
-            if lfs.isfile(fullname) then
-                report("using library %a",fullname)
-                local preloaded = file.nameonly(lib)
-                local data = io.loaddata(fullname,true)
-                original = original + #data
-                local data, delta = self_compact(data)
-                right[#right+1] = lib
-                result[#result+1] = m_begin_closure
-                result[#result+1] = format(m_preloaded,preloaded,preloaded)
-                result[#result+1] = data
-                result[#result+1] = m_end_closure
-                stripped = stripped + delta
-            else
-                report("skipping library %a",fullname)
-                wrong[#wrong+1] = lib
-            end
-        end
-        right = #right > 0 and concat(right," ") or "-"
-        wrong = #wrong > 0 and concat(wrong," ") or "-"
-        report("used libraries: %a",right)
-        report("skipped libraries: %a",wrong)
-        report("original bytes: %a",original)
-        report("stripped bytes: %a",stripped)
-        result[#result+1] = format(m_report,right,wrong,original,stripped)
-    else
-        report("no valid library path found")
-    end
-    return concat(result, "\n\n")
-end
-
-function merger.selfcreate(libs,list,target)
-    if target then
-        self_save(target,self_swap(self_fake(),self_libs(libs,list)))
-    end
-end
-
-function merger.selfmerge(name,libs,list,target)
-    self_save(target or name,self_swap(self_load(name),self_libs(libs,list)))
-end
-
-function merger.selfclean(name)
-    self_save(name,self_swap(self_load(name),self_nothing()))
-end
diff --git a/lualibs.dtx b/lualibs.dtx
index b8fe169..b8c74c0 100644
--- a/lualibs.dtx
+++ b/lualibs.dtx
@@ -450,13 +450,9 @@ lualibs.module_info = {
 config           = config or { }
 config.lualibs   = config.lualibs or { }
 
-if config.lualibs.prefer_merged == nil then
-  lualibs.prefer_merged = true
-end
-if config.lualibs.load_extended == nil then
-  lualibs.load_extended = true
-end
-config.lualibs.verbose = config.lualibs.verbose == false
+lualibs.prefer_merged   = config.lualibs.prefer_merged or true
+lualibs.load_extended   = config.lualibs.load_extended or true
+config.lualibs.verbose  = config.lualibs.verbose       or false
 
 %    \end{macrocode}
 %     The lualibs may be loaded in scripts.
@@ -526,13 +522,17 @@ lualibs.loadmodule = loadmodule
 %
 %    \begin{macrocode}
 
-if lualibs.basic_loaded ~= true then
+if lualibs.basic_loaded        ~= true
+or config.lualibs.force_reload == true
+then
   loadmodule"lualibs-basic.lua"
   loadmodule"lualibs-compat.lua" --- restore stuff gone since v1.*
 end
 
-if  lualibs.load_extended   == true
-and lualibs.extended_loaded ~= true then
+if  lualibs.load_extended       == true
+and lualibs.extended_loaded     ~= true
+or  config.lualibs.force_reload == true
+then
   loadmodule"lualibs-extended.lua"
 end
 
diff --git a/test-lualibs.lua b/test-lualibs.lua
new file mode 100644
index 0000000..b74fde1
--- /dev/null
+++ b/test-lualibs.lua
@@ -0,0 +1,81 @@
+#!/usr/bin/env texlua
+
+local luafiles = {
+  "lualibs-boolean.lua",   "lualibs-compat.lua",
+  "lualibs-dir.lua",       "lualibs-file.lua",
+  "lualibs-function.lua",  "lualibs-io.lua",
+  "lualibs-lpeg.lua",      "lualibs-lua.lua",
+  "lualibs-math.lua",      "lualibs-md5.lua",
+  "lualibs-number.lua",    "lualibs-os.lua",
+  "lualibs-package.lua",   "lualibs-set.lua",
+  "lualibs-string.lua",    "lualibs-table.lua",
+  "lualibs-trac-inf.lua",  "lualibs-unicode.lua",
+  "lualibs-url.lua",       "lualibs-util-deb.lua",
+  "lualibs-util-dim.lua",  "lualibs-util-env.lua",
+  "lualibs-util-jsn.lua",  "lualibs-util-lua.lua",
+  "lualibs-util-prs.lua",  "lualibs-util-sta.lua",
+  "lualibs-util-sto.lua",  "lualibs-util-str.lua",
+  "lualibs-util-tab.lua",  "lualibs-util-fmt.lua",
+  "lualibs-util-tpl.lua",  "lualibs.lua",
+  "lualibs-basic.lua",     "lualibs-basic-merged.lua",
+  "lualibs-extended.lua",  "lualibs-extended-merged.lua",
+}
+
+local test_cmd = "texluac -p %s &> /dev/null"
+
+local check_wellformed = function (file)
+  io.write"testing "
+  io.write(file)
+  io.write" ... "
+  local exit_status = os.execute(string.format(test_cmd, file))
+  if exit_status == 0 then
+    io.write"SUCCESS!\n"
+    return true
+  end
+  io.write"FAIL :-/\n"
+  return false
+end
+
+local check_files check_files = function (lst, n)
+  if n == nil then
+    return check_files(lst, 1)
+  end
+  local this = lst[n]
+  if this then
+    if check_wellformed(this) then
+      return check_files(lst, n+1)
+    else
+      return false
+    end
+  end
+  return true
+end
+
+config = { lualibs = { force_reload = true } }
+
+local load_all = function ( )
+
+  io.write"testing merged packages ... "
+  config.lualibs.prefer_merged = true
+  if not pcall(function () dofile"lualibs.lua"end) then
+    io.write"FAIL :-/\n"
+  end
+  io.write"SUCCESS\n"
+
+  io.write"testing files ... "
+  config.lualibs.prefer_merged = false
+  if not pcall(function () dofile"lualibs.lua"end) then
+    io.write"FAIL :-/\n"
+  end
+  io.write"SUCCESS\n"
+  return true
+end
+
+local main = function ( )
+  local retval = 0
+  retval = check_files(luafiles) and retval or 1
+  retval = load_all()            and retval or 1
+  os.exit(retval)
+end
+
+return main()