From 59cf0ae4a208ec4acd1d63229432f5eaaba00967 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 19 Oct 2012 21:55:19 +0200 Subject: reflect move l-utils -> util-mrg --- lualibs-util-mrg.lua | 176 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 lualibs-util-mrg.lua (limited to 'lualibs-util-mrg.lua') diff --git a/lualibs-util-mrg.lua b/lualibs-util-mrg.lua new file mode 100644 index 0000000..ebc27b8 --- /dev/null +++ b/lualibs-util-mrg.lua @@ -0,0 +1,176 @@ +if not modules then modules = { } end modules ['l-utils'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- hm, quite unreadable + +local gsub = string.gsub +local concat = table.concat +local type, next = type, next + +if not utils then utils = { } end +if not utils.merger then utils.merger = { } end +if not utils.lua then utils.lua = { } end + +utils.merger.m_begin = "begin library merge" +utils.merger.m_end = "end library merge" +utils.merger.pattern = + "%c+" .. + "%-%-%s+" .. utils.merger.m_begin .. + "%c+(.-)%c+" .. + "%-%-%s+" .. utils.merger.m_end .. + "%c+" + +function utils.merger._self_fake_() + return + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. utils.merger.m_begin .. "\n\n" .. + "-- " .. utils.merger.m_end .. "\n\n" +end + +function utils.report(...) + print(...) +end + +utils.merger.strip_comment = true + +function utils.merger._self_load_(name) + local f, data = io.open(name), "" + if f then + utils.report("reading merge from %s",name) + data = f:read("*all") + f:close() + else + utils.report("unknown file to merge %s",name) + end + if data and utils.merger.strip_comment then + -- saves some 20K + data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") + end + return data or "" +end + +function utils.merger._self_save_(name, data) + if data ~= "" then + local f = io.open(name,'w') + if f then + utils.report("saving merge from %s",name) + f:write(data) + f:close() + end + end +end + +function utils.merger._self_swap_(data,code) + if data ~= "" then + return (gsub(data,utils.merger.pattern, function(s) + return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" + end, 1)) + else + return "" + end +end + +--~ stripper: +--~ +--~ data = gsub(data,"%-%-~[^\n]*\n","") +--~ data = gsub(data,"\n\n+","\n") + +function utils.merger._self_libs_(libs,list) + local result, f, frozen = { }, nil, false + result[#result+1] = "\n" + if type(libs) == 'string' then libs = { libs } end + if type(list) == 'string' then list = { list } end + local foundpath = nil + for i=1,#libs do + local lib = libs[i] + for j=1,#list do + local pth = gsub(list[j],"\\","/") -- file.clean_path + utils.report("checking library path %s",pth) + local name = pth .. "/" .. lib + if lfs.isfile(name) then + foundpath = pth + end + end + if foundpath then break end + end + if foundpath then + utils.report("using library path %s",foundpath) + local right, wrong = { }, { } + for i=1,#libs do + local lib = libs[i] + local fullname = foundpath .. "/" .. lib + if lfs.isfile(fullname) then + -- right[#right+1] = lib + utils.report("merging library %s",fullname) + result[#result+1] = "do -- create closure to overcome 200 locals limit" + result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = "end -- of closure" + else + -- wrong[#wrong+1] = lib + utils.report("no library %s",fullname) + end + end + if #right > 0 then + utils.report("merged libraries: %s",concat(right," ")) + end + if #wrong > 0 then + utils.report("skipped libraries: %s",concat(wrong," ")) + end + else + utils.report("no valid library path found") + end + return concat(result, "\n\n") +end + +function utils.merger.selfcreate(libs,list,target) + if target then + utils.merger._self_save_( + target, + utils.merger._self_swap_( + utils.merger._self_fake_(), + utils.merger._self_libs_(libs,list) + ) + ) + end +end + +function utils.merger.selfmerge(name,libs,list,target) + utils.merger._self_save_( + target or name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + utils.merger._self_libs_(libs,list) + ) + ) +end + +function utils.merger.selfclean(name) + utils.merger._self_save_( + name, + utils.merger._self_swap_( + utils.merger._self_load_(name), + "" + ) + ) +end + +function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true + -- utils.report("compiling",luafile,"into",lucfile) + os.remove(lucfile) + local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) + if strip ~= false then + command = "-s " .. command + end + local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) + if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + -- utils.report("removing",luafile) + os.remove(luafile) + end + return done +end + -- cgit v1.2.3 From 672629eb217dd4b7e56b63c11fdceb983dd0eacd Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 19 Oct 2012 21:59:40 +0200 Subject: update util-mrg --- lualibs-util-mrg.lua | 167 ++++++++++++++++++++------------------------------- 1 file changed, 65 insertions(+), 102 deletions(-) (limited to 'lualibs-util-mrg.lua') diff --git a/lualibs-util-mrg.lua b/lualibs-util-mrg.lua index ebc27b8..d59745b 100644 --- a/lualibs-util-mrg.lua +++ b/lualibs-util-mrg.lua @@ -8,89 +8,86 @@ if not modules then modules = { } end modules ['l-utils'] = { -- hm, quite unreadable -local gsub = string.gsub +local gsub, format = string.gsub, string.format local concat = table.concat local type, next = type, next -if not utils then utils = { } end -if not utils.merger then utils.merger = { } end -if not utils.lua then utils.lua = { } end +utilities = utilities or {} +local merger = utilities.merger or { } +utilities.merger = merger +utilities.report = logs and logs.reporter("system") or print -utils.merger.m_begin = "begin library merge" -utils.merger.m_end = "end library merge" -utils.merger.pattern = +merger.strip_comment = true + +local m_begin_merge = "begin library merge" +local m_end_merge = "end library merge" +local m_begin_closure = "do -- create closure to overcome 200 locals limit" +local m_end_closure = "end -- of closure" + +local m_pattern = "%c+" .. - "%-%-%s+" .. utils.merger.m_begin .. + "%-%-%s+" .. m_begin_merge .. "%c+(.-)%c+" .. - "%-%-%s+" .. utils.merger.m_end .. + "%-%-%s+" .. m_end_merge .. "%c+" -function utils.merger._self_fake_() - return - "-- " .. "created merged file" .. "\n\n" .. - "-- " .. utils.merger.m_begin .. "\n\n" .. - "-- " .. utils.merger.m_end .. "\n\n" -end +local m_format = + "\n\n-- " .. m_begin_merge .. + "\n%s\n" .. + "-- " .. m_end_merge .. "\n\n" + +local m_faked = + "-- " .. "created merged file" .. "\n\n" .. + "-- " .. m_begin_merge .. "\n\n" .. + "-- " .. m_end_merge .. "\n\n" -function utils.report(...) - print(...) +local function self_fake() + return m_faked end -utils.merger.strip_comment = true +local function self_nothing() + return "" +end -function utils.merger._self_load_(name) - local f, data = io.open(name), "" - if f then - utils.report("reading merge from %s",name) - data = f:read("*all") - f:close() +local function self_load(name) + local data = io.loaddata(name) or "" + if data == "" then + utilities.report("merge: unknown file %s",name) else - utils.report("unknown file to merge %s",name) - end - if data and utils.merger.strip_comment then - -- saves some 20K - data = gsub(data,"%-%-~[^\n\r]*[\r\n]", "") + utilities.report("merge: inserting %s",name) end return data or "" end -function utils.merger._self_save_(name, data) +local function self_save(name, data) if data ~= "" then - local f = io.open(name,'w') - if f then - utils.report("saving merge from %s",name) - f:write(data) - f:close() + if merger.strip_comment then + local n = #data + -- saves some 20K .. scite comments + data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") + -- saves some 20K .. ldx comments + data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") + utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) end + io.savedata(name,data) + utilities.report("merge: saving %s",name) end end -function utils.merger._self_swap_(data,code) - if data ~= "" then - return (gsub(data,utils.merger.pattern, function(s) - return "\n\n" .. "-- "..utils.merger.m_begin .. "\n" .. code .. "\n" .. "-- "..utils.merger.m_end .. "\n\n" - end, 1)) - else - return "" - end +local function self_swap(data,code) + return data ~= "" and (gsub(data,m_pattern, function() return format(m_format,code) end, 1)) or "" end ---~ stripper: ---~ ---~ data = gsub(data,"%-%-~[^\n]*\n","") ---~ data = gsub(data,"\n\n+","\n") - -function utils.merger._self_libs_(libs,list) - local result, f, frozen = { }, nil, false +local function self_libs(libs,list) + local result, f, frozen, foundpath = { }, nil, false, nil result[#result+1] = "\n" if type(libs) == 'string' then libs = { libs } end if type(list) == 'string' then list = { list } end - local foundpath = nil for i=1,#libs do local lib = libs[i] for j=1,#list do local pth = gsub(list[j],"\\","/") -- file.clean_path - utils.report("checking library path %s",pth) + utilities.report("merge: checking library path %s",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then foundpath = pth @@ -99,78 +96,44 @@ function utils.merger._self_libs_(libs,list) if foundpath then break end end if foundpath then - utils.report("using library path %s",foundpath) + utilities.report("merge: using library path %s",foundpath) local right, wrong = { }, { } for i=1,#libs do local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then - -- right[#right+1] = lib - utils.report("merging library %s",fullname) - result[#result+1] = "do -- create closure to overcome 200 locals limit" + utilities.report("merge: using library %s",fullname) + right[#right+1] = lib + result[#result+1] = m_begin_closure result[#result+1] = io.loaddata(fullname,true) - result[#result+1] = "end -- of closure" + result[#result+1] = m_end_closure else - -- wrong[#wrong+1] = lib - utils.report("no library %s",fullname) + utilities.report("merge: skipping library %s",fullname) + wrong[#wrong+1] = lib end end if #right > 0 then - utils.report("merged libraries: %s",concat(right," ")) + utilities.report("merge: used libraries: %s",concat(right," ")) end if #wrong > 0 then - utils.report("skipped libraries: %s",concat(wrong," ")) + utilities.report("merge: skipped libraries: %s",concat(wrong," ")) end else - utils.report("no valid library path found") + utilities.report("merge: no valid library path found") end return concat(result, "\n\n") end -function utils.merger.selfcreate(libs,list,target) +function merger.selfcreate(libs,list,target) if target then - utils.merger._self_save_( - target, - utils.merger._self_swap_( - utils.merger._self_fake_(), - utils.merger._self_libs_(libs,list) - ) - ) + self_save(target,self_swap(self_fake(),self_libs(libs,list))) end end -function utils.merger.selfmerge(name,libs,list,target) - utils.merger._self_save_( - target or name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - utils.merger._self_libs_(libs,list) - ) - ) +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) end -function utils.merger.selfclean(name) - utils.merger._self_save_( - name, - utils.merger._self_swap_( - utils.merger._self_load_(name), - "" - ) - ) +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) end - -function utils.lua.compile(luafile, lucfile, cleanup, strip) -- defaults: cleanup=false strip=true - -- utils.report("compiling",luafile,"into",lucfile) - os.remove(lucfile) - local command = "-o " .. string.quote(lucfile) .. " " .. string.quote(luafile) - if strip ~= false then - command = "-s " .. command - end - local done = (os.spawn("texluac " .. command) == 0) or (os.spawn("luac " .. command) == 0) - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - -- utils.report("removing",luafile) - os.remove(luafile) - end - return done -end - -- cgit v1.2.3 From ee229c4cb8b72d7d69c7283a9817a601d37e22b7 Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Fri, 19 Oct 2012 22:11:40 +0200 Subject: update util-tab --- lualibs-util-mrg.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lualibs-util-mrg.lua') diff --git a/lualibs-util-mrg.lua b/lualibs-util-mrg.lua index d59745b..8d6c5dd 100644 --- a/lualibs-util-mrg.lua +++ b/lualibs-util-mrg.lua @@ -1,4 +1,4 @@ -if not modules then modules = { } end modules ['l-utils'] = { +if not modules then modules = { } end modules ['util-mrg'] = { version = 1.001, comment = "companion to luat-lib.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", -- cgit v1.2.3 From 64d4b28bb8f9bf40a96edf73d3a8bc276df5992e Mon Sep 17 00:00:00 2001 From: Philipp Gesang Date: Sun, 7 Apr 2013 21:57:43 +0200 Subject: update util-mrg --- lualibs-util-mrg.lua | 136 +++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 27 deletions(-) (limited to 'lualibs-util-mrg.lua') diff --git a/lualibs-util-mrg.lua b/lualibs-util-mrg.lua index 8d6c5dd..78b23dc 100644 --- a/lualibs-util-mrg.lua +++ b/lualibs-util-mrg.lua @@ -12,13 +12,17 @@ local gsub, format = string.gsub, string.format local concat = table.concat local type, next = type, next -utilities = utilities or {} +local P, R, S, V, Ct, C, Cs, Cc, Cp, Cmt, Cb, Cg = lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Ct, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Cp, lpeg.Cmt, lpeg.Cb, lpeg.Cg +local lpegmatch, patterns = lpeg.match, lpeg.patterns + +utilities = utilities or { } local merger = utilities.merger or { } utilities.merger = merger -utilities.report = logs and logs.reporter("system") or print - merger.strip_comment = true +local report = logs.reporter("system","merge") +utilities.report = report + local m_begin_merge = "begin library merge" local m_end_merge = "end library merge" local m_begin_closure = "do -- create closure to overcome 200 locals limit" @@ -41,6 +45,15 @@ local m_faked = "-- " .. m_begin_merge .. "\n\n" .. "-- " .. m_end_merge .. "\n\n" +local m_report = [[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] + +local m_preloaded = [[package.loaded[%q] = package.loaded[%q] or true]] + local function self_fake() return m_faked end @@ -52,25 +65,87 @@ end local function self_load(name) local data = io.loaddata(name) or "" if data == "" then - utilities.report("merge: unknown file %s",name) + report("unknown file %a",name) else - utilities.report("merge: inserting %s",name) + report("inserting file %a",name) end return data or "" end +-- -- saves some 20K .. scite comments +-- data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") +-- -- saves some 20K .. ldx comments +-- data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") + +local space = patterns.space +local eol = patterns.newline +local equals = P("=")^0 +local open = P("[") * Cg(equals,"init") * P("[") * P("\n")^-1 +local close = P("]") * C(equals) * P("]") +local closeeq = Cmt(close * Cb("init"), function(s,i,a,b) return a == b end) +local longstring = open * (1 - closeeq)^0 * close + +local quoted = patterns.quoted +local emptyline = space^0 * eol +local operator1 = P("<=") + P(">=") + P("~=") + P("..") + S("/^<>=*+%%") +local operator2 = S("*+/") +local operator3 = S("-") +local separator = S(",;") + +local ignore = (P("]") * space^1 * P("=") * space^1 * P("]")) / "]=[" + + (P("=") * space^1 * P("{")) / "={" + + (P("(") * space^1) / "(" + + (P("{") * (space+eol)^1 * P("}")) / "{}" +local strings = quoted -- / function (s) print("<<"..s..">>") return s end +local longcmt = (emptyline^0 * P("--") * longstring * emptyline^0) / "" +local longstr = longstring +local comment = emptyline^0 * P("--") * P("-")^0 * (1-eol)^0 * emptyline^1 / "\n" +local pack = ((eol+space)^0 / "") * operator1 * ((eol+space)^0 / "") + + ((eol+space)^0 / "") * operator2 * ((space)^0 / "") + + ((eol+space)^1 / "") * operator3 * ((space)^1 / "") + + ((space)^0 / "") * separator * ((space)^0 / "") +local lines = emptyline^2 / "\n" +local spaces = (space * space) / " " +----- spaces = ((space+eol)^1 ) / " " + +local compact = Cs ( ( + ignore + + strings + + longcmt + + longstr + + comment + + pack + + lines + + spaces + + 1 +)^1 ) + +local strip = Cs((emptyline^2/"\n" + 1)^0) +local stripreturn = Cs((1-P("return") * space^1 * P(1-space-eol)^1 * (space+eol)^0 * P(-1))^1) + +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end + +local function self_compact(data) + local delta = 0 + if merger.strip_comment then + local before = #data + data = lpegmatch(compact,data) + data = lpegmatch(strip,data) -- also strips in longstrings ... alas + -- data = string.strip(data) + local after = #data + delta = before - after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data = format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data, delta +end + local function self_save(name, data) if data ~= "" then - if merger.strip_comment then - local n = #data - -- saves some 20K .. scite comments - data = gsub(data,"%-%-~[^\n\r]*[\r\n]","") - -- saves some 20K .. ldx comments - data = gsub(data,"%-%-%[%[ldx%-%-.-%-%-ldx%]%]%-%-","") - utilities.report("merge: %s bytes of comment stripped, %s bytes of code left",n-#data,#data) - end io.savedata(name,data) - utilities.report("merge: saving %s",name) + report("saving %s with size %s",name,#data) end end @@ -87,7 +162,7 @@ local function self_libs(libs,list) local lib = libs[i] for j=1,#list do local pth = gsub(list[j],"\\","/") -- file.clean_path - utilities.report("merge: checking library path %s",pth) + report("checking library path %a",pth) local name = pth .. "/" .. lib if lfs.isfile(name) then foundpath = pth @@ -96,30 +171,37 @@ local function self_libs(libs,list) if foundpath then break end end if foundpath then - utilities.report("merge: using library path %s",foundpath) - local right, wrong = { }, { } + report("using library path %a",foundpath) + local right, wrong, original, stripped = { }, { }, 0, 0 for i=1,#libs do local lib = libs[i] local fullname = foundpath .. "/" .. lib if lfs.isfile(fullname) then - utilities.report("merge: using library %s",fullname) + report("using library %a",fullname) + local preloaded = file.nameonly(lib) + local data = io.loaddata(fullname,true) + original = original + #data + local data, delta = self_compact(data) right[#right+1] = lib result[#result+1] = m_begin_closure - result[#result+1] = io.loaddata(fullname,true) + result[#result+1] = format(m_preloaded,preloaded,preloaded) + result[#result+1] = data result[#result+1] = m_end_closure + stripped = stripped + delta else - utilities.report("merge: skipping library %s",fullname) + report("skipping library %a",fullname) wrong[#wrong+1] = lib end end - if #right > 0 then - utilities.report("merge: used libraries: %s",concat(right," ")) - end - if #wrong > 0 then - utilities.report("merge: skipped libraries: %s",concat(wrong," ")) - end + right = #right > 0 and concat(right," ") or "-" + wrong = #wrong > 0 and concat(wrong," ") or "-" + report("used libraries: %a",right) + report("skipped libraries: %a",wrong) + report("original bytes: %a",original) + report("stripped bytes: %a",stripped) + result[#result+1] = format(m_report,right,wrong,original,stripped) else - utilities.report("merge: no valid library path found") + report("no valid library path found") end return concat(result, "\n\n") end -- cgit v1.2.3