beta 2012.11.07 23:16

3 files changed, 135 insertions, 15 deletions

diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 75d09a583..96b153adf 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -2016,6 +2016,7 @@ if not modules then modules = { } end modules ['l-io'] = {
 local io = io
 local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
 local concat = table.concat
+local floor = math.floor
 local type = type
 
 if string.find(os.getenv("PATH"),";") then
@@ -2024,10 +2025,49 @@ else
     io.fileseparator, io.pathseparator = "/" , ":"
 end
 
+local function readall(f)
+    return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+    local size = f:seek("end")
+    if size == 0 then
+        return ""
+    elseif size < 1024*1024 then
+        f:seek("set",0)
+        return f:read('*all')
+    else
+        local done = f:seek("set",0)
+        if size < 1024*1024 then
+            step = 1024 * 1024
+        elseif size > 16*1024*1024 then
+            step = 16*1024*1024
+        else
+            step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+        end
+        local data = { }
+        while true do
+            local r = f:read(step)
+            if not r then
+                return concat(data)
+            else
+                data[#data+1] = r
+            end
+        end
+    end
+end
+
+io.readall = readall
+
 function io.loaddata(filename,textmode) -- return nil if empty
     local f = io.open(filename,(textmode and 'r') or 'rb')
     if f then
-        local data = f:read('*all')
+     -- local data = f:read('*all')
+        local data = readall(f)
         f:close()
         if #data > 0 then
             return data
@@ -8520,13 +8560,13 @@ function xml.load(filename,settings)
     local data = ""
     if type(filename) == "string" then
      -- local data = io.loaddata(filename) - -todo: check type in io.loaddata
-        local f = io.open(filename,'r')
+        local f = io.open(filename,'r') -- why not 'rb'
         if f then
-            data = f:read("*all")
+            data = f:read("*all") -- io.readall(f) ... only makes sense for large files
             f:close()
         end
     elseif filename then -- filehandle
-        data = filename:read("*all")
+        data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
     end
     if settings then
         settings.currentresource = filename
@@ -15224,7 +15264,7 @@ function loaders.file(specification,filetype)
             if trace_locating then
                 report_files("file loader, '%s' loaded",filename)
             end
-            local s = f:read("*a")
+            local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
             if checkgarbage then
                 checkgarbage(#s)
             end
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 75d09a583..96b153adf 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -2016,6 +2016,7 @@ if not modules then modules = { } end modules ['l-io'] = {
 local io = io
 local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
 local concat = table.concat
+local floor = math.floor
 local type = type
 
 if string.find(os.getenv("PATH"),";") then
@@ -2024,10 +2025,49 @@ else
     io.fileseparator, io.pathseparator = "/" , ":"
 end
 
+local function readall(f)
+    return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+    local size = f:seek("end")
+    if size == 0 then
+        return ""
+    elseif size < 1024*1024 then
+        f:seek("set",0)
+        return f:read('*all')
+    else
+        local done = f:seek("set",0)
+        if size < 1024*1024 then
+            step = 1024 * 1024
+        elseif size > 16*1024*1024 then
+            step = 16*1024*1024
+        else
+            step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+        end
+        local data = { }
+        while true do
+            local r = f:read(step)
+            if not r then
+                return concat(data)
+            else
+                data[#data+1] = r
+            end
+        end
+    end
+end
+
+io.readall = readall
+
 function io.loaddata(filename,textmode) -- return nil if empty
     local f = io.open(filename,(textmode and 'r') or 'rb')
     if f then
-        local data = f:read('*all')
+     -- local data = f:read('*all')
+        local data = readall(f)
         f:close()
         if #data > 0 then
             return data
@@ -8520,13 +8560,13 @@ function xml.load(filename,settings)
     local data = ""
     if type(filename) == "string" then
      -- local data = io.loaddata(filename) - -todo: check type in io.loaddata
-        local f = io.open(filename,'r')
+        local f = io.open(filename,'r') -- why not 'rb'
         if f then
-            data = f:read("*all")
+            data = f:read("*all") -- io.readall(f) ... only makes sense for large files
             f:close()
         end
     elseif filename then -- filehandle
-        data = filename:read("*all")
+        data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
     end
     if settings then
         settings.currentresource = filename
@@ -15224,7 +15264,7 @@ function loaders.file(specification,filetype)
             if trace_locating then
                 report_files("file loader, '%s' loaded",filename)
             end
-            local s = f:read("*a")
+            local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
             if checkgarbage then
                 checkgarbage(#s)
             end
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 75d09a583..96b153adf 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -2016,6 +2016,7 @@ if not modules then modules = { } end modules ['l-io'] = {
 local io = io
 local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
 local concat = table.concat
+local floor = math.floor
 local type = type
 
 if string.find(os.getenv("PATH"),";") then
@@ -2024,10 +2025,49 @@ else
     io.fileseparator, io.pathseparator = "/" , ":"
 end
 
+local function readall(f)
+    return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+    local size = f:seek("end")
+    if size == 0 then
+        return ""
+    elseif size < 1024*1024 then
+        f:seek("set",0)
+        return f:read('*all')
+    else
+        local done = f:seek("set",0)
+        if size < 1024*1024 then
+            step = 1024 * 1024
+        elseif size > 16*1024*1024 then
+            step = 16*1024*1024
+        else
+            step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+        end
+        local data = { }
+        while true do
+            local r = f:read(step)
+            if not r then
+                return concat(data)
+            else
+                data[#data+1] = r
+            end
+        end
+    end
+end
+
+io.readall = readall
+
 function io.loaddata(filename,textmode) -- return nil if empty
     local f = io.open(filename,(textmode and 'r') or 'rb')
     if f then
-        local data = f:read('*all')
+     -- local data = f:read('*all')
+        local data = readall(f)
         f:close()
         if #data > 0 then
             return data
@@ -8520,13 +8560,13 @@ function xml.load(filename,settings)
     local data = ""
     if type(filename) == "string" then
      -- local data = io.loaddata(filename) - -todo: check type in io.loaddata
-        local f = io.open(filename,'r')
+        local f = io.open(filename,'r') -- why not 'rb'
         if f then
-            data = f:read("*all")
+            data = f:read("*all") -- io.readall(f) ... only makes sense for large files
             f:close()
         end
     elseif filename then -- filehandle
-        data = filename:read("*all")
+        data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
     end
     if settings then
         settings.currentresource = filename
@@ -15224,7 +15264,7 @@ function loaders.file(specification,filetype)
             if trace_locating then
                 report_files("file loader, '%s' loaded",filename)
             end
-            local s = f:read("*a")
+            local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
             if checkgarbage then
                 checkgarbage(#s)
             end