summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'scripts')
-rw-r--r--scripts/context/lua/mtxrun.lua50
-rw-r--r--scripts/context/stubs/mswin/mtxrun.lua50
-rw-r--r--scripts/context/stubs/unix/mtxrun50
3 files changed, 135 insertions, 15 deletions
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua
index 75d09a583..96b153adf 100644
--- a/scripts/context/lua/mtxrun.lua
+++ b/scripts/context/lua/mtxrun.lua
@@ -2016,6 +2016,7 @@ if not modules then modules = { } end modules ['l-io'] = {
local io = io
local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
local concat = table.concat
+local floor = math.floor
local type = type
if string.find(os.getenv("PATH"),";") then
@@ -2024,10 +2025,49 @@ else
io.fileseparator, io.pathseparator = "/" , ":"
end
+local function readall(f)
+ return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+ local size = f:seek("end")
+ if size == 0 then
+ return ""
+ elseif size < 1024*1024 then
+ f:seek("set",0)
+ return f:read('*all')
+ else
+ local done = f:seek("set",0)
+ if size < 1024*1024 then
+ step = 1024 * 1024
+ elseif size > 16*1024*1024 then
+ step = 16*1024*1024
+ else
+ step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+ end
+ local data = { }
+ while true do
+ local r = f:read(step)
+ if not r then
+ return concat(data)
+ else
+ data[#data+1] = r
+ end
+ end
+ end
+end
+
+io.readall = readall
+
function io.loaddata(filename,textmode) -- return nil if empty
local f = io.open(filename,(textmode and 'r') or 'rb')
if f then
- local data = f:read('*all')
+ -- local data = f:read('*all')
+ local data = readall(f)
f:close()
if #data > 0 then
return data
@@ -8520,13 +8560,13 @@ function xml.load(filename,settings)
local data = ""
if type(filename) == "string" then
-- local data = io.loaddata(filename) - -todo: check type in io.loaddata
- local f = io.open(filename,'r')
+ local f = io.open(filename,'r') -- why not 'rb'
if f then
- data = f:read("*all")
+ data = f:read("*all") -- io.readall(f) ... only makes sense for large files
f:close()
end
elseif filename then -- filehandle
- data = filename:read("*all")
+ data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
end
if settings then
settings.currentresource = filename
@@ -15224,7 +15264,7 @@ function loaders.file(specification,filetype)
if trace_locating then
report_files("file loader, '%s' loaded",filename)
end
- local s = f:read("*a")
+ local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
if checkgarbage then
checkgarbage(#s)
end
diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua
index 75d09a583..96b153adf 100644
--- a/scripts/context/stubs/mswin/mtxrun.lua
+++ b/scripts/context/stubs/mswin/mtxrun.lua
@@ -2016,6 +2016,7 @@ if not modules then modules = { } end modules ['l-io'] = {
local io = io
local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
local concat = table.concat
+local floor = math.floor
local type = type
if string.find(os.getenv("PATH"),";") then
@@ -2024,10 +2025,49 @@ else
io.fileseparator, io.pathseparator = "/" , ":"
end
+local function readall(f)
+ return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+ local size = f:seek("end")
+ if size == 0 then
+ return ""
+ elseif size < 1024*1024 then
+ f:seek("set",0)
+ return f:read('*all')
+ else
+ local done = f:seek("set",0)
+ if size < 1024*1024 then
+ step = 1024 * 1024
+ elseif size > 16*1024*1024 then
+ step = 16*1024*1024
+ else
+ step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+ end
+ local data = { }
+ while true do
+ local r = f:read(step)
+ if not r then
+ return concat(data)
+ else
+ data[#data+1] = r
+ end
+ end
+ end
+end
+
+io.readall = readall
+
function io.loaddata(filename,textmode) -- return nil if empty
local f = io.open(filename,(textmode and 'r') or 'rb')
if f then
- local data = f:read('*all')
+ -- local data = f:read('*all')
+ local data = readall(f)
f:close()
if #data > 0 then
return data
@@ -8520,13 +8560,13 @@ function xml.load(filename,settings)
local data = ""
if type(filename) == "string" then
-- local data = io.loaddata(filename) - -todo: check type in io.loaddata
- local f = io.open(filename,'r')
+ local f = io.open(filename,'r') -- why not 'rb'
if f then
- data = f:read("*all")
+ data = f:read("*all") -- io.readall(f) ... only makes sense for large files
f:close()
end
elseif filename then -- filehandle
- data = filename:read("*all")
+ data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
end
if settings then
settings.currentresource = filename
@@ -15224,7 +15264,7 @@ function loaders.file(specification,filetype)
if trace_locating then
report_files("file loader, '%s' loaded",filename)
end
- local s = f:read("*a")
+ local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
if checkgarbage then
checkgarbage(#s)
end
diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun
index 75d09a583..96b153adf 100644
--- a/scripts/context/stubs/unix/mtxrun
+++ b/scripts/context/stubs/unix/mtxrun
@@ -2016,6 +2016,7 @@ if not modules then modules = { } end modules ['l-io'] = {
local io = io
local byte, find, gsub, format = string.byte, string.find, string.gsub, string.format
local concat = table.concat
+local floor = math.floor
local type = type
if string.find(os.getenv("PATH"),";") then
@@ -2024,10 +2025,49 @@ else
io.fileseparator, io.pathseparator = "/" , ":"
end
+local function readall(f)
+ return f:read("*all")
+end
+
+-- The next one is upto 50% faster on large files and less memory consumption due
+-- to less intermediate large allocations. This phenomena was discussed on the
+-- luatex dev list.
+
+local function readall(f)
+ local size = f:seek("end")
+ if size == 0 then
+ return ""
+ elseif size < 1024*1024 then
+ f:seek("set",0)
+ return f:read('*all')
+ else
+ local done = f:seek("set",0)
+ if size < 1024*1024 then
+ step = 1024 * 1024
+ elseif size > 16*1024*1024 then
+ step = 16*1024*1024
+ else
+ step = floor(size/(1024*1024)) * 1024 * 1024 / 8
+ end
+ local data = { }
+ while true do
+ local r = f:read(step)
+ if not r then
+ return concat(data)
+ else
+ data[#data+1] = r
+ end
+ end
+ end
+end
+
+io.readall = readall
+
function io.loaddata(filename,textmode) -- return nil if empty
local f = io.open(filename,(textmode and 'r') or 'rb')
if f then
- local data = f:read('*all')
+ -- local data = f:read('*all')
+ local data = readall(f)
f:close()
if #data > 0 then
return data
@@ -8520,13 +8560,13 @@ function xml.load(filename,settings)
local data = ""
if type(filename) == "string" then
-- local data = io.loaddata(filename) - -todo: check type in io.loaddata
- local f = io.open(filename,'r')
+ local f = io.open(filename,'r') -- why not 'rb'
if f then
- data = f:read("*all")
+ data = f:read("*all") -- io.readall(f) ... only makes sense for large files
f:close()
end
elseif filename then -- filehandle
- data = filename:read("*all")
+ data = filename:read("*all") -- io.readall(f) ... only makes sense for large files
end
if settings then
settings.currentresource = filename
@@ -15224,7 +15264,7 @@ function loaders.file(specification,filetype)
if trace_locating then
report_files("file loader, '%s' loaded",filename)
end
- local s = f:read("*a")
+ local s = f:read("*a") -- io.readall(f) is faster but we never have large files here
if checkgarbage then
checkgarbage(#s)
end