diff options
Diffstat (limited to 'tex')
52 files changed, 3789 insertions, 173 deletions
diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index 06c0d4626..253b3250a 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2018.08.10 16:51} +\newcontextversion{2018.08.14 23:10} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index 33917ad55..633142980 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2018.08.10 16:51} +\edef\contextversion{2018.08.14 23:10} %D For those who want to use this: diff --git a/tex/context/base/mkiv/anch-pos.lua b/tex/context/base/mkiv/anch-pos.lua index 99763edae..47fee067f 100644 --- a/tex/context/base/mkiv/anch-pos.lua +++ b/tex/context/base/mkiv/anch-pos.lua @@ -25,7 +25,7 @@ more efficient.</p> -- save much here (at least not now) local tostring, next, rawget, rawset, setmetatable, tonumber = tostring, next, rawget, rawset, setmetatable, tonumber -local sort, sortedhash, sortedkeys = table.sort, table.sortedhash, table.sortedkeys +local sort = table.sort local format, gmatch = string.format, string.gmatch local rawget = rawget local lpegmatch = lpeg.match @@ -46,6 +46,8 @@ local commands = commands local context = context local ctxnode = context.nodes.flush +local ctx_latelua = context.latelua + local tex = tex local texgetcount = tex.getcount local texsetcount = tex.setcount @@ -489,7 +491,8 @@ scanners.bposcolumnregistered = function() -- tag local tag = scanstring() insert(columns,tag) column = tag - ctxnode(new_latelua_node(function() b_column(tag) end)) +-- ctxnode(new_latelua_node(function() b_column(tag) end)) + ctx_latelua(function() b_column(tag) end) end scanners.eposcolumn = function() @@ -498,7 +501,8 @@ scanners.eposcolumn = function() end scanners.eposcolumnregistered = function() - ctxnode(new_latelua_node(e_column)) +-- ctxnode(new_latelua_node(e_column)) + ctx_latelua(e_column) remove(columns) column = columns[#columns] end @@ -635,7 +639,8 @@ scanners.parpos = function() -- todo: relate to localpar (so this is an intermed end local tag = f_p_tag(nofparagraphs) tobesaved[tag] = t - ctxnode(new_latelua_node(function() enhance(tobesaved[tag]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[tag]) end)) + ctx_latelua(function() enhance(tobesaved[tag]) end) end scanners.dosetposition = function() -- name @@ -649,7 +654,8 @@ scanners.dosetposition = function() -- name n = nofparagraphs > 0 and nofparagraphs or nil, r2l = texgetcount("inlinelefttoright") == 1 or nil, } - ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) + ctx_latelua(function() enhance(tobesaved[name]) end) end scanners.dosetpositionwhd = function() -- name w h d extra @@ -669,7 +675,8 @@ scanners.dosetpositionwhd = function() -- name w h d extra n = nofparagraphs > 0 and nofparagraphs or nil, r2l = texgetcount("inlinelefttoright") == 1 or nil, } - ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) + ctx_latelua(function() enhance(tobesaved[name]) end) end scanners.dosetpositionbox = function() -- name box @@ -688,7 +695,8 @@ scanners.dosetpositionbox = function() -- name box n = nofparagraphs > 0 and nofparagraphs or nil, r2l = texgetcount("inlinelefttoright") == 1 or nil, } - ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) + ctx_latelua(function() enhance(tobesaved[name]) end) end scanners.dosetpositionplus = function() -- name w h d extra @@ -709,7 +717,8 @@ scanners.dosetpositionplus = function() -- name w h d extra e = scanstring(), r2l = texgetcount("inlinelefttoright") == 1 or nil, } - ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) + ctx_latelua(function() enhance(tobesaved[name]) end) end scanners.dosetpositionstrut = function() -- name @@ -727,7 +736,8 @@ scanners.dosetpositionstrut = function() -- name n = nofparagraphs > 0 and nofparagraphs or nil, r2l = texgetcount("inlinelefttoright") == 1 or nil, } - ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) + ctx_latelua(function() enhance(tobesaved[name]) end) end scanners.dosetpositionstrutkind = function() -- name @@ -747,7 +757,8 @@ scanners.dosetpositionstrutkind = function() -- name n = nofparagraphs > 0 and nofparagraphs or nil, r2l = texgetcount("inlinelefttoright") == 1 or nil, } - ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) +-- ctxnode(new_latelua_node(function() enhance(tobesaved[name]) end)) + ctx_latelua(function() enhance(tobesaved[name]) end) end function jobpositions.getreserved(tag,n) diff --git a/tex/context/base/mkiv/attr-col.lua b/tex/context/base/mkiv/attr-col.lua index 28e63b177..f970fb8e7 100644 --- a/tex/context/base/mkiv/attr-col.lua +++ b/tex/context/base/mkiv/attr-col.lua @@ -406,10 +406,29 @@ function colors.setmodel(name,weightgray) weightgray = true end end - colors.model = name -- global, not useful that way - colors.default = models[name] or 1 -- global - colors.weightgray = weightgray -- global - return colors.default + local default = models[name] or 1 + + colors.model = name -- global, not useful that way + colors.default = default -- global + colors.weightgray = weightgray -- global + + -- avoid selective checking is no need for it + + local forced = colors.forced + + if forced == nil then + -- unset + colors.forced = default + elseif forced == false then + -- assumed mixed + elseif forced ~= default then + -- probably mixed + colors.forced = false + else + -- stil the same + end + + return default end function colors.register(name, colorspace, ...) -- passing 9 vars is faster (but not called that often) diff --git a/tex/context/base/mkiv/cldf-bas.lua b/tex/context/base/mkiv/cldf-bas.lua index 15e941db2..02f8e3e5b 100644 --- a/tex/context/base/mkiv/cldf-bas.lua +++ b/tex/context/base/mkiv/cldf-bas.lua @@ -31,10 +31,13 @@ local concat = table.concat local context = context local ctxcore = context.core local variables = interfaces.variables +local sprint = context.sprint local nodepool = nodes.pool local new_rule = nodepool.rule local new_glyph = nodepool.glyph +local new_latelua = nodepool.latelua + local current_attr = nodes.current_attr local current_font = font.current @@ -172,3 +175,7 @@ context.registers = { -- not really a register but kind of belongs here newchar = function(name,u) context([[\chardef\%s=%s\relax]],name,u) end, } + +function context.latelua(f) + sprint(new_latelua(f)) -- maybe just context +end diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 0de7057d6..84022b992 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2018.08.10 16:51} +\newcontextversion{2018.08.14 23:10} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index bc599f373..357707a14 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -42,7 +42,7 @@ %D has to match \type {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2018.08.10 16:51} +\edef\contextversion{2018.08.14 23:10} \edef\contextkind {beta} %D For those who want to use this: @@ -99,6 +99,7 @@ \loadmarkfile{luat-cod} \loadmarkfile{luat-bas} \loadmarkfile{luat-lib} +\loadmarkfile{luat-soc} \loadmarkfile{catc-ini} \loadmarkfile{catc-act} diff --git a/tex/context/base/mkiv/core-con.mkiv b/tex/context/base/mkiv/core-con.mkiv index d0e53833d..3ac33f46f 100644 --- a/tex/context/base/mkiv/core-con.mkiv +++ b/tex/context/base/mkiv/core-con.mkiv @@ -946,4 +946,23 @@ data {#1}% \relax} +%D For those who sart counting at zero: +%D +%D \starttyping +%D \defineconversionset [zero] [n,zero] [n] +%D +%D \setuphead [sectionconversionset=zero] +%D +%D \starttext +%D \startchapter [title=Introduction] +%D \startsection [title=First topic] \stopsection +%D \startsection [title=Second topic] \stopsection +%D \stopchapter +%D \stoptext +%D \stoptyping + +\def\zeronumberconversion#1{\number\numexpr#1-\plusone\relax} + +\defineconversion [zero] [\zeronumberconversion] + \protect \endinput diff --git a/tex/context/base/mkiv/core-two.lua b/tex/context/base/mkiv/core-two.lua index 1e59004be..3ab2112b9 100644 --- a/tex/context/base/mkiv/core-two.lua +++ b/tex/context/base/mkiv/core-two.lua @@ -32,7 +32,7 @@ end job.register('job.passes.collected', tobesaved, initializer, nil) -local function allocate(id) +local function define(id) local p = tobesaved[id] if not p then p = { } @@ -41,10 +41,8 @@ local function allocate(id) return p end -jobpasses.define = allocate - -function jobpasses.save(id,str,index) - local jti = allocate(id) +local function save(id,str,index) + local jti = define(id) if index then jti[index] = str else @@ -52,30 +50,30 @@ function jobpasses.save(id,str,index) end end -function jobpasses.savetagged(id,tag,str) - local jti = allocate(id) +local function savetagged(id,tag,str) + local jti = define(id) jti[tag] = str end -function jobpasses.getdata(id,index,default) +local function getdata(id,index,default) local jti = collected[id] local value = jti and jti[index] return value ~= "" and value or default or "" end -function jobpasses.getfield(id,index,tag,default) +local function getfield(id,index,tag,default) local jti = collected[id] jti = jti and jti[index] local value = jti and jti[tag] return value ~= "" and value or default or "" end -function jobpasses.getcollected(id) +local function getcollected(id) return collected[id] or { } end -function jobpasses.gettobesaved(id) - return allocate(id) +local function gettobesaved(id) + return define(id) end local function get(id) @@ -87,23 +85,17 @@ end local function first(id) local jti = collected[id] - if jti and #jti > 0 then - return jti[1] - end + return jti and jti[1] end local function last(id) local jti = collected[id] - if jti and #jti > 0 then - return jti[#jti] - end + return jti and jti[#jti] end local function find(id,n) local jti = collected[id] - if jti and jti[n] then - return jti[n] - end + return jti and jti[n] or nil end local function count(id) @@ -132,44 +124,49 @@ end local check = first --- - -jobpasses.get = get -jobpasses.first = first -jobpasses.last = last -jobpasses.find = find -jobpasses.list = list -jobpasses.count = count -jobpasses.check = check -jobpasses.inlist = inlist +jobpasses.define = define +jobpasses.save = save +jobpasses.savetagged = savetagged +jobpasses.getdata = getdata +jobpasses.getfield = getfield +jobpasses.getcollected = getcollected +jobpasses.gettobesaved = gettobesaved +jobpasses.get = get +jobpasses.first = first +jobpasses.last = last +jobpasses.find = find +jobpasses.list = list +jobpasses.count = count +jobpasses.check = check +jobpasses.inlist = inlist -- interface local implement = interfaces.implement -implement { name = "gettwopassdata", actions = { get , context }, arguments = "string" } +implement { name = "gettwopassdata", actions = { get, context }, arguments = "string" } implement { name = "getfirsttwopassdata",actions = { first, context }, arguments = "string" } -implement { name = "getlasttwopassdata", actions = { last , context }, arguments = "string" } -implement { name = "findtwopassdata", actions = { find , context }, arguments = "2 strings" } -implement { name = "gettwopassdatalist", actions = { list , context }, arguments = "string" } +implement { name = "getlasttwopassdata", actions = { last, context }, arguments = "string" } +implement { name = "findtwopassdata", actions = { find, context }, arguments = "2 strings" } +implement { name = "gettwopassdatalist", actions = { list, context }, arguments = "string" } implement { name = "counttwopassdata", actions = { count, context }, arguments = "string" } implement { name = "checktwopassdata", actions = { check, context }, arguments = "string" } implement { name = "definetwopasslist", - actions = jobpasses.define, + actions = define, arguments = "string" } implement { name = "savetwopassdata", - actions = jobpasses.save, + actions = save, arguments = "2 strings", } implement { name = "savetaggedtwopassdata", - actions = jobpasses.savetagged, + actions = savetagged, arguments = "3 strings", } @@ -178,3 +175,23 @@ implement { actions = { inlist, commands.doifelse }, arguments = "2 strings", } + +-- local ctx_latelua = context.latelua + +-- implement { +-- name = "lazysavetwopassdata", +-- arguments = "3 strings", +-- public = true, +-- actions = function(a,b,c) +-- ctx_latelua(function() save(a,c) end) +-- end, +-- } + +-- implement { +-- name = "lazysavetaggedtwopassdata", +-- arguments = "3 strings", +-- public = true, +-- actions = function(a,b,c) +-- ctx_latelua(function() savetagged(a,b,c) end) +-- end, +-- } diff --git a/tex/context/base/mkiv/core-two.mkiv b/tex/context/base/mkiv/core-two.mkiv index f83d63042..aae4902bc 100644 --- a/tex/context/base/mkiv/core-two.mkiv +++ b/tex/context/base/mkiv/core-two.mkiv @@ -74,10 +74,10 @@ \registerctxluafile{core-two}{} \def\immediatesavetwopassdata #1#2#3{\normalexpanded{\noexpand\clf_savetwopassdata{#1}{#3}}} -\def\savetwopassdata #1#2#3{\normalexpanded{\noexpand\ctxlatecommand{savetwopassdata('#1',"#3")}}} -\def\lazysavetwopassdata #1#2#3{\normalexpanded{\noexpand\ctxlatecommand{savetwopassdata('#1',"#3")}}} -\def\savetaggedtwopassdata #1#2#3#4{\normalexpanded{\noexpand\clf_savetaggedtwopassdata{#1}{#3}{#4}}} -\def\lazysavetaggedtwopassdata#1#2#3#4{\normalexpanded{\noexpand\ctxlatecommand{savetaggedtwopassdata('#1','#3',"#4")}}} +\def \lazysavetwopassdata #1#2#3{\normalexpanded{\noexpand\ctxlatecommand{savetwopassdata("#1","#3")}}} +\let \savetwopassdata \lazysavetwopassdata +\def \savetaggedtwopassdata#1#2#3#4{\normalexpanded{\noexpand\clf_savetaggedtwopassdata{#1}{#3}{#4}}} +\def\lazysavetaggedtwopassdata#1#2#3#4{\normalexpanded{\noexpand\ctxlatecommand{savetaggedtwopassdata("#1",'#3',"#4")}}} % temp hack: needs a proper \starteverytimeluacode diff --git a/tex/context/base/mkiv/data-res.lua b/tex/context/base/mkiv/data-res.lua index e3c5c32b9..0c2735fc2 100644 --- a/tex/context/base/mkiv/data-res.lua +++ b/tex/context/base/mkiv/data-res.lua @@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['data-res'] = { -- todo: cache:/// home:/// selfautoparent:/// (sometime end 2012) local gsub, find, lower, upper, match, gmatch = string.gsub, string.find, string.lower, string.upper, string.match, string.gmatch -local concat, insert, remove, sortedkeys, sortedhash = table.concat, table.insert, table.remove, table.sortedkeys, table.sortedhash +local concat, insert, remove = table.concat, table.insert, table.remove local next, type, rawget = next, type, rawget local os = os diff --git a/tex/context/base/mkiv/font-map.lua b/tex/context/base/mkiv/font-map.lua index 140702ec8..a7fbfe49e 100644 --- a/tex/context/base/mkiv/font-map.lua +++ b/tex/context/base/mkiv/font-map.lua @@ -214,7 +214,51 @@ local unknown = f_single(0xFFFD) -- end -- end -local hash = table.setmetatableindex(function(t,k) +-- local hash = table.setmetatableindex(function(t,k) +-- local v +-- if k >= 0x00E000 and k <= 0x00F8FF then +-- v = unknown +-- elseif k >= 0x0F0000 and k <= 0x0FFFFF then +-- v = unknown +-- elseif k >= 0x100000 and k <= 0x10FFFF then +-- v = unknown +-- elseif k < 0xD7FF or (k > 0xDFFF and k <= 0xFFFF) then +-- v = f_single(k) +-- else +-- k = k - 0x10000 +-- v = f_double(rshift(k,10)+0xD800,k%1024+0xDC00) +-- end +-- t[k] = v +-- return v +-- end) +-- +-- table.makeweak(hash) +-- +-- local function tounicode(unicode) +-- if type(unicode) == "table" then +-- local t = { } +-- for l=1,#unicode do +-- t[l] = hash[unicode[l]] +-- end +-- return concat(t) +-- else +-- return hash[unicode] +-- end +-- end + +local hash = { } +local conc = { } + +-- table.makeweak(hash) + +table.setmetatableindex(hash,function(t,k) + if type(k) == "table" then + local n = #k + for l=1,n do + conc[l] = hash[k[l]] + end + return concat(conc,"",1,n) + end local v if k >= 0x00E000 and k <= 0x00F8FF then v = unknown @@ -232,18 +276,8 @@ local hash = table.setmetatableindex(function(t,k) return v end) -table.makeweak(hash) - -local function tounicode(unicode,name) - if type(unicode) == "table" then - local t = { } - for l=1,#unicode do - t[l] = hash[unicode[l]] - end - return concat(t) - else - return hash[unicode] - end +local function tounicode(unicode) + return hash[unicode] end local function fromunicode16(str) diff --git a/tex/context/base/mkiv/l-lpeg.lua b/tex/context/base/mkiv/l-lpeg.lua index 827564464..750d5e698 100644 --- a/tex/context/base/mkiv/l-lpeg.lua +++ b/tex/context/base/mkiv/l-lpeg.lua @@ -1135,7 +1135,7 @@ end do - local trailingzeros = zero^0 * -digit -- suggested by Roberto R + local trailingzeros = zero^0 * -digit -- suggested by Roberto local stripper = Cs(( digits * ( period * trailingzeros / "" @@ -1145,15 +1145,15 @@ do lpeg.patterns.stripzeros = stripper -- multiple in string - local nonzero = digit - zero - + local nonzero = digit - zero local trailingzeros = zero^1 * endofstring local stripper = Cs( (1-period)^0 * ( - (period * trailingzeros/"") + - period * (nonzero^1 + (trailingzeros/"") + zero^1)^0 + period * trailingzeros/"" + + period * (nonzero^1 + (trailingzeros/"") + zero^1)^0 + + endofstring )) - lpeg.patterns.stripzero = stripper -- slightly more efficient + lpeg.patterns.stripzero = stripper -- slightly more efficient but expects a float ! -- local sample = "bla 11.00 bla 11 bla 0.1100 bla 1.00100 bla 0.00 bla 0.001 bla 1.1100 bla 0.100100100 bla 0.00100100100" -- collectgarbage("collect") @@ -1164,7 +1164,7 @@ do end --- for practical reasone we keep this here: +-- for practical reasons we keep this here: local byte_to_HEX = { } local byte_to_hex = { } diff --git a/tex/context/base/mkiv/lpdf-ini.lua b/tex/context/base/mkiv/lpdf-ini.lua index 79ed3470c..c53d90848 100644 --- a/tex/context/base/mkiv/lpdf-ini.lua +++ b/tex/context/base/mkiv/lpdf-ini.lua @@ -447,13 +447,6 @@ do end -local function merge_t(a,b) - local t = { } - for k,v in next, a do t[k] = v end - for k,v in next, b do t[k] = v end - return setmetatable(t,getmetatable(a)) -end - local tostring_a, tostring_d do diff --git a/tex/context/base/mkiv/lpdf-wid.lua b/tex/context/base/mkiv/lpdf-wid.lua index 8647a7b39..11cd80623 100644 --- a/tex/context/base/mkiv/lpdf-wid.lua +++ b/tex/context/base/mkiv/lpdf-wid.lua @@ -239,10 +239,12 @@ local function flushembeddedfiles() e[#e+1] = pdfstring(tag) e[#e+1] = reference -- already a reference else - -- messy spec ... when annot not in named else twice in menu list acrobat + -- -- messy spec ... when annot not in named else twice in menu list acrobat end end - lpdf.addtonames("EmbeddedFiles",pdfreference(pdfflushobject(pdfdictionary{ Names = e }))) + if #e > 0 then + lpdf.addtonames("EmbeddedFiles",pdfreference(pdfflushobject(pdfdictionary{ Names = e }))) + end end end @@ -612,7 +614,8 @@ local function insertrendering(specification) local option = settings_to_hash(specification.option) if not mf[label] then local filename = specification.filename - local isurl = find(filename,"://",1,true) + local isurl = find(filename,"://",1,true) + local mimetype = specification.mimetype or specification.mime -- local start = pdfdictionary { -- Type = pdfconstant("MediaOffset"), -- S = pdfconstant("T"), -- time @@ -648,13 +651,16 @@ local function insertrendering(specification) if isurl then descriptor.FS = pdfconstant("URL") elseif option[v_embed] then - descriptor.EF = codeinjections.embedfile { file = filename } + descriptor.EF = codeinjections.embedfile { + file = filename, + mimetype = mimetype, -- yes or no + } end local clip = pdfdictionary { Type = pdfconstant("MediaClip"), S = pdfconstant("MCD"), N = label, - CT = specification.mime, + CT = mimetype, Alt = pdfarray { "", "file not found" }, -- language id + message D = pdfreference(pdfflushobject(descriptor)), -- P = pdfreference(pdfflushobject(parameters)), diff --git a/tex/context/base/mkiv/luat-cod.lua b/tex/context/base/mkiv/luat-cod.lua index f74c53e82..790f741c1 100644 --- a/tex/context/base/mkiv/luat-cod.lua +++ b/tex/context/base/mkiv/luat-cod.lua @@ -53,7 +53,9 @@ local strip = false if arg then for i=-1,#arg do if arg[i] == "--c:strip" then s function lua.registercode(filename,options) local barename = gsub(filename,"%.[%a%d]+$","") - if barename == filename then filename = filename .. ".lua" end + if barename == filename then + filename = filename .. ".lua" + end local basename = match(barename,"^.+[/\\](.-)$") or barename if not bytedone[basename] then local opts = { } @@ -157,12 +159,14 @@ environment.initexmode = INITEXMODE if not environment.luafilechunk then function environment.luafilechunk(filename) + local fullname = filename if sourcepath ~= "" then - filename = sourcepath .. "/" .. filename + fullname = sourcepath .. "/" .. filename end - local data = loadfile(filename) - texio.write("term and log","<",data and "+ " or "- ",filename,">") + local data = loadfile(fullname) + texio.write("term and log","<",data and "+ " or "- ",fullname,">") if data then +-- package.loaded[gsub(filename,"%..-$"] = data() end return data diff --git a/tex/context/base/mkiv/luat-soc.lua b/tex/context/base/mkiv/luat-soc.lua deleted file mode 100644 index 9342a4b33..000000000 --- a/tex/context/base/mkiv/luat-soc.lua +++ /dev/null @@ -1,11 +0,0 @@ --- This is just a loader. The package handler knows about the TEX tree. - --- require "luatex/lua/socket.lua" --- require "luatex/lua/ltn12.lua" --- require "luatex/lua/mime.lua" --- require "luatex/lua/socket/http.lua" --- require "luatex/lua/socket/url.lua" --- require "luatex/lua/socket/tp.lua" --- require "luatex/lua/socket/ftp.lua" - --- "luatex/lua/socket/smtp.lua" diff --git a/tex/context/base/mkiv/luat-soc.mkiv b/tex/context/base/mkiv/luat-soc.mkiv new file mode 100644 index 000000000..e17ff22d3 --- /dev/null +++ b/tex/context/base/mkiv/luat-soc.mkiv @@ -0,0 +1,52 @@ +%D \module +%D [ file=luat-soc, +%D version=2018.08.05, +%D title=\CONTEXT\ Lua Macros, +%D subtitle=Socket Libraries, +%D author=Hans Hagen, +%D date=\currentdate, +%D copyright={PRAGMA ADE \& \CONTEXT\ Development Team}] +%C +%C This module is part of the \CONTEXT\ macro||package and is +%C therefore copyrighted by \PRAGMA. See mreadme.pdf for +%C details. + +\writestatus{loading}{ConTeXt Lua Macros / Socket Libraries} + +%D In \LUATEX\ we provide the socket library that is more or less the standard one +%D for \LUA. It has been around for a while and seems to be pretty stable. The +%D binary module is copmpiled into \LUATEX\ and the accompanying .lua files are +%D preloaded. These files are mostly written by Diego Nehab, Andre Carregal, Javier +%D Guerra, and Fabio Mascarenhas with contributions from Diego Nehab, Mike Pall, +%D David Burgess, Leonardo Godinho, Thomas Harning Jr., and Gary NG. The originals +%D are part of and copyrighted by the Kepler project. +%D +%D Here we reload a slightly reworked version of these \type {.lua} files. We keep +%D the same (documented) interface but streamlined some fo the code. No more +%D modules, no more pre 5.2 \LUA, etc. Also, as it loads into the \CONTEXT +%D ecosystem, we plug in some logging. (and maybe tracing in the future). As we +%D don't support serial ports in \LUATEX, related code has been dropped. +%D +%D The files are reformatted so that we can more easilly add additional features +%D and|/|or tracing options. Any error introduced there is our fault! The url module +%D might be replaced by the one in \CONTEXT. When we need mbox a suitable variant +%D will be provided. + +%D Currently we preload the related \LUA\ code in \LUATEX, but that might change at +%D some point. We're prepared for that. + +\registerctxluafile{util-soc-imp-reset} {} + +\registerctxluafile{util-soc-imp-socket} {} +%registerctxluafile{util-soc-imp-copas} {} +\registerctxluafile{util-soc-imp-ltn12} {} +%registerctxluafile{util-soc-imp-mbox} {} +\registerctxluafile{util-soc-imp-mime} {} +\registerctxluafile{util-soc-imp-url} {} +\registerctxluafile{util-soc-imp-headers}{} +\registerctxluafile{util-soc-imp-http} {} +\registerctxluafile{util-soc-imp-tp} {} +%registerctxluafile{util-soc-imp-ftp} {} +%registerctxluafile{util-soc-imp-smtp} {} + +\endinput diff --git a/tex/context/base/mkiv/mlib-pps.lua b/tex/context/base/mkiv/mlib-pps.lua index 05c29dad3..0c52aa0b9 100644 --- a/tex/context/base/mkiv/mlib-pps.lua +++ b/tex/context/base/mkiv/mlib-pps.lua @@ -101,9 +101,10 @@ local f_f3 = formatters["%.3F"] local f_gray = formatters["%.3F g %.3F G"] local f_rgb = formatters["%.3F %.3F %.3F rg %.3F %.3F %.3F RG"] local f_cmyk = formatters["%.3F %.3F %.3F %.3F k %.3F %.3F %.3F %.3F K"] -local f_cm_b = formatters["q %F %F %F %F %F %F cm"] -local f_shade = formatters["MpSh%s"] +local f_cm_b = formatters["q %.6F %.6F %.6F %.6F %.6F %.6F cm"] +local f_shade = formatters["MpSh%s"] +local f_spot = formatters["/%s cs /%s CS %s SCN %s scn"] local s_cm_e = "Q" directives.register("metapost.stripzeros",function() @@ -112,12 +113,9 @@ directives.register("metapost.stripzeros",function() f_gray = formatters["%.3N g %.3N G"] f_rgb = formatters["%.3N %.3N %.3N rg %.3N %.3N %.3N RG"] f_cmyk = formatters["%.3N %.3N %.3N %.3N k %.3N %.3N %.3N %.3N K"] - f_cm_b = formatters["q %N %N %N %N %N %N cm"] - f_shade = formatters["MpSh%s"] + f_cm_b = formatters["q %.6N %.6N %.6N %.6N %.6N %.6N cm"] end) -local f_spot = formatters["/%s cs /%s CS %s SCN %s scn"] - local function checked_color_pair(color,...) if not color then return innercolor, outercolor diff --git a/tex/context/base/mkiv/mult-prm.lua b/tex/context/base/mkiv/mult-prm.lua index dd19e40b8..77c53beb9 100644 --- a/tex/context/base/mkiv/mult-prm.lua +++ b/tex/context/base/mkiv/mult-prm.lua @@ -251,6 +251,7 @@ return { "expandglyphsinfont", "explicitdiscretionary", "explicithyphenpenalty", + "fixupboxesmode", "fontid", "formatname", "gleaders", @@ -448,6 +449,7 @@ return { "pdfnormaldeviate", "pdfobj", "pdfobjcompresslevel", + "pdfomitcharset", "pdfomitcidset", "pdfoutline", "pdfoutput", diff --git a/tex/context/base/mkiv/node-fin.lua b/tex/context/base/mkiv/node-fin.lua index 3139263ab..3e7a4cd1b 100644 --- a/tex/context/base/mkiv/node-fin.lua +++ b/tex/context/base/mkiv/node-fin.lua @@ -16,10 +16,8 @@ local next, type, format = next, type, string.format local attributes, nodes, node = attributes, nodes, node local nuts = nodes.nuts -local tonut = nuts.tonut local getnext = nuts.getnext -local getprev = nuts.getprev local getid = nuts.getid local getlist = nuts.getlist local getleader = nuts.getleader @@ -605,7 +603,6 @@ local function stacked(attribute,head,default) -- no triggering, no inheritance, elseif id == rule_code then check = getwidth(stack) ~= 0 end - if check then local a = getattr(stack,attribute) if a then @@ -698,7 +695,7 @@ local function stacker(attribute,head,default) -- no triggering, no inheritance, end local n = nsstep(a) if n then - head = insert_node_before(head,current,tonut(n)) -- a + head = insert_node_before(head,current,n) -- a end attrib = a if leader then @@ -717,7 +714,7 @@ local function stacker(attribute,head,default) -- no triggering, no inheritance, if stacked then local n = nsend() while n do - head = insert_node_after(head,previous,tonut(n)) + head = insert_node_after(head,previous,n) n = nsend() end end @@ -780,7 +777,7 @@ end -- end -- local n = nsstep(a) -- if n then --- head = insert_node_before(head,current,tonut(n)) -- a +-- head = insert_node_before(head,current,n) -- a -- end -- attrib = a -- if leader then @@ -800,7 +797,7 @@ end -- if stacked then -- local n = nsend() -- while n do --- head = insert_node_after(head,previous,tonut(n)) +-- head = insert_node_after(head,previous,n) -- n = nsend() -- end -- end diff --git a/tex/context/base/mkiv/node-ref.lua b/tex/context/base/mkiv/node-ref.lua index 27d209701..da72337b8 100644 --- a/tex/context/base/mkiv/node-ref.lua +++ b/tex/context/base/mkiv/node-ref.lua @@ -444,6 +444,108 @@ local function inject_areas(head,attribute,make,stack,done,skip,parent,pardir,tx return head, pardir, txtdir end +-- -- not faster either: +-- +-- local findattr = node.direct.find_attribute +-- +-- local function inject_areas(head,attribute,make,stack,done,skip,parent,pardir,txtdir) -- main +-- local first, last, firstdir, reference +-- if not pardir then +-- pardir = "===" +-- end +-- if not texdir then +-- txtdir = "===" +-- end +-- local someatt = findattr(head,attribute) +-- if someatt then +-- local current = head +-- while current do +-- local id = getid(current) +-- if id == hlist_code or id == vlist_code then +-- local r = getattr(current,attribute) +-- -- test \goto{test}[page(2)] test \gotobox{test}[page(2)] +-- -- test \goto{\TeX}[page(2)] test \gotobox{\hbox {x} \hbox {x}}[page(2)] +-- -- if r and (not skip or r >) skip then -- maybe no > test +-- -- inject_list(id,current,r,make,stack,pardir,txtdir) +-- -- end +-- if r then +-- if not reference then +-- reference, first, last, firstdir = r, current, current, txtdir +-- elseif r == reference then +-- -- same link +-- last = current +-- elseif (done[reference] or 0) == 0 then +-- if not skip or r > skip then -- maybe no > test +-- head, current = inject_range(head,first,last,reference,make,stack,parent,pardir,firstdir) +-- reference, first, last, firstdir = nil, nil, nil, nil +-- end +-- else +-- reference, first, last, firstdir = r, current, current, txtdir +-- end +-- done[r] = (done[r] or 0) + 1 +-- end +-- local list = getlist(current) +-- if list then +-- local h +-- h, pardir, txtdir = inject_areas(list,attribute,make,stack,done,r or skip or 0,current,pardir,txtdir) +-- if h ~= current then +-- setlist(current,h) +-- end +-- end +-- if r then +-- done[r] = done[r] - 1 +-- end +-- elseif id == dir_code then +-- txtdir = getdir(current) +-- elseif id == localpar_code then -- only test at begin +-- pardir = getdir(current) +-- elseif id == glue_code and getsubtype(current) == leftskip_code then -- any glue at the left? +-- -- +-- else +-- local r = getattr(current,attribute) +-- if not r then +-- -- just go on, can be kerns +-- elseif not reference then +-- reference, first, last, firstdir = r, current, current, txtdir +-- elseif r == reference then +-- last = current +-- elseif (done[reference] or 0) == 0 then -- or id == glue_code and getsubtype(current) == right_skip_code +-- if not skip or r > skip then -- maybe no > test +-- head, current = inject_range(head,first,last,reference,make,stack,parent,pardir,firstdir) +-- reference, first, last, firstdir = nil, nil, nil, nil +-- end +-- else +-- reference, first, last, firstdir = r, current, current, txtdir +-- end +-- end +-- current = getnext(current) +-- end +-- if reference and (done[reference] or 0) == 0 then +-- head = inject_range(head,first,last,reference,make,stack,parent,pardir,firstdir) +-- end +-- else +-- local current = head +-- while current do +-- local id = getid(current) +-- if id == hlist_code or id == vlist_code then +-- local list = getlist(current) +-- if list then +-- local h = inject_areas(list,attribute,make,stack,done,skip or 0,current,pardir,txtdir) +-- if h ~= current then +-- setlist(current,h) +-- end +-- end +-- elseif id == dir_code then +-- txtdir = getdir(current) +-- elseif id == localpar_code then -- only test at begin +-- pardir = getdir(current) +-- end +-- current = getnext(current) +-- end +-- end +-- return head, pardir, txtdir +-- end + -- -- maybe first check for glyphs and use a goto: -- -- local function inject_areas(head,attribute,make,stack,done,skip,parent,pardir,txtdir) -- main diff --git a/tex/context/base/mkiv/node-res.lua b/tex/context/base/mkiv/node-res.lua index 39d47f647..b591cafdf 100644 --- a/tex/context/base/mkiv/node-res.lua +++ b/tex/context/base/mkiv/node-res.lua @@ -428,7 +428,8 @@ end function nutpool.latelua(code) local n = copy_nut(latelua) - setfield(n,"string",code) + -- setfield(n,"string",code) + setdata(n,code) return n end diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf Binary files differindex afcce6971..e95010141 100644 --- a/tex/context/base/mkiv/status-files.pdf +++ b/tex/context/base/mkiv/status-files.pdf diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf Binary files differindex 2eb9c7104..5e0c029e7 100644 --- a/tex/context/base/mkiv/status-lua.pdf +++ b/tex/context/base/mkiv/status-lua.pdf diff --git a/tex/context/base/mkiv/strc-con.mkvi b/tex/context/base/mkiv/strc-con.mkvi index a5360f52a..ed7e401f0 100644 --- a/tex/context/base/mkiv/strc-con.mkvi +++ b/tex/context/base/mkiv/strc-con.mkvi @@ -1093,7 +1093,7 @@ \endgroup \edef\noexpand\currentconstructionlistentry {\the\scratchcounter}% \edef\noexpand\currentconstructionattribute {\the\lastdestinationattribute}% - \edef\noexpand\currentconstructionsynchronize{\ctxlatecommand{enhancelist(\the\scratchcounter)}}% + \edef\noexpand\currentconstructionsynchronize{\clf_deferredenhancelist\scratchcounter}% }% \fi} @@ -1103,11 +1103,11 @@ \def\reinstateconstructionnumberentry#1% was xdef {\edef\currentconstructionattribute {\clf_getinternallistreference#1}% - \edef\currentconstructionsynchronize{\ctxlatecommand{enhancelist(#1)}}} + \edef\currentconstructionsynchronize{\clf_deferredenhancelist#1}} \def\reinstatecachedconstructionnumberentry#1% was xdef | #1 = cached index can be different from real {\edef\currentconstructionattribute {\clf_getinternalcachedlistreference#1}% destination - \edef\currentconstructionsynchronize{\ctxlatecommand{enhancelist(#1)}}} + \edef\currentconstructionsynchronize{\clf_deferredenhancelist#1}} \installstructurelistprocessor{construction}{\usestructurelistprocessor{number+title}} diff --git a/tex/context/base/mkiv/strc-lst.lua b/tex/context/base/mkiv/strc-lst.lua index 13bdf7786..1d2a8bb39 100644 --- a/tex/context/base/mkiv/strc-lst.lua +++ b/tex/context/base/mkiv/strc-lst.lua @@ -35,6 +35,8 @@ local commands = commands local implement = interfaces.implement local conditionals = tex.conditionals +local ctx_latelua = context.latelua + local structures = structures local lists = structures.lists local sections = structures.sections @@ -307,7 +309,7 @@ local synchronizepage = function(r) -- bah ... will move return synchronizepage(r) end -function lists.enhance(n) +local function enhancelist(n) local l = cached[n] if not l then report_lists("enhancing %a, unknown internal",n) @@ -348,6 +350,8 @@ function lists.enhance(n) end end +lists.enhance = enhancelist + -- we can use level instead but we can also decide to remove level from the metadata local nesting = { } @@ -1071,8 +1075,16 @@ implement { implement { name = "enhancelist", - actions = lists.enhance, - arguments = "integer" + arguments = "integer", + actions = enhancelist, +} + +implement { + name = "deferredenhancelist", + arguments = "integer", + actions = function(n) + ctx_latelua(function() enhancelist(n) end) + end, } implement { diff --git a/tex/context/base/mkiv/strc-lst.mkvi b/tex/context/base/mkiv/strc-lst.mkvi index c450e1884..5f6512b82 100644 --- a/tex/context/base/mkiv/strc-lst.mkvi +++ b/tex/context/base/mkiv/strc-lst.mkvi @@ -145,8 +145,9 @@ {\endgroup} % \unexpanded + \def\strc_lists_inject_enhance#listindex#internal% - {\normalexpanded{\ctxlatecommand{enhancelist(\number#listindex)}}} + {\normalexpanded{\clf_deferredenhancelist#listindex}} \unexpanded\def\strc_lists_inject_yes[#settings][#userdata]% can be used directly {\setupcurrentlist[\c!type=userdata,\c!location=\v!none,#settings]% grouped (use \let... @@ -168,7 +169,7 @@ userdata {\detokenize\expandafter{\normalexpanded{#userdata}}} \relax \edef\currentlistnumber{\the\scratchcounter}% -\setxvalue{\??listlocations\currentlist}{\the\locationcount}% + \setxvalue{\??listlocations\currentlist}{\the\locationcount}% \ifx\p_location\v!here % this branch injects nodes ! \strc_lists_inject_enhance{\currentlistnumber}{\the\locationcount}% diff --git a/tex/context/base/mkiv/strc-ref.lua b/tex/context/base/mkiv/strc-ref.lua index 2c9765a44..951c9a44a 100644 --- a/tex/context/base/mkiv/strc-ref.lua +++ b/tex/context/base/mkiv/strc-ref.lua @@ -52,6 +52,8 @@ local context = context local commands = commands local implement = interfaces.implement +local ctx_latelua = context.latelua + local texgetcount = tex.getcount local texsetcount = tex.setcount local texconditionals = tex.conditionals @@ -436,17 +438,27 @@ end references.synchronizepage = synchronizepage -function references.enhance(prefix,tag) +local function enhancereference(prefix,tag) local l = tobesaved[prefix][tag] if l then synchronizepage(l.references) end end +references.enhance = enhancereference + +-- implement { +-- name = "enhancereference", +-- arguments = "2 strings", +-- actions = references.enhance, +-- } + implement { - name = "enhancereference", - actions = references.enhance, + name = "deferredenhancereference", arguments = "2 strings", + actions = function(prefix,tag) + ctx_latelua(function() enhancereference(prefix,tag) end) + end, } -- -- -- related to strc-ini.lua -- -- -- diff --git a/tex/context/base/mkiv/strc-ref.mkvi b/tex/context/base/mkiv/strc-ref.mkvi index 8b887754c..0ae2cfccc 100644 --- a/tex/context/base/mkiv/strc-ref.mkvi +++ b/tex/context/base/mkiv/strc-ref.mkvi @@ -128,7 +128,7 @@ \newcount\lastdestinationattribute \def\strc_references_finish#prefix#reference#internal% - {\normalexpanded{\ctxlatecommand{enhancereference("#prefix","#reference")}}} + {\normalexpanded{\clf_deferredenhancereference{#prefix}{#reference}}} \let\dofinishreference\strc_references_finish % used at lua end diff --git a/tex/context/base/mkiv/strc-reg.lua b/tex/context/base/mkiv/strc-reg.lua index 919290c8f..61e13e7e4 100644 --- a/tex/context/base/mkiv/strc-reg.lua +++ b/tex/context/base/mkiv/strc-reg.lua @@ -48,6 +48,7 @@ local v_last = variables.last local v_text = variables.text local context = context +local ctx_latelua = context.latelua local implement = interfaces.implement @@ -564,9 +565,7 @@ local function storeregister(rawdata) -- metadata, references, entries return #entries end -registers.store = storeregister - -function registers.enhance(name,n) +local function enhanceregister(name,n) local data = tobesaved[name].metadata.notsaved and collected[name] or tobesaved[name] local entry = data.entries[n] if entry then @@ -574,7 +573,7 @@ function registers.enhance(name,n) end end -function registers.extend(name,tag,rawdata) -- maybe do lastsection internally +local function extendregister(name,tag,rawdata) -- maybe do lastsection internally if type(tag) == "string" then tag = tagged[tag] end @@ -618,6 +617,10 @@ function registers.extend(name,tag,rawdata) -- maybe do lastsection internally end end +registers.store = storeregister +registers.enhance = enhanceregister +registers.extend = extendregister + function registers.get(tag,n) local list = tobesaved[tag] return list and list.entries[n] @@ -625,13 +628,21 @@ end implement { name = "enhanceregister", - actions = registers.enhance, arguments = { "string", "integer" }, + actions = enhanceregister, +} + +implement { + name = "deferredenhanceregister", + arguments = { "string", "integer" }, + actions = function(name,n) + ctx_latelua(function() enhanceregister(name,n) end) + end, } implement { name = "extendregister", - actions = registers.extend, + actions = extendregister, arguments = "2 strings", } diff --git a/tex/context/base/mkiv/strc-reg.mkiv b/tex/context/base/mkiv/strc-reg.mkiv index 3f8331745..1b77f135f 100644 --- a/tex/context/base/mkiv/strc-reg.mkiv +++ b/tex/context/base/mkiv/strc-reg.mkiv @@ -305,7 +305,7 @@ \ifx\currentregisterownnumber\v!yes \glet\currentregistersynchronize\relax \else - \xdef\currentregistersynchronize{\ctxlatecommand{enhanceregister("\currentregister",\currentregisternumber)}}% + \xdef\currentregistersynchronize{\clf_deferredenhanceregister{\currentregister}\currentregisternumber}% \fi \currentregistersynchronize % here? % needs thinking ... bla\index{bla}. will break before the . but adding a @@ -341,7 +341,7 @@ % internal \locationcount % view {\interactionparameter\c!focus}% \relax % this will change - \xdef\currentregistersynchronize{\ctxlatecommand{enhanceregister("\currentregister",\currentregisternumber)}}% + \xdef\currentregistersynchronize{\clf_deferredenhanceregister{\currentregister}\currentregisternumber}% \currentregistersynchronize % here? \dostarttagged\t!registerlocation\currentregister \attribute\destinationattribute\lastdestinationattribute \signalcharacter % no \strut as it will be removed during cleanup diff --git a/tex/context/base/mkiv/syst-ini.mkiv b/tex/context/base/mkiv/syst-ini.mkiv index 1ca2bf1ac..b62dbfd7f 100644 --- a/tex/context/base/mkiv/syst-ini.mkiv +++ b/tex/context/base/mkiv/syst-ini.mkiv @@ -1276,5 +1276,6 @@ \ifdefined\breakafterdirmode \else \newcount\breakafterdirmode \fi \ifdefined\exceptionpenalty \else \newcount\exceptionpenalty \fi \ifdefined\luacopyinputnodes \else \newcount\luacopyinputnodes \fi +\ifdefined\fixupboxesmode \else \newcount\fixupboxesmode \fi \protect \endinput diff --git a/tex/context/base/mkiv/util-deb.lua b/tex/context/base/mkiv/util-deb.lua index b8db0c583..9488a728b 100644 --- a/tex/context/base/mkiv/util-deb.lua +++ b/tex/context/base/mkiv/util-deb.lua @@ -98,7 +98,7 @@ end setmetatableindex(names,function(t,name) local v = setmetatableindex(function(t,source) local v = setmetatableindex(function(t,line) - local v = { total = 0, count = 0 } + local v = { total = 0, count = 0, nesting = 0 } t[line] = v return v end) @@ -128,12 +128,24 @@ local function hook(where) end local data = names[name][source][line] if where == "call" then - data.count = data.count + 1 - insert(data,ticks()) + local nesting = data.nesting + if nesting == 0 then + data.count = data.count + 1 + insert(data,ticks()) + data.nesting = 1 + else + data.nesting = nesting + 1 + end elseif where == "return" then - local t = remove(data) - if t then - data.total = data.total + ticks() - t + local nesting = data.nesting + if nesting == 1 then + local t = remove(data) + if t then + data.total = data.total + ticks() - t + end + data.nesting = 0 + else + data.nesting = nesting - 1 end end end diff --git a/tex/context/base/mkiv/util-soc-imp-copas.lua b/tex/context/base/mkiv/util-soc-imp-copas.lua new file mode 100644 index 000000000..8e2278eb2 --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-copas.lua @@ -0,0 +1,930 @@ +-- original file : copas.lua +-- for more into : see util-soc.lua + +local socket = socket or require("socket") +local ssl = ssl or nil -- only loaded upon demand + +local WATCH_DOG_TIMEOUT = 120 +local UDP_DATAGRAM_MAX = 8192 + +local type, next, pcall, getmetatable, tostring = type, next, pcall, getmetatable, tostring +local min, max, random = math.min, math.max, math.random +local find = string.find +local insert, remove = table.insert, table.remove + +local gettime = socket.gettime +local selectsocket = socket.select + +local createcoroutine = coroutine.create +local resumecoroutine = coroutine.resume +local yieldcoroutine = coroutine.yield +local runningcoroutine = coroutine.running + +-- Redefines LuaSocket functions with coroutine safe versions (this allows the use +-- of socket.http from within copas). + +-- Meta information is public even if beginning with an "_" + +local report = logs and logs.reporter("copas") or function(fmt,first,...) + if fmt then + fmt = "copas: " .. fmt + if first then + print(format(fmt,first,...)) + else + print(fmt) + end + end +end + +local copas = { + + _COPYRIGHT = "Copyright (C) 2005-2016 Kepler Project", + _DESCRIPTION = "Coroutine Oriented Portable Asynchronous Services", + _VERSION = "Copas 2.0.1", + + autoclose = true, + running = false, + + report = report, + +} + +local function statushandler(status, ...) + if status then + return ... + end + local err = (...) + if type(err) == "table" then + err = err[1] + end + report("error: %s",tostring(err)) + return nil, err +end + +function socket.protect(func) + return function(...) + return statushandler(pcall(func,...)) + end +end + +function socket.newtry(finalizer) + return function (...) + local status = (...) + if not status then + local detail = select(2,...) + pcall(finalizer,detail) + report("error: %s",tostring(detail)) + return + end + return ... + end +end + +-- Simple set implementation based on LuaSocket's tinyirc.lua example +-- adds a FIFO queue for each value in the set + +local function newset() + local reverse = { } + local set = { } + local queue = { } + setmetatable(set, { + __index = { + insert = + function(set, value) + if not reverse[value] then + local n = #set +1 + set[n] = value + reverse[value] = n + end + end, + remove = + function(set, value) + local index = reverse[value] + if index then + reverse[value] = nil + local n = #set + local top = set[n] + set[n] = nil + if top ~= value then + reverse[top] = index + set[index] = top + end + end + end, + push = + function (set, key, itm) + local entry = queue[key] + if entry == nil then -- hm can it be false then? + queue[key] = { itm } + else + entry[#entry + 1] = itm + end + end, + pop = + function (set, key) + local top = queue[key] + if top ~= nil then + local ret = remove(top,1) + if top[1] == nil then + queue[key] = nil + end + return ret + end + end + } + } ) + return set +end + +local _sleeping = { + times = { }, -- list with wake-up times + cos = { }, -- list with coroutines, index matches the 'times' list + lethargy = { }, -- list of coroutines sleeping without a wakeup time + + insert = + function() + end, + remove = + function() + end, + push = + function(self, sleeptime, co) + if not co then + return + end + if sleeptime < 0 then + --sleep until explicit wakeup through copas.wakeup + self.lethargy[co] = true + return + else + sleeptime = gettime() + sleeptime + end + local t = self.times + local c = self.cos + local i = 1 + local n = #t + while i <= n and t[i] <= sleeptime do + i = i + 1 + end + insert(t,i,sleeptime) + insert(c,i,co) + end, + getnext = + -- returns delay until next sleep expires, or nil if there is none + function(self) + local t = self.times + local delay = t[1] and t[1] - gettime() or nil + return delay and max(delay, 0) or nil + end, + pop = + -- find the thread that should wake up to the time + function(self, time) + local t = self.times + local c = self.cos + if #t == 0 or time < t[1] then + return + end + local co = c[1] + remove(t,1) + remove(c,1) + return co + end, + wakeup = + function(self, co) + local let = self.lethargy + if let[co] then + self:push(0, co) + let[co] = nil + else + local c = self.cos + local t = self.times + for i=1,#c do + if c[i] == co then + remove(c,i) + remove(t,i) + self:push(0, co) + return + end + end + end + end +} + +local _servers = newset() -- servers being handled +local _reading = newset() -- sockets currently being read +local _writing = newset() -- sockets currently being written + +local _reading_log = { } +local _writing_log = { } + +local _is_timeout = { -- set of errors indicating a timeout + timeout = true, -- default LuaSocket timeout + wantread = true, -- LuaSec specific timeout + wantwrite = true, -- LuaSec specific timeout +} + +-- Coroutine based socket I/O functions. + +local function isTCP(socket) + return not find(tostring(socket),"^udp") +end + +-- Reads a pattern from a client and yields to the reading set on timeouts UDP: a +-- UDP socket expects a second argument to be a number, so it MUST be provided as +-- the 'pattern' below defaults to a string. Will throw a 'bad argument' error if +-- omitted. + +local function copasreceive(client, pattern, part) + if not pattern or pattern == "" then + pattern = "*l" + end + local current_log = _reading_log + local s, err + repeat + s, err, part = client:receive(pattern, part) + if s or (not _is_timeout[err]) then + current_log[client] = nil + return s, err, part + end + if err == "wantwrite" then + current_log = _writing_log + current_log[client] = gettime() + yieldcoroutine(client, _writing) + else + current_log = _reading_log + current_log[client] = gettime() + yieldcoroutine(client, _reading) + end + until false +end + +-- Receives data from a client over UDP. Not available for TCP. (this is a copy of +-- receive() method, adapted for receivefrom() use). + +local function copasreceivefrom(client, size) + local s, err, port + if not size or size == 0 then + size = UDP_DATAGRAM_MAX + end + repeat + -- upon success err holds ip address + s, err, port = client:receivefrom(size) + if s or err ~= "timeout" then + _reading_log[client] = nil + return s, err, port + end + _reading_log[client] = gettime() + yieldcoroutine(client, _reading) + until false +end + +-- Same as above but with special treatment when reading chunks, unblocks on any +-- data received. + +local function copasreceivepartial(client, pattern, part) + if not pattern or pattern == "" then + pattern = "*l" + end + local logger = _reading_log + local queue = _reading + local s, err + repeat + s, err, part = client:receive(pattern, part) + if s or (type(pattern) == "number" and part ~= "" and part) or not _is_timeout[err] then + logger[client] = nil + return s, err, part + end + if err == "wantwrite" then + logger = _writing_log + queue = _writing + else + logger = _reading_log + queue = _reading + end + logger[client] = gettime() + yieldcoroutine(client, queue) + until false +end + +-- Sends data to a client. The operation is buffered and yields to the writing set +-- on timeouts Note: from and to parameters will be ignored by/for UDP sockets + +local function copassend(client, data, from, to) + if not from then + from = 1 + end + local lastIndex = from - 1 + local logger = _writing_log + local queue = _writing + local s, err + repeat + s, err, lastIndex = client:send(data, lastIndex + 1, to) + -- Adds extra coroutine swap and garantees that high throughput doesn't take + -- other threads to starvation. + if random(100) > 90 then + logger[client] = gettime() + yieldcoroutine(client, queue) + end + if s or not _is_timeout[err] then + logger[client] = nil + return s, err,lastIndex + end + if err == "wantread" then + logger = _reading_log + queue = _reading + else + logger = _writing_log + queue = _writing + end + logger[client] = gettime() + yieldcoroutine(client, queue) + until false +end + +-- Sends data to a client over UDP. Not available for TCP. (this is a copy of send() +-- method, adapted for sendto() use). + +local function copassendto(client, data, ip, port) + repeat + local s, err = client:sendto(data, ip, port) + -- Adds extra coroutine swap and garantees that high throughput doesn't + -- take other threads to starvation. + if random(100) > 90 then + _writing_log[client] = gettime() + yieldcoroutine(client, _writing) + end + if s or err ~= "timeout" then + _writing_log[client] = nil + return s, err + end + _writing_log[client] = gettime() + yieldcoroutine(client, _writing) + until false +end + +-- Waits until connection is completed. + +local function copasconnect(skt, host, port) + skt:settimeout(0) + local ret, err, tried_more_than_once + repeat + ret, err = skt:connect (host, port) + -- A non-blocking connect on Windows results in error "Operation already in + -- progress" to indicate that it is completing the request async. So + -- essentially it is the same as "timeout". + if ret or (err ~= "timeout" and err ~= "Operation already in progress") then + -- Once the async connect completes, Windows returns the error "already + -- connected" to indicate it is done, so that error should be ignored. + -- Except when it is the first call to connect, then it was already + -- connected to something else and the error should be returned. + if not ret and err == "already connected" and tried_more_than_once then + ret = 1 + err = nil + end + _writing_log[skt] = nil + return ret, err + end + tried_more_than_once = tried_more_than_once or true + _writing_log[skt] = gettime() + yieldcoroutine(skt, _writing) + until false +end + +-- Peforms an (async) ssl handshake on a connected TCP client socket. Replacec all +-- previous socket references, with the returned new ssl wrapped socket Throws error +-- and does not return nil+error, as that might silently fail in code like this. + +local function copasdohandshake(skt, sslt) -- extra ssl parameters + if not ssl then + ssl = require("ssl") + end + if not ssl then + report("error: no ssl library") + return + end + local nskt, err = ssl.wrap(skt, sslt) + if not nskt then + report("error: %s",tostring(err)) + return + end + nskt:settimeout(0) + local queue + repeat + local success, err = nskt:dohandshake() + if success then + return nskt + elseif err == "wantwrite" then + queue = _writing + elseif err == "wantread" then + queue = _reading + else + report("error: %s",tostring(err)) + return + end + yieldcoroutine(nskt, queue) + until false +end + +-- Flushes a client write buffer. + +local function copasflush(client) +end + +-- Public. + +copas.connect = copassconnect +copas.send = copassend +copas.sendto = copassendto +copas.receive = copasreceive +copas.receivefrom = copasreceivefrom +copas.copasreceivepartial = copasreceivepartial +copas.copasreceivePartial = copasreceivepartial +copas.dohandshake = copasdohandshake +copas.flush = copasflush + +-- Wraps a TCP socket to use Copas methods (send, receive, flush and settimeout). + +local function _skt_mt_tostring(self) + return tostring(self.socket) .. " (copas wrapped)" +end + +local _skt_mt_tcp_index = { + send = + function(self, data, from, to) + return copassend (self.socket, data, from, to) + end, + receive = + function (self, pattern, prefix) + if self.timeout == 0 then + return copasreceivePartial(self.socket, pattern, prefix) + else + return copasreceive(self.socket, pattern, prefix) + end + end, + + flush = + function (self) + return copasflush(self.socket) + end, + + settimeout = + function (self, time) + self.timeout = time + return true + end, + -- TODO: socket.connect is a shortcut, and must be provided with an alternative + -- if ssl parameters are available, it will also include a handshake + connect = + function(self, ...) + local res, err = copasconnect(self.socket, ...) + if res and self.ssl_params then + res, err = self:dohandshake() + end + return res, err + end, + close = + function(self, ...) + return self.socket:close(...) + end, + -- TODO: socket.bind is a shortcut, and must be provided with an alternative + bind = + function(self, ...) + return self.socket:bind(...) + end, + -- TODO: is this DNS related? hence blocking? + getsockname = + function(self, ...) + return self.socket:getsockname(...) + end, + getstats = + function(self, ...) + return self.socket:getstats(...) + end, + setstats = + function(self, ...) + return self.socket:setstats(...) + end, + listen = + function(self, ...) + return self.socket:listen(...) + end, + accept = + function(self, ...) + return self.socket:accept(...) + end, + setoption = + function(self, ...) + return self.socket:setoption(...) + end, + -- TODO: is this DNS related? hence blocking? + getpeername = + function(self, ...) + return self.socket:getpeername(...) + end, + shutdown = + function(self, ...) + return self.socket:shutdown(...) + end, + dohandshake = + function(self, sslt) + self.ssl_params = sslt or self.ssl_params + local nskt, err = copasdohandshake(self.socket, self.ssl_params) + if not nskt then + return nskt, err + end + self.socket = nskt + return self + end, +} + +local _skt_mt_tcp = { + __tostring = _skt_mt_tostring, + __index = _skt_mt_tcp_index, +} + +-- wraps a UDP socket, copy of TCP one adapted for UDP. + +local _skt_mt_udp_index = { + -- UDP sending is non-blocking, but we provide starvation prevention, so replace + -- anyway. + sendto = + function (self, ...) + return copassendto(self.socket,...) + end, + receive = + function (self, size) + return copasreceive(self.socket, size or UDP_DATAGRAM_MAX) + end, + receivefrom = + function (self, size) + return copasreceivefrom(self.socket, size or UDP_DATAGRAM_MAX) + end, + -- TODO: is this DNS related? hence blocking? + setpeername = + function(self, ...) + return self.socket:getpeername(...) + end, + setsockname = + function(self, ...) + return self.socket:setsockname(...) + end, + -- do not close client, as it is also the server for udp. + close = + function(self, ...) + return true + end +} + +local _skt_mt_udp = { + __tostring = _skt_mt_tostring, + __index = _skt_mt_udp_index, +} + +for k, v in next, _skt_mt_tcp_index do + if not _skt_mt_udp_index[k] then + _skt_mt_udp_index[k] = v + end +end + +-- Wraps a LuaSocket socket object in an async Copas based socket object. + +-- @param skt the socket to wrap +-- @sslt (optional) Table with ssl parameters, use an empty table to use ssl with defaults +-- @return wrapped socket object + +local function wrap(skt, sslt) + if getmetatable(skt) == _skt_mt_tcp or getmetatable(skt) == _skt_mt_udp then + return skt -- already wrapped + end + skt:settimeout(0) + if isTCP(skt) then + return setmetatable ({ socket = skt, ssl_params = sslt }, _skt_mt_tcp) + else + return setmetatable ({ socket = skt }, _skt_mt_udp) + end +end + +copas.wrap = wrap + +-- Wraps a handler in a function that deals with wrapping the socket and doing +-- the optional ssl handshake. + +function copas.handler(handler, sslparams) + return function (skt,...) + skt = wrap(skt) + if sslparams then + skt:dohandshake(sslparams) + end + return handler(skt,...) + end +end + +-- Error handling (a handler per coroutine). + +local _errhandlers = { } + +function copas.setErrorHandler(err) + local co = runningcoroutine() + if co then + _errhandlers[co] = err + end +end + +local function _deferror (msg, co, skt) + report("%s (%s) (%s)", msg, tostring(co), tostring(skt)) +end + +-- Thread handling + +local function _doTick (co, skt, ...) + if not co then + return + end + + local ok, res, new_q = resumecoroutine(co, skt, ...) + + if ok and res and new_q then + new_q:insert(res) + new_q:push(res, co) + else + if not ok then + pcall(_errhandlers[co] or _deferror, res, co, skt) + end + -- Do not auto-close UDP sockets, as the handler socket is also the server socket. + if skt and copas.autoclose and isTCP(skt) then + skt:close() + end + _errhandlers[co] = nil + end +end + +-- Accepts a connection on socket input. + +local function _accept(input, handler) + local client = input:accept() + if client then + client:settimeout(0) + local co = createcoroutine(handler) + _doTick (co, client) + -- _reading:insert(client) + end + return client +end + +-- Handle threads on a queue. + +local function _tickRead(skt) + _doTick(_reading:pop(skt), skt) +end + +local function _tickWrite(skt) + _doTick(_writing:pop(skt), skt) +end + +-- Adds a server/handler pair to Copas dispatcher. + +local function addTCPserver(server, handler, timeout) + server:settimeout(timeout or 0) + _servers[server] = handler + _reading:insert(server) +end + +local function addUDPserver(server, handler, timeout) + server:settimeout(timeout or 0) + local co = createcoroutine(handler) + _reading:insert(server) + _doTick(co, server) +end + +function copas.addserver(server, handler, timeout) + if isTCP(server) then + addTCPserver(server, handler, timeout) + else + addUDPserver(server, handler, timeout) + end +end + +function copas.removeserver(server, keep_open) + local s = server + local mt = getmetatable(server) + if mt == _skt_mt_tcp or mt == _skt_mt_udp then + s = server.socket + end + _servers[s] = nil + _reading:remove(s) + if keep_open then + return true + end + return server:close() +end + +-- Adds an new coroutine thread to Copas dispatcher. Create a coroutine that skips +-- the first argument, which is always the socket passed by the scheduler, but `nil` +-- in case of a task/thread + +function copas.addthread(handler, ...) + local thread = createcoroutine(function(_, ...) return handler(...) end) + _doTick(thread, nil, ...) + return thread +end + +-- tasks registering + +local _tasks = { } + +-- Lets tasks call the default _tick(). + +local function addtaskRead(tsk) + tsk.def_tick = _tickRead + _tasks[tsk] = true +end + +-- Lets tasks call the default _tick(). + +local function addtaskWrite(tsk) + tsk.def_tick = _tickWrite + _tasks[tsk] = true +end + +local function tasks() + return next, _tasks +end + +-- A task to check ready to read events. + +local _readable_t = { + events = + function(self) + local i = 0 + return function () + i = i + 1 + return self._evs[i] + end + end, + tick = + function(self, input) + local handler = _servers[input] + if handler then + input = _accept(input, handler) + else + _reading:remove(input) + self.def_tick(input) + end + end +} + +addtaskRead(_readable_t) + +-- A task to check ready to write events. + +local _writable_t = { + events = + function(self) + local i = 0 + return function() + i = i + 1 + return self._evs[i] + end + end, + tick = + function(self, output) + _writing:remove(output) + self.def_tick(output) + end +} + +addtaskWrite(_writable_t) + +--sleeping threads task + +local _sleeping_t = { + tick = function(self, time, ...) + _doTick(_sleeping:pop(time), ...) + end +} + +-- yields the current coroutine and wakes it after 'sleeptime' seconds. +-- If sleeptime<0 then it sleeps until explicitly woken up using 'wakeup' +function copas.sleep(sleeptime) + yieldcoroutine((sleeptime or 0), _sleeping) +end + +-- Wakes up a sleeping coroutine 'co'. + +function copas.wakeup(co) + _sleeping:wakeup(co) +end + +-- Checks for reads and writes on sockets + +local last_cleansing = 0 + +local function _select(timeout) + + local now = gettime() + + local r_evs, w__evs, err = selectsocket(_reading, _writing, timeout) + + _readable_t._evs = r_evs + _writable_t._evs = w_evs + + if (last_cleansing - now) > WATCH_DOG_TIMEOUT then + + last_cleansing = now + + -- Check all sockets selected for reading, and check how long they have been + -- waiting for data already, without select returning them as readable. + + for skt, time in next, _reading_log do + + if not r_evs[skt] and (time - now) > WATCH_DOG_TIMEOUT then + + -- This one timedout while waiting to become readable, so move it in + -- the readable list and try and read anyway, despite not having + -- been returned by select. + + local n = #r_evs + 1 + _reading_log[skt] = nil + r_evs[n] = skt + r_evs[skt] = n + end + end + + -- Do the same for writing. + + for skt, time in next, _writing_log do + if not w_evs[skt] and (time - now) > WATCH_DOG_TIMEOUT then + local n = #w_evs + 1 + _writing_log[skt] = nil + w_evs[n] = skt + w_evs[skt] = n + end + end + + end + + if err == "timeout" and #r_evs + #w_evs > 0 then + return nil + else + return err + end + +end + +-- Check whether there is something to do. It returns false if there are no sockets +-- for read/write nor tasks scheduled (which means Copas is in an empty spin). + +local function copasfinished() + return not (next(_reading) or next(_writing) or _sleeping:getnext()) +end + +-- Dispatcher loop step. It listens to client requests and handles them and returns +-- false if no data was handled (timeout), or true if there was data handled (or nil +-- + error message). + +local function copasstep(timeout) + _sleeping_t:tick(gettime()) + + local nextwait = _sleeping:getnext() + if nextwait then + timeout = timeout and min(nextwait,timeout) or nextwait + elseif finished() then + return false + end + + local err = _select(timeout) + if err then + if err == "timeout" then + return false + end + return nil, err + end + + for task in tasks() do + for event in task:events() do + tsk:tick(event) + end + end + return true +end + +copas.finished = copasfinished +copas.step = copasstep + +-- Dispatcher endless loop. It listens to client requests and handles them forever. + +function copas.loop(timeout) + copas.running = true + while not copasfinished() do + copasstep(timeout) + end + copas.running = false +end + +if logs then + _G.copas = copas + package.loaded.copas = copas + -- report("module (re)installed") +end + +return copas diff --git a/tex/context/base/mkiv/util-soc-imp-ftp.lua b/tex/context/base/mkiv/util-soc-imp-ftp.lua new file mode 100644 index 000000000..b9f5f15db --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-ftp.lua @@ -0,0 +1,400 @@ +-- original file : ftp.lua +-- for more into : see util-soc.lua + +local setmetatable, type, next = setmetatable, type, next +local find, format, gsub, match = string.find, string.format, string.gsub, string.match +local concat = table.concat +local mod = math.mod + +local socket = socket or require("socket") +local url = socket.url or require("socket.url") +local tp = socket.tp or require("socket.tp") +local ltn12 = ltn12 or require("ltn12") + +local tcpsocket = socket.tcp +local trysocket = socket.try +local skipsocket = socket.skip +local sinksocket = socket.sink +local selectsocket = socket.select +local bindsocket = socket.bind +local newtrysocket = socket.newtry +local sourcesocket = socket.source +local protectsocket = socket.protect + +local parseurl = url.parse +local unescapeurl = url.unescape + +local pumpall = ltn12.pump.all +local pumpstep = ltn12.pump.step +local sourcestring = ltn12.source.string +local sinktable = ltn12.sink.table + +local ftp = { + TIMEOUT = 60, + USER = "ftp", + PASSWORD = "anonymous@anonymous.org", +} + +socket.ftp = ftp + +local PORT = 21 + +local methods = { } +local mt = { __index = methods } + +function ftp.open(server, port, create) + local tp = trysocket(tp.connect(server, port or PORT, ftp.TIMEOUT, create)) + local f = setmetatable({ tp = tp }, metat) + f.try = newtrysocket(function() f:close() end) + return f +end + +function methods.portconnect(self) + local try = self.try + local server = self.server + try(server:settimeout(ftp.TIMEOUT)) + self.data = try(server:accept()) + try(self.data:settimeout(ftp.TIMEOUT)) +end + +function methods.pasvconnect(self) + local try = self.try + self.data = try(tcpsocket()) + self(self.data:settimeout(ftp.TIMEOUT)) + self(self.data:connect(self.pasvt.address, self.pasvt.port)) +end + +function methods.login(self, user, password) + local try = self.try + local tp = self.tp + try(tp:command("user", user or ftp.USER)) + local code, reply = try(tp:check{"2..", 331}) + if code == 331 then + try(tp:command("pass", password or ftp.PASSWORD)) + try(tp:check("2..")) + end + return 1 +end + +function methods.pasv(self) + local try = self.try + local tp = self.tp + try(tp:command("pasv")) + local code, reply = try(self.tp:check("2..")) + local pattern = "(%d+)%D(%d+)%D(%d+)%D(%d+)%D(%d+)%D(%d+)" + local a, b, c, d, p1, p2 = skipsocket(2, find(reply, pattern)) + try(a and b and c and d and p1 and p2, reply) + local address = format("%d.%d.%d.%d", a, b, c, d) + local port = p1*256 + p2 + local server = self.server + self.pasvt = { + address = address, + port = port, + } + if server then + server:close() + self.server = nil + end + return address, port +end + +function methods.epsv(self) + local try = self.try + local tp = self.tp + try(tp:command("epsv")) + local code, reply = try(tp:check("229")) + local pattern = "%((.)(.-)%1(.-)%1(.-)%1%)" + local d, prt, address, port = match(reply, pattern) + try(port, "invalid epsv response") + local address = tp:getpeername() + local server = self.server + self.pasvt = { + address = address, + port = port, + } + if self.server then + server:close() + self.server = nil + end + return address, port +end + +function methods.port(self, address, port) + local try = self.try + local tp = self.tp + self.pasvt = nil + if not address then + address, port = try(tp:getsockname()) + self.server = try(bindsocket(address, 0)) + address, port = try(self.server:getsockname()) + try(self.server:settimeout(ftp.TIMEOUT)) + end + local pl = mod(port,256) + local ph = (port - pl)/256 + local arg = gsub(format("%s,%d,%d", address, ph, pl), "%.", ",") + try(tp:command("port", arg)) + try(tp:check("2..")) + return 1 +end + +function methods.eprt(self, family, address, port) + local try = self.try + local tp = self.tp + self.pasvt = nil + if not address then + address, port = try(tp:getsockname()) + self.server = try(bindsocket(address, 0)) + address, port = try(self.server:getsockname()) + try(self.server:settimeout(ftp.TIMEOUT)) + end + local arg = format("|%s|%s|%d|", family, address, port) + try(tp:command("eprt", arg)) + try(tp:check("2..")) + return 1 +end + +function methods.send(self, sendt) + local try = self.try + local tp = self.tp + -- so we try a table or string ? + try(self.pasvt or self.server, "need port or pasv first") + if self.pasvt then + self:pasvconnect() + end + local argument = sendt.argument or unescapeurl(gsub(sendt.path or "", "^[/\\]", "")) + if argument == "" then + argument = nil + end + local command = sendt.command or "stor" + try(tp:command(command, argument)) + local code, reply = try(tp:check{"2..", "1.."}) + if not self.pasvt then + self:portconnect() + end + local step = sendt.step or pumpstep + local readt = { tp } + local checkstep = function(src, snk) + local readyt = selectsocket(readt, nil, 0) + if readyt[tp] then + code = try(tp:check("2..")) + end + return step(src, snk) + end + local sink = sinksocket("close-when-done", self.data) + try(pumpall(sendt.source, sink, checkstep)) + if find(code, "1..") then + try(tp:check("2..")) + end + self.data:close() + local sent = skipsocket(1, self.data:getstats()) + self.data = nil + return sent +end + +function methods.receive(self, recvt) + local try = self.try + local tp = self.tp + try(self.pasvt or self.server, "need port or pasv first") + if self.pasvt then self:pasvconnect() end + local argument = recvt.argument or unescapeurl(gsub(recvt.path or "", "^[/\\]", "")) + if argument == "" then + argument = nil + end + local command = recvt.command or "retr" + try(tp:command(command, argument)) + local code,reply = try(tp:check{"1..", "2.."}) + if code >= 200 and code <= 299 then + recvt.sink(reply) + return 1 + end + if not self.pasvt then + self:portconnect() + end + local source = sourcesocket("until-closed", self.data) + local step = recvt.step or pumpstep + try(pumpall(source, recvt.sink, step)) + if find(code, "1..") then + try(tp:check("2..")) + end + self.data:close() + self.data = nil + return 1 +end + +function methods.cwd(self, dir) + local try = self.try + local tp = self.tp + try(tp:command("cwd", dir)) + try(tp:check(250)) + return 1 +end + +function methods.type(self, typ) + local try = self.try + local tp = self.tp + try(tp:command("type", typ)) + try(tp:check(200)) + return 1 +end + +function methods.greet(self) + local try = self.try + local tp = self.tp + local code = try(tp:check{"1..", "2.."}) + if find(code, "1..") then + try(tp:check("2..")) + end + return 1 +end + +function methods.quit(self) + local try = self.try + try(self.tp:command("quit")) + try(self.tp:check("2..")) + return 1 +end + +function methods.close(self) + local data = self.data + if data then + data:close() + end + local server = self.server + if server then + server:close() + end + local tp = self.tp + if tp then + tp:close() + end +end + +local function override(t) + if t.url then + local u = parseurl(t.url) + for k, v in next, t do + u[k] = v + end + return u + else + return t + end +end + +local function tput(putt) + putt = override(putt) + local host = putt.host + trysocket(host, "missing hostname") + local f = ftp.open(host, putt.port, putt.create) + f:greet() + f:login(putt.user, putt.password) + local typ = putt.type + if typ then + f:type(typ) + end + f:epsv() + local sent = f:send(putt) + f:quit() + f:close() + return sent +end + +local default = { + path = "/", + scheme = "ftp", +} + +local function genericform(u) + local t = trysocket(parseurl(u, default)) + trysocket(t.scheme == "ftp", "wrong scheme '" .. t.scheme .. "'") + trysocket(t.host, "missing hostname") + local pat = "^type=(.)$" + if t.params then + local typ = skipsocket(2, find(t.params, pat)) + t.type = typ + trysocket(typ == "a" or typ == "i", "invalid type '" .. typ .. "'") + end + return t +end + +ftp.genericform = genericform + +local function sput(u, body) + local putt = genericform(u) + putt.source = sourcestring(body) + return tput(putt) +end + +ftp.put = protectsocket(function(putt, body) + if type(putt) == "string" then + return sput(putt, body) + else + return tput(putt) + end +end) + +local function tget(gett) + gett = override(gett) + local host = gett.host + trysocket(host, "missing hostname") + local f = ftp.open(host, gett.port, gett.create) + f:greet() + f:login(gett.user, gett.password) + if gett.type then + f:type(gett.type) + end + f:epsv() + f:receive(gett) + f:quit() + return f:close() +end + +local function sget(u) + local gett = genericform(u) + local t = { } + gett.sink = sinktable(t) + tget(gett) + return concat(t) +end + +ftp.command = protectsocket(function(cmdt) + cmdt = override(cmdt) + local command = cmdt.command + local argument = cmdt.argument + local check = cmdt.check + local host = cmdt.host + trysocket(host, "missing hostname") + trysocket(command, "missing command") + local f = ftp.open(host, cmdt.port, cmdt.create) + local try = f.try + local tp = f.tp + f:greet() + f:login(cmdt.user, cmdt.password) + if type(command) == "table" then + local argument = argument or { } + for i=1,#command do + local cmd = command[i] + try(tp:command(cmd, argument[i])) + if check and check[i] then + try(tp:check(check[i])) + end + end + else + try(tp:command(command, argument)) + if check then + try(tp:check(check)) + end + end + f:quit() + return f:close() +end) + +ftp.get = protectsocket(function(gett) + if type(gett) == "string" then + return sget(gett) + else + return tget(gett) + end +end) + +return ftp diff --git a/tex/context/base/mkiv/util-soc-imp-headers.lua b/tex/context/base/mkiv/util-soc-imp-headers.lua new file mode 100644 index 000000000..ee889956c --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-headers.lua @@ -0,0 +1,144 @@ +-- original file : headers.lua +-- for more into : see util-soc.lua + +local next = next +local lower = string.lower +local concat = table.concat + +local socket = socket or require("socket") + +local canonic = { + ["accept"] = "Accept", + ["accept-charset"] = "Accept-Charset", + ["accept-encoding"] = "Accept-Encoding", + ["accept-language"] = "Accept-Language", + ["accept-ranges"] = "Accept-Ranges", + ["action"] = "Action", + ["alternate-recipient"] = "Alternate-Recipient", + ["age"] = "Age", + ["allow"] = "Allow", + ["arrival-date"] = "Arrival-Date", + ["authorization"] = "Authorization", + ["bcc"] = "Bcc", + ["cache-control"] = "Cache-Control", + ["cc"] = "Cc", + ["comments"] = "Comments", + ["connection"] = "Connection", + ["content-description"] = "Content-Description", + ["content-disposition"] = "Content-Disposition", + ["content-encoding"] = "Content-Encoding", + ["content-id"] = "Content-ID", + ["content-language"] = "Content-Language", + ["content-length"] = "Content-Length", + ["content-location"] = "Content-Location", + ["content-md5"] = "Content-MD5", + ["content-range"] = "Content-Range", + ["content-transfer-encoding"] = "Content-Transfer-Encoding", + ["content-type"] = "Content-Type", + ["cookie"] = "Cookie", + ["date"] = "Date", + ["diagnostic-code"] = "Diagnostic-Code", + ["dsn-gateway"] = "DSN-Gateway", + ["etag"] = "ETag", + ["expect"] = "Expect", + ["expires"] = "Expires", + ["final-log-id"] = "Final-Log-ID", + ["final-recipient"] = "Final-Recipient", + ["from"] = "From", + ["host"] = "Host", + ["if-match"] = "If-Match", + ["if-modified-since"] = "If-Modified-Since", + ["if-none-match"] = "If-None-Match", + ["if-range"] = "If-Range", + ["if-unmodified-since"] = "If-Unmodified-Since", + ["in-reply-to"] = "In-Reply-To", + ["keywords"] = "Keywords", + ["last-attempt-date"] = "Last-Attempt-Date", + ["last-modified"] = "Last-Modified", + ["location"] = "Location", + ["max-forwards"] = "Max-Forwards", + ["message-id"] = "Message-ID", + ["mime-version"] = "MIME-Version", + ["original-envelope-id"] = "Original-Envelope-ID", + ["original-recipient"] = "Original-Recipient", + ["pragma"] = "Pragma", + ["proxy-authenticate"] = "Proxy-Authenticate", + ["proxy-authorization"] = "Proxy-Authorization", + ["range"] = "Range", + ["received"] = "Received", + ["received-from-mta"] = "Received-From-MTA", + ["references"] = "References", + ["referer"] = "Referer", + ["remote-mta"] = "Remote-MTA", + ["reply-to"] = "Reply-To", + ["reporting-mta"] = "Reporting-MTA", + ["resent-bcc"] = "Resent-Bcc", + ["resent-cc"] = "Resent-Cc", + ["resent-date"] = "Resent-Date", + ["resent-from"] = "Resent-From", + ["resent-message-id"] = "Resent-Message-ID", + ["resent-reply-to"] = "Resent-Reply-To", + ["resent-sender"] = "Resent-Sender", + ["resent-to"] = "Resent-To", + ["retry-after"] = "Retry-After", + ["return-path"] = "Return-Path", + ["sender"] = "Sender", + ["server"] = "Server", + ["smtp-remote-recipient"] = "SMTP-Remote-Recipient", + ["status"] = "Status", + ["subject"] = "Subject", + ["te"] = "TE", + ["to"] = "To", + ["trailer"] = "Trailer", + ["transfer-encoding"] = "Transfer-Encoding", + ["upgrade"] = "Upgrade", + ["user-agent"] = "User-Agent", + ["vary"] = "Vary", + ["via"] = "Via", + ["warning"] = "Warning", + ["will-retry-until"] = "Will-Retry-Until", + ["www-authenticate"] = "WWW-Authenticate", + ["x-mailer"] = "X-Mailer", +} + +setmetatable(canonic, { + __index = function(t,k) + socket.report("invalid header: %s",k) + t[k] = k + return k + end +}) + +local function normalizeheaders(headers) + if not headers then + return { } + end + local normalized = { } + for k, v in next, headers do + normalized[#normalized+1] = canonic[k] .. ": " .. v + end + normalized[#normalized+1] = "" + normalized[#normalized+1] = "" + return concat(normalized,"\r\n") +end + +local function lowerheaders(lowered,headers) + if not lowered then + return { } + end + if not headers then + lowered, headers = { }, lowered + end + for k, v in next, headers do + lowered[lower(k)] = v + end + return lowered +end + +socket.headers = { + canonic = canonic, + normalize = normalizeheaders, + lower = lowerheaders, +} + +return socket.headers diff --git a/tex/context/base/mkiv/util-soc-imp-http.lua b/tex/context/base/mkiv/util-soc-imp-http.lua new file mode 100644 index 000000000..98789fa7b --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-http.lua @@ -0,0 +1,432 @@ +-- original file : http.lua +-- for more into : see util-soc.lua + +local tostring, tonumber, setmetatable, next, type = tostring, tonumber, setmetatable, next, type +local find, lower, format, gsub, match = string.find, string.lower, string.format, string.gsub, string.match +local concat = table.concat + +local socket = socket or require("socket") +local url = socket.url or require("socket.url") +local ltn12 = ltn12 or require("ltn12") +local mime = mime or require("mime") +local headers = socket.headers or require("socket.headers") + +local normalizeheaders = headers.normalize + +local parseurl = url.parse +local buildurl = url.build +local absoluteurl = url.absolute +local unescapeurl = url.unescape + +local skipsocket = socket.skip +local sinksocket = socket.sink +local sourcesocket = socket.source +local trysocket = socket.try +local tcpsocket = socket.tcp +local newtrysocket = socket.newtry +local protectsocket = socket.protect + +local emptysource = ltn12.source.empty +local stringsource = ltn12.source.string +local rewindsource = ltn12.source.rewind +local pumpstep = ltn12.pump.step +local pumpall = ltn12.pump.all +local sinknull = ltn12.sink.null +local sinktable = ltn12.sink.table + +local mimeb64 = mime.b64 + +-- todo: localize ltn12 + +local http = { + TIMEOUT = 60, -- connection timeout in seconds + USERAGENT = socket._VERSION, -- user agent field sent in request +} + +socket.http = http + +local PORT = 80 +local SCHEMES = { + http = true, +} + +-- Reads MIME headers from a connection, unfolding where needed + +local function receiveheaders(sock, headers) + if not headers then + headers = { } + end + -- get first line + local line, err = sock:receive() + if err then + return nil, err + end + -- headers go until a blank line is found + while line ~= "" do + -- get field-name and value + local name, value = skipsocket(2, find(line, "^(.-):%s*(.*)")) + if not (name and value) then + return nil, "malformed reponse headers" + end + name = lower(name) + -- get next line (value might be folded) + line, err = sock:receive() + if err then + return nil, err + end + -- unfold any folded values + while find(line, "^%s") do + value = value .. line + line = sock:receive() + if err then + return nil, err + end + end + -- save pair in table + local found = headers[name] + if found then + value = found .. ", " .. value + end + headers[name] = value + end + return headers +end + +-- Extra sources and sinks + +socket.sourcet["http-chunked"] = function(sock, headers) + return setmetatable ( + { + getfd = function() return sock:getfd() end, + dirty = function() return sock:dirty() end, + }, { + __call = function() + local line, err = sock:receive() + if err then + return nil, err + end + local size = tonumber(gsub(line, ";.*", ""), 16) + if not size then + return nil, "invalid chunk size" + end + if size > 0 then + local chunk, err, part = sock:receive(size) + if chunk then + sock:receive() + end + return chunk, err + else + headers, err = receiveheaders(sock, headers) + if not headers then + return nil, err + end + end + end + } + ) +end + +socket.sinkt["http-chunked"] = function(sock) + return setmetatable( + { + getfd = function() return sock:getfd() end, + dirty = function() return sock:dirty() end, + }, + { + __call = function(self, chunk, err) + if not chunk then + chunk = "" + end + return sock:send(format("%X\r\n%s\r\n",#chunk,chunk)) + end + }) +end + +-- Low level HTTP API + +local methods = { } +local mt = { __index = methods } + +local function openhttp(host, port, create) + local c = trysocket((create or tcpsocket)()) + local h = setmetatable({ c = c }, mt) + local try = newtrysocket(function() h:close() end) + h.try = try + try(c:settimeout(http.TIMEOUT)) + try(c:connect(host, port or PORT)) + return h +end + +http.open = openhttp + +function methods.sendrequestline(self, method, uri) + local requestline = format("%s %s HTTP/1.1\r\n", method or "GET", uri) + return self.try(self.c:send(requestline)) +end + +function methods.sendheaders(self,headers) + self.try(self.c:send(normalizeheaders(headers))) + return 1 +end + +function methods.sendbody(self, headers, source, step) + if not source then + source = emptysource() + end + if not step then + step = pumpstep + end + local mode = "http-chunked" + if headers["content-length"] then + mode = "keep-open" + end + return self.try(pumpall(source, sinksocket(mode, self.c), step)) +end + +function methods.receivestatusline(self) + local try = self.try + local status = try(self.c:receive(5)) + if status ~= "HTTP/" then + return nil, status -- HTTP/0.9 + end + status = try(self.c:receive("*l", status)) + local code = skipsocket(2, find(status, "HTTP/%d*%.%d* (%d%d%d)")) + return try(tonumber(code), status) +end + +function methods.receiveheaders(self) + return self.try(receiveheaders(self.c)) +end + +function methods.receivebody(self, headers, sink, step) + if not sink then + sink = sinknull() + end + if not step then + step = pumpstep + end + local length = tonumber(headers["content-length"]) + local encoding = headers["transfer-encoding"] -- shortcut + local mode = "default" -- connection close + if encoding and encoding ~= "identity" then + mode = "http-chunked" + elseif length then + mode = "by-length" + end + --hh: so length can be nil + return self.try(pumpall(sourcesocket(mode, self.c, length), sink, step)) +end + +function methods.receive09body(self, status, sink, step) + local source = rewindsource(sourcesocket("until-closed", self.c)) + source(status) + return self.try(pumpall(source, sink, step)) +end + +function methods.close(self) + return self.c:close() +end + +-- High level HTTP API + +local function adjusturi(request) + if not request.proxy and not http.PROXY then + request = { + path = trysocket(request.path, "invalid path 'nil'"), + params = request.params, + query = request.query, + fragment = request.fragment, + } + end + return buildurl(request) +end + +local function adjustheaders(request) + local headers = { + ["user-agent"] = http.USERAGENT, + ["host"] = gsub(request.authority, "^.-@", ""), + ["connection"] = "close, TE", + ["te"] = "trailers" + } + local username = request.user + local password = request.password + if username and password then + headers["authorization"] = "Basic " .. (mimeb64(username .. ":" .. unescapeurl(password))) + end + local proxy = request.proxy or http.PROXY + if proxy then + proxy = parseurl(proxy) + local username = proxy.user + local password = proxy.password + if username and password then + headers["proxy-authorization"] = "Basic " .. (mimeb64(username .. ":" .. password)) + end + end + local requestheaders = request.headers + if requestheaders then + headers = lowerheaders(headers,requestheaders) + end + return headers +end + +-- default url parts + +local default = { + host = "", + port = PORT, + path = "/", + scheme = "http" +} + +local function adjustrequest(originalrequest) + local url = originalrequest.url + local request = url and parseurl(url,default) or { } + for k, v in next, originalrequest do + request[k] = v + end + local host = request.host + local port = request.port + local uri = request.uri + if not host or host == "" then + trysocket(nil, "invalid host '" .. tostring(host) .. "'") + end + if port == "" then + request.port = PORT + end + if not uri or uri == "" then + request.uri = adjusturi(request) + end + request.headers = adjustheaders(request) + local proxy = request.proxy or http.PROXY + if proxy then + proxy = parseurl(proxy) + request.host = proxy.host + request.port = proxy.port or 3128 + end + return request +end + +local maxredericts = 4 +local validredirects = { [301] = true, [302] = true, [303] = true, [307] = true } +local validmethods = { [false] = true, GET = true, HEAD = true } + +local function shouldredirect(request, code, headers) + local location = headers.location + if not location then + return false + end + location = gsub(location, "%s", "") + if location == "" then + return false + end + local scheme = match(location, "^([%w][%w%+%-%.]*)%:") + if scheme and not SCHEMES[scheme] then + return false + end + local method = request.method + local redirect = request.redirect + local redirects = request.nredirects or 0 + return redirect and validredirects[code] and validmethods[method] and redirects <= maxredericts +end + +local function shouldreceivebody(request, code) + if request.method == "HEAD" then + return nil + end + if code == 204 or code == 304 then + return nil + end + if code >= 100 and code < 200 then + return nil + end + return 1 +end + +local tredirect, trequest, srequest + +tredirect = function(request, location) + local result, code, headers, status = trequest { + url = absoluteurl(request.url,location), + source = request.source, + sink = request.sink, + headers = request.headers, + proxy = request.proxy, + nredirects = (request.nredirects or 0) + 1, + create = request.create, + } + if not headers then + headers = { } + end + if not headers.location then + headers.location = location + end + return result, code, headers, status +end + +trequest = function(originalrequest) + local request = adjustrequest(originalrequest) + local connection = openhttp(request.host, request.port, request.create) + local headers = request.headers + connection:sendrequestline(request.method, request.uri) + connection:sendheaders(headers) + if request.source then + connection:sendbody(headers, request.source, request.step) + end + local code, status = connection:receivestatusline() + if not code then + connection:receive09body(status, request.sink, request.step) + return 1, 200 + end + while code == 100 do + headers = connection:receiveheaders() + code, status = connection:receivestatusline() + end + headers = connection:receiveheaders() + if shouldredirect(request, code, headers) and not request.source then + connection:close() + return tredirect(originalrequest,headers.location) + end + if shouldreceivebody(request, code) then + connection:receivebody(headers, request.sink, request.step) + end + connection:close() + return 1, code, headers, status +end + +-- turns an url and a body into a generic request + +local function genericform(url, body) + local buffer = { } + local request = { + url = url, + sink = sinktable(buffer), + target = buffer, + } + if body then + request.source = stringsource(body) + request.method = "POST" + request.headers = { + ["content-length"] = #body, + ["content-type"] = "application/x-www-form-urlencoded" + } + end + return request +end + +http.genericform = genericform + +srequest = function(url, body) + local request = genericform(url, body) + local _, code, headers, status = trequest(request) + return concat(request.target), code, headers, status +end + +http.request = protectsocket(function(request, body) + if type(request) == "string" then + return srequest(request, body) + else + return trequest(request) + end +end) + +return http diff --git a/tex/context/base/mkiv/util-soc-imp-ltn12.lua b/tex/context/base/mkiv/util-soc-imp-ltn12.lua new file mode 100644 index 000000000..0a389896b --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-ltn12.lua @@ -0,0 +1,388 @@ +-- original file : ltn12.lua +-- for more into : see util-soc.lua + +local select, unpack = select, unpack +local insert, remove = table.insert, table.remove +local sub = string.sub + +local report = logs and logs.reporter("ltn12") or function(fmt,first,...) + if fmt then + fmt = "ltn12: " .. fmt + if first then + print(format(fmt,first,...)) + else + print(fmt) + end + end +end + +local filter = { } +local source = { } +local sink = { } +local pump = { } + +local ltn12 = { + + _VERSION = "LTN12 1.0.3", + + BLOCKSIZE = 2048, + + filter = filter, + source = source, + sink = sink, + pump = pump, + + report = report, + +} + +-- returns a high level filter that cycles a low-level filter + +function filter.cycle(low, ctx, extra) + if low then + return function(chunk) + return (low(ctx, chunk, extra)) + end + end +end + +-- chains a bunch of filters together + +function filter.chain(...) + local arg = { ... } + local n = select('#',...) + local top = 1 + local index = 1 + local retry = "" + return function(chunk) + retry = chunk and retry + while true do + local action = arg[index] + if index == top then + chunk = action(chunk) + if chunk == "" or top == n then + return chunk + elseif chunk then + index = index + 1 + else + top = top + 1 + index = top + end + else + chunk = action(chunk or "") + if chunk == "" then + index = index - 1 + chunk = retry + elseif chunk then + if index == n then + return chunk + else + index = index + 1 + end + else + report("error: filter returned inappropriate 'nil'") + return + end + end + end + end +end + +-- create an empty source + +local function empty() + return nil +end + +function source.empty() + return empty +end + +-- returns a source that just outputs an error + +local function sourceerror(err) + return function() + return nil, err + end +end + +source.error = sourceerror + +-- creates a file source + +function source.file(handle, io_err) + if handle then + local blocksize = ltn12.BLOCKSIZE + return function() + local chunk = handle:read(blocksize) + if not chunk then + handle:close() + end + return chunk + end + else + return sourceerror(io_err or "unable to open file") + end +end + +-- turns a fancy source into a simple source + +function source.simplify(src) + return function() + local chunk, err_or_new = src() + if err_or_new then + src = err_or_new + end + if chunk then + return chunk + else + return nil, err_or_new + end + end +end + +-- creates string source + +function source.string(s) + if s then + local blocksize = ltn12.BLOCKSIZE + local i = 1 + return function() + local nexti = i + blocksize + local chunk = sub(s, i, nexti - 1) + i = nexti + if chunk ~= "" then + return chunk + else + return nil + end + end + else return source.empty() end +end + +-- creates rewindable source + +function source.rewind(src) + local t = { } + return function(chunk) + if chunk then + insert(t, chunk) + else + chunk = remove(t) + if chunk then + return chunk + else + return src() + end + end + end +end + +-- chains a source with one or several filter(s) + +function source.chain(src, f, ...) + if ... then + f = filter.chain(f, ...) + end + local last_in = "" + local last_out = "" + local state = "feeding" + local err + return function() + if not last_out then + report("error: source is empty") + return + end + while true do + if state == "feeding" then + last_in, err = src() + if err then + return nil, err + end + last_out = f(last_in) + if not last_out then + if last_in then + report("error: filter returned inappropriate 'nil'") + end + return nil + elseif last_out ~= "" then + state = "eating" + if last_in then + last_in = "" + end + return last_out + end + else + last_out = f(last_in) + if last_out == "" then + if last_in == "" then + state = "feeding" + else + report("error: filter returned nothing") + return + end + elseif not last_out then + if last_in then + report("filter returned inappropriate 'nil'") + end + return nil + else + return last_out + end + end + end + end +end + +-- creates a source that produces contents of several sources, one after the +-- other, as if they were concatenated + +function source.cat(...) + local arg = { ... } + local src = remove(arg,1) + return function() + while src do + local chunk, err = src() + if chunk then + return chunk + end + if err then + return nil, err + end + src = remove(arg,1) + end + end +end + +-- creates a sink that stores into a table + +function sink.table(t) + if not t then + t = { } + end + local f = function(chunk, err) + if chunk then + insert(t, chunk) + end + return 1 + end + return f, t +end + +-- turns a fancy sink into a simple sink + +function sink.simplify(snk) + return function(chunk, err) + local ret, err_or_new = snk(chunk, err) + if not ret then + return nil, err_or_new + end + if err_or_new then + snk = err_or_new + end + return 1 + end +end + +-- creates a sink that discards data + +local function null() + return 1 +end + +function sink.null() + return null +end + +-- creates a sink that just returns an error + +local function sinkerror(err) + return function() + return nil, err + end +end + +sink.error = sinkerror + +-- creates a file sink + +function sink.file(handle, io_err) + if handle then + return function(chunk, err) + if not chunk then + handle:close() + return 1 + else + return handle:write(chunk) + end + end + else + return sinkerror(io_err or "unable to open file") + end +end + +-- chains a sink with one or several filter(s) + +function sink.chain(f, snk, ...) + if ... then + local args = { f, snk, ... } + snk = remove(args, #args) + f = filter.chain(unpack(args)) + end + return function(chunk, err) + if chunk ~= "" then + local filtered = f(chunk) + local done = chunk and "" + while true do + local ret, snkerr = snk(filtered, err) + if not ret then + return nil, snkerr + end + if filtered == done then + return 1 + end + filtered = f(done) + end + else + return 1 + end + end +end + +-- pumps one chunk from the source to the sink + +function pump.step(src, snk) + local chunk, src_err = src() + local ret, snk_err = snk(chunk, src_err) + if chunk and ret then + return 1 + else + return nil, src_err or snk_err + end +end + +-- pumps all data from a source to a sink, using a step function + +function pump.all(src, snk, step) + if not step then + step = pump.step + end + while true do + local ret, err = step(src, snk) + if not ret then + if err then + return nil, err + else + return 1 + end + end + end +end + +if logs then + _G.ltn12 = ltn12 + package.loaded.ltn12 = ltn12 + -- report("module (re)installed") +end + +return ltn12 diff --git a/tex/context/base/mkiv/util-soc-imp-mime.lua b/tex/context/base/mkiv/util-soc-imp-mime.lua new file mode 100644 index 000000000..b1a5827ac --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-mime.lua @@ -0,0 +1,105 @@ +-- original file : mime.lua +-- for more into : see util-soc.lua + +local type, tostring = type, tostring + +local mime = require("mime.core") +local ltn12 = ltn12 or require("ltn12") + +local filtercycle = ltn12.filter.cycle + +local report = logs and logs.reporter("mime") or function(fmt,first,...) + if fmt then + fmt = "mime: " .. fmt + if first then + print(format(fmt,first,...)) + else + print(fmt) + end + end +end + +mime.report = report + +local encodet = { } +local decodet = { } +local wrapt = { } + +mime.encodet = encodet +mime.decodet = decodet +mime.wrapt = wrapt + +local mime_b64 = mime.b64 +local mime_qp = mime.qp +local mime_unb64 = mime.unb64 +local mime_unqp = mime.unqp +local mime_wrp = mime.wrp +local mime_qpwrp = mime.qpwrp +local mime_eol = mime_eol +local mime_dot = mime_dot + +encodet['base64'] = function() + return filtercycle(mime_b64,"") +end + +encodet['quoted-printable'] = function(mode) + return filtercycle(mime_qp, "", mode == "binary" and "=0D=0A" or "\r\n") +end + +decodet['base64'] = function() + return filtercycle(mime_unb64, "") +end + +decodet['quoted-printable'] = function() + return filtercycle(mime_unqp, "") +end + +local wraptext = function(length) + if not length then + length = 76 + end + return filtercycle(mime_wrp, length, length) +end + +local wrapquoted = function() + return filtercycle(mime_qpwrp, 76, 76) +end + +wrapt['text'] = wraptext +wrapt['base64'] = wraptext +wrapt['default'] = wraptext +wrapt['quoted-printable'] = wrapquoted + +function mime.normalize(marker) + return filtercycle(mime_eol, 0, marker) +end + +function mime.stuff() + return filtercycle(mime_dot, 2) +end + +local function choose(list) + return function(name, opt1, opt2) + if type(name) ~= "string" then + name, opt1, opt2 = "default", name, opt1 + end + local filter = list[name or "nil"] + if filter then + return filter(opt1, opt2) + else + report("error: unknown key '%s'",tostring(name)) + end + end +end + +mime.encode = choose(encodet) +mime.decode = choose(decodet) +mime.wrap = choose(wrapt) + +if logs then + _G.mime = mime + package.loaded.mime = mime + -- report("module (re)installed") +end + +return mime diff --git a/tex/context/base/mkiv/util-soc-imp-reset.lua b/tex/context/base/mkiv/util-soc-imp-reset.lua new file mode 100644 index 000000000..a4a489b0f --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-reset.lua @@ -0,0 +1,13 @@ +local loaded = package.loaded + +loaded["socket"] = nil +loaded["copas"] = nil +loaded["ltn12"] = nil +loaded["mbox"] = nil +loaded["mime"] = nil +loaded["socket.url"] = nil +loaded["socket.headers"] = nil +loaded["socket.tp"] = nil +loaded["socket.http"] = nil +loaded["socket.ftp"] = nil +loaded["socket.smtp"] = nil diff --git a/tex/context/base/mkiv/util-soc-imp-smtp.lua b/tex/context/base/mkiv/util-soc-imp-smtp.lua new file mode 100644 index 000000000..c13a02688 --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-smtp.lua @@ -0,0 +1,265 @@ +-- original file : smtp.lua +-- for more into : see util-soc.lua + +local type, setmetatable, next = type, setmetatable, next +local find, lower, format = string.find, string.lower, string.format +local osdate, osgetenv = os.data, os.getenv +local random = math.random + +local socket = socket or require("socket") +local headers = socket.headers or require("socket.headers") +local ltn12 = ltn12 or require("ltn12") +local tp = socket.tp or require("socket.tp") +local mime = mime or require("mime") + +local mimeb64 = mime.b64 +local mimestuff = mime.stuff + +local skipsocket = socket.skip +local trysocket = socket.try +local newtrysocket = socket.newtry +local protectsocket = socket.protect + +local normalizeheaders = headers.normalize +local lowerheaders = headers.lower + +local createcoroutine = coroutine.create +local resumecoroutine = coroutine.resume +local yieldcoroutine = coroutine.resume + +local smtp = { + TIMEOUT = 60, + SERVER = "localhost", + PORT = 25, + DOMAIN = osgetenv("SERVER_NAME") or "localhost", + ZONE = "-0000", +} + +socket.smtp = smtp + +local methods = { } +local mt = { __index = methods } + +function methods.greet(self, domain) + local try = self.try + local tp = self.tp + try(tp:check("2..")) + try(tp:command("EHLO", domain or _M.DOMAIN)) + return skipsocket(1, try(tp:check("2.."))) +end + +function methods.mail(self, from) + local try = self.try + local tp = self.tp + try(tp:command("MAIL", "FROM:" .. from)) + return try(tp:check("2..")) +end + +function methods.rcpt(self, to) + local try = self.try + local tp = self.tp + try(tp:command("RCPT", "TO:" .. to)) + return try(tp:check("2..")) +end + +function methods.data(self, src, step) + local try = self.try + local tp = self.tp + try(tp:command("DATA")) + try(tp:check("3..")) + try(tp:source(src, step)) + try(tp:send("\r\n.\r\n")) + return try(tp:check("2..")) +end + +function methods.quit(self) + local try = self.try + local tp = self.tp + try(tp:command("QUIT")) + return try(tp:check("2..")) +end + +function methods.close(self) + return self.tp:close() +end + +function methods.login(self, user, password) + local try = self.try + local tp = self.tp + try(tp:command("AUTH", "LOGIN")) + try(tp:check("3..")) + try(tp:send(mimeb64(user) .. "\r\n")) + try(tp:check("3..")) + try(tp:send(mimeb64(password) .. "\r\n")) + return try(tp:check("2..")) +end + +function methods.plain(self, user, password) + local try = self.try + local tp = self.tp + local auth = "PLAIN " .. mimeb64("\0" .. user .. "\0" .. password) + try(tp:command("AUTH", auth)) + return try(tp:check("2..")) +end + +function methods.auth(self, user, password, ext) + if not user or not password then + return 1 + end + local try = self.try + if find(ext, "AUTH[^\n]+LOGIN") then + return self:login(user,password) + elseif find(ext, "AUTH[^\n]+PLAIN") then + return self:plain(user,password) + else + try(nil, "authentication not supported") + end +end + +function methods.send(self, mail) + self:mail(mail.from) + local receipt = mail.rcpt + if type(receipt) == "table" then + for i=1,#receipt do + self:rcpt(receipt[i]) + end + elseif receipt then + self:rcpt(receipt) + end + self:data(ltn12.source.chain(mail.source, mimestuff()), mail.step) +end + +local function opensmtp(self, server, port, create) + if not server or server == "" then + server = smtp.SERVER + end + if not port or port == "" then + port = smtp.PORT + end + local s = { + tp = trysocket(tp.connect(server, port, smtp.TIMEOUT, create)), + try = newtrysocket(function() + s:close() + end), + } + setmetatable(s, mt) + return s +end + +smtp.open = opensmtp + +local nofboundaries = 0 + +local function newboundary() + nofboundaries = nofboundaries + 1 + return format('%s%05d==%05u', osdate('%d%m%Y%H%M%S'), random(0,99999), nofboundaries) +end + +local send_message + +local function send_headers(headers) + yieldcoroutine(normalizeheaders(headers)) +end + +local function send_multipart(message) + local boundary = newboundary() + local headers = lowerheaders(message.headers) + local body = message.body + local preamble = body.preamble + local epilogue = body.epilogue + local content = headers['content-type'] or 'multipart/mixed' + headers['content-type'] = content .. '; boundary="' .. boundary .. '"' + send_headers(headers) + if preamble then + yieldcoroutine(preamble) + yieldcoroutine("\r\n") + end + for i=1,#body do + yieldcoroutine("\r\n--" .. boundary .. "\r\n") + send_message(body[i]) + end + yieldcoroutine("\r\n--" .. boundary .. "--\r\n\r\n") + if epilogue then + yieldcoroutine(epilogue) + yieldcoroutine("\r\n") + end +end + +local default_content_type = 'text/plain; charset="UTF-8"' + +local function send_source(message) + local headers = lowerheaders(message.headers) + if not headers['content-type'] then + headers['content-type'] = default_content_type + end + send_headers(headers) + local getchunk = message.body + while true do + local chunk, err = getchunk() + if err then + yieldcoroutine(nil, err) + elseif chunk then + yieldcoroutine(chunk) + else + break + end + end +end + +local function send_string(message) + local headers = lowerheaders(message.headers) + if not headers['content-type'] then + headers['content-type'] = default_content_type + end + send_headers(headers) + yieldcoroutine(message.body) +end + +function send_message(message) + local body = message.body + if type(body) == "table" then + send_multipart(message) + elseif type(body) == "function" then + send_source(message) + else + send_string(message) + end +end + +local function adjust_headers(message) + local headers = lowerheaders(message.headers) + if not headers["date"] then + headers["date"] = osdate("!%a, %d %b %Y %H:%M:%S ") .. (message.zone or smtp.ZONE) + end + if not headers["x-mailer"] then + headers["x-mailer"] = socket._VERSION + end + headers["mime-version"] = "1.0" + return headers +end + +function smtp.message(message) + message.headers = adjust_headers(message) + local action = createcoroutine(function() + send_message(message) + end) + return function() + local ret, a, b = resumecoroutine(action) + if ret then + return a, b + else + return nil, a + end + end +end + +smtp.send = protectsocket(function(mail) + local snd = opensmtp(mail.server, mail.port, mail.create) + local ext = snd:greet(mail.domain) + snd:auth(mail.user, mail.password, ext) + snd:send(mail) + snd:quit() + return snd:close() +end) + +return smtp diff --git a/tex/context/base/mkiv/util-soc-imp-socket.lua b/tex/context/base/mkiv/util-soc-imp-socket.lua new file mode 100644 index 000000000..0ad685d75 --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-socket.lua @@ -0,0 +1,190 @@ +-- original file : socket.lua +-- for more into : see util-soc.lua + +local type, tostring, setmetatable = type, tostring, setmetatable +local min = math.min +local format = string.format + +local socket = require("socket.core") + +local connect = socket.connect +local tcp4 = socket.tcp4 +local tcp6 = socket.tcp6 +local getaddrinfo = socket.dns.getaddrinfo + +local report = logs and logs.reporter("socket") or function(fmt,first,...) + if fmt then + fmt = "socket: " .. fmt + if first then + print(format(fmt,first,...)) + else + print(fmt) + end + end +end + +socket.report = report + +function socket.connect4(address, port, laddress, lport) + return connect(address, port, laddress, lport, "inet") +end + +function socket.connect6(address, port, laddress, lport) + return connect(address, port, laddress, lport, "inet6") +end + +function socket.bind(host, port, backlog) + if host == "*" or host == "" then + host = defaulthost + end + local addrinfo, err = getaddrinfo(host) + if not addrinfo then + return nil, err + end + for i=1,#addrinfo do + local alt = addrinfo[i] + local sock, err = (alt.family == "inet" and tcp4 or tcp6)() + if not sock then + return nil, err or "unknown error" + end + sock:setoption("reuseaddr", true) + local res, err = sock:bind(alt.addr, port) + if res then + res, err = sock:listen(backlog) + if res then + return sock + else + sock:close() + end + else + sock:close() + end + end + return nil, "invalid address" +end + +socket.try = socket.newtry() + +function socket.choose(list) + return function(name, opt1, opt2) + if type(name) ~= "string" then + name, opt1, opt2 = "default", name, opt1 + end + local f = list[name or "nil"] + if f then + return f(opt1, opt2) + else + report("error: unknown key '%s'",tostring(name)) + end + end +end + +local sourcet = { } +local sinkt = { } + +socket.sourcet = sourcet +socket.sinkt = sinkt + +socket.BLOCKSIZE = 2048 + +sinkt["close-when-done"] = function(sock) + return setmetatable ( + { + getfd = function() return sock:getfd() end, + dirty = function() return sock:dirty() end, + }, + { + __call = function(self, chunk, err) + if chunk then + return sock:send(chunk) + else + sock:close() + return 1 -- why 1 + end + end + } + ) +end + +sinkt["keep-open"] = function(sock) + return setmetatable ( + { + getfd = function() return sock:getfd() end, + dirty = function() return sock:dirty() end, + }, { + __call = function(self, chunk, err) + if chunk then + return sock:send(chunk) + else + return 1 -- why 1 + end + end + } + ) +end + +sinkt["default"] = sinkt["keep-open"] + +socket.sink = socket.choose(sinkt) + +sourcet["by-length"] = function(sock, length) + local blocksize = socket.BLOCKSIZE + return setmetatable ( + { + getfd = function() return sock:getfd() end, + dirty = function() return sock:dirty() end, + }, + { + __call = function() + if length <= 0 then + return nil + end + local chunk, err = sock:receive(min(blocksize,length)) + if err then + return nil, err + end + length = length - #chunk + return chunk + end + } + ) +end + +sourcet["until-closed"] = function(sock) + local blocksize = socket.BLOCKSIZE + local done = false + return setmetatable ( + { + getfd = function() return sock:getfd() end, + dirty = function() return sock:dirty() end, + }, { + __call = function() + if done then + return nil + end + local chunk, status, partial = sock:receive(blocksize) + if not status then + return chunk + elseif status == "closed" then + sock:close() + done = true + return partial + else + return nil, status + end + end + } + ) +end + +sourcet["default"] = sourcet["until-closed"] + +socket.source = socket.choose(sourcet) + +if logs then + _G.socket = socket + package.loaded.socket = socket + -- report("module (re)installed") +end + +return socket diff --git a/tex/context/base/mkiv/util-soc-imp-tp.lua b/tex/context/base/mkiv/util-soc-imp-tp.lua new file mode 100644 index 000000000..de3f3f5af --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-tp.lua @@ -0,0 +1,142 @@ +-- original file : tp.lua +-- for more into : see util-soc.lua + +local setmetatable, next, type, tonumber = setmetatable, next, type, tonumber +local find, upper = string.find, string,upper + +local socket = socket or require("socket") +local ltn12 = ltn12 or require("ltn12") + +local skipsocket = socket.skip +local sinksocket = socket.sink +local tcpsocket = socket.tcp + +local ltn12pump = ltn12.pump +local pumpall = ltn12pump.all +local pumpstep = ltn12pump.step + +local tp = { + TIMEOUT = 60, +} + +socket.tp = tp + +local function get_reply(c) + local line, err = c:receive() + local reply = line + if err then return + nil, err + end + local code, sep = skipsocket(2, find(line, "^(%d%d%d)(.?)")) + if not code then + return nil, "invalid server reply" + end + if sep == "-" then + local current + repeat + line, err = c:receive() + if err then + return nil, err + end + current, sep = skipsocket(2, find(line, "^(%d%d%d)(.?)")) + reply = reply .. "\n" .. line + until code == current and sep == " " + end + return code, reply +end + +local methods = { } +local mt = { __index = methods } + +function methods.getpeername(self) + return self.c:getpeername() +end + +function methods.getsockname(self) + return self.c:getpeername() +end + +function methods.check(self, ok) + local code, reply = get_reply(self.c) + if not code then + return nil, reply + end + local c = tonumber(code) + local t = type(ok) + if t == "function" then + return ok(c,reply) + elseif t == "table" then + for i=1,#ok do + if find(code,ok[i]) then + return c, reply + end + end + return nil, reply + elseif find(code, ok) then + return c, reply + else + return nil, reply + end +end + +function methods.command(self, cmd, arg) + cmd = upper(cmd) + if arg then + cmd = cmd .. " " .. arg .. "\r\n" + else + cmd = cmd .. "\r\n" + end + return self.c:send(cmd) +end + +function methods.sink(self, snk, pat) + local chunk, err = self.c:receive(pat) + return snk(chunk, err) +end + +function methods.send(self, data) + return self.c:send(data) +end + +function methods.receive(self, pat) + return self.c:receive(pat) +end + +function methods.getfd(self) + return self.c:getfd() +end + +function methods.dirty(self) + return self.c:dirty() +end + +function methods.getcontrol(self) + return self.c +end + +function methods.source(self, source, step) + local sink = sinksocket("keep-open", self.c) + local ret, err = pumpall(source, sink, step or pumpstep) + return ret, err +end + +function methods.close(self) + self.c:close() + return 1 +end + +function tp.connect(host, port, timeout, create) + local c, e = (create or tcpsocket)() + if not c then + return nil, e + end + c:settimeout(timeout or tp.TIMEOUT) + local r, e = c:connect(host, port) + if not r then + c:close() + return nil, e + end + return setmetatable({ c = c }, mt) +end + +return tp diff --git a/tex/context/base/mkiv/util-soc-imp-url.lua b/tex/context/base/mkiv/util-soc-imp-url.lua new file mode 100644 index 000000000..5f2c82841 --- /dev/null +++ b/tex/context/base/mkiv/util-soc-imp-url.lua @@ -0,0 +1,266 @@ +-- original file : url.lua +-- for more into : see util-soc.lua + +local tonumber, tostring, type = tonumber, tostring, type + +local gsub, sub, match, find, format, byte, char = string.gsub, string.sub, string.match, string.find, string.format, string.byte, string.char +local insert = table.insert + +local socket = socket or require("socket") + +local url = { + _VERSION = "URL 1.0.3", +} + +socket.url = url + +function url.escape(s) + return (gsub(s, "([^A-Za-z0-9_])", function(c) + return format("%%%02x", byte(c)) + end)) +end + +local function make_set(t) -- table.tohash + local s = { } + for i=1,#t do + s[t[i]] = true + end + return s +end + +local segment_set = make_set { + "-", "_", ".", "!", "~", "*", "'", "(", + ")", ":", "@", "&", "=", "+", "$", ",", +} + +local function protect_segment(s) + return gsub(s, "([^A-Za-z0-9_])", function(c) + if segment_set[c] then + return c + else + return format("%%%02X", byte(c)) + end + end) +end + +function url.unescape(s) + return (gsub(s, "%%(%x%x)", function(hex) + return char(tonumber(hex,16)) + end)) +end + +local function absolute_path(base_path, relative_path) + if find(relative_path,"^/") then + return relative_path + end + local path = gsub(base_path, "[^/]*$", "") + path = path .. relative_path + path = gsub(path, "([^/]*%./)", function (s) + if s ~= "./" then + return s + else + return "" + end + end) + path = gsub(path, "/%.$", "/") + local reduced + while reduced ~= path do + reduced = path + path = gsub(reduced, "([^/]*/%.%./)", function (s) + if s ~= "../../" then + return "" + else + return s + end + end) + end + path = gsub(reduced, "([^/]*/%.%.)$", function (s) + if s ~= "../.." then + return "" + else + return s + end + end) + return path +end + +function url.parse(url, default) + local parsed = { } + for k, v in next, default or parsed do + parsed[k] = v + end + if not url or url == "" then + return nil, "invalid url" + end + url = gsub(url, "#(.*)$", function(f) + parsed.fragment = f + return "" + end) + url = gsub(url, "^([%w][%w%+%-%.]*)%:", function(s) + parsed.scheme = s + return "" + end) + url = gsub(url, "^//([^/]*)", function(n) + parsed.authority = n + return "" + end) + url = gsub(url, "%?(.*)", function(q) + parsed.query = q + return "" + end) + url = gsub(url, "%;(.*)", function(p) + parsed.params = p + return "" + end) + if url ~= "" then + parsed.path = url + end + local authority = parsed.authority + if not authority then + return parsed + end + authority = gsub(authority,"^([^@]*)@", function(u) + parsed.userinfo = u + return "" + end) + authority = gsub(authority, ":([^:%]]*)$", function(p) + parsed.port = p + return "" + end) + if authority ~= "" then + parsed.host = match(authority, "^%[(.+)%]$") or authority + end + local userinfo = parsed.userinfo + if not userinfo then + return parsed + end + userinfo = gsub(userinfo, ":([^:]*)$", function(p) + parsed.password = p + return "" + end) + parsed.user = userinfo + return parsed +end + +function url.build(parsed) + local url = parsed.path or "" + if parsed.params then + url = url .. ";" .. parsed.params + end + if parsed.query then + url = url .. "?" .. parsed.query + end + local authority = parsed.authority + if parsed.host then + authority = parsed.host + if find(authority, ":") then -- IPv6? + authority = "[" .. authority .. "]" + end + if parsed.port then + authority = authority .. ":" .. tostring(parsed.port) + end + local userinfo = parsed.userinfo + if parsed.user then + userinfo = parsed.user + if parsed.password then + userinfo = userinfo .. ":" .. parsed.password + end + end + if userinfo then authority = userinfo .. "@" .. authority end + end + if authority then + url = "//" .. authority .. url + end + if parsed.scheme then + url = parsed.scheme .. ":" .. url + end + if parsed.fragment then + url = url .. "#" .. parsed.fragment + end + return url +end + +function url.absolute(base_url, relative_url) + local base_parsed + if type(base_url) == "table" then + base_parsed = base_url + base_url = url.build(base_parsed) + else + base_parsed = url.parse(base_url) + end + local relative_parsed = url.parse(relative_url) + if not base_parsed then + return relative_url + elseif not relative_parsed then + return base_url + elseif relative_parsed.scheme then + return relative_url + else + relative_parsed.scheme = base_parsed.scheme + if not relative_parsed.authority then + relative_parsed.authority = base_parsed.authority + if not relative_parsed.path then + relative_parsed.path = base_parsed.path + if not relative_parsed.params then + relative_parsed.params = base_parsed.params + if not relative_parsed.query then + relative_parsed.query = base_parsed.query + end + end + else + relative_parsed.path = absolute_path(base_parsed.path or "", relative_parsed.path) + end + end + return url.build(relative_parsed) + end +end + +function url.parse_path(path) + local parsed = { } + path = path or "" + gsub(path, "([^/]+)", function (s) + insert(parsed, s) + end) + for i=1,#parsed do + parsed[i] = url.unescape(parsed[i]) + end + if sub(path, 1, 1) == "/" then + parsed.is_absolute = 1 + end + if sub(path, -1, -1) == "/" then + parsed.is_directory = 1 + end + return parsed +end + +function url.build_path(parsed, unsafe) + local path = "" + local n = #parsed + if unsafe then + for i = 1, n-1 do + path = path .. parsed[i] .. "/" + end + if n > 0 then + path = path .. parsed[n] + if parsed.is_directory then + path = path .. "/" + end + end + else + for i = 1, n-1 do + path = path .. protect_segment(parsed[i]) .. "/" + end + if n > 0 then + path = path .. protect_segment(parsed[n]) + if parsed.is_directory then + path = path .. "/" + end + end + end + if parsed.is_absolute then + path = "/" .. path + end + return path +end + +return url diff --git a/tex/context/base/mkiv/util-soc.lua b/tex/context/base/mkiv/util-soc.lua index 3a52ee86d..29b93635c 100644 --- a/tex/context/base/mkiv/util-soc.lua +++ b/tex/context/base/mkiv/util-soc.lua @@ -6,6 +6,29 @@ if not modules then modules = { } end modules ['util-soc'] = { license = "see context related readme files" } +--[[-- + +In LuaTeX we provide the socket library that is more or less the standard one for +Lua. It has been around for a while and seems to be pretty stable. The binary +module is copmpiled into LuaTeX and the accompanying .lua files are preloaded. +These files are mostly written by Diego Nehab, Andre Carregal, Javier Guerra, and +Fabio Mascarenhas with contributions from Diego Nehab, Mike Pall, David Burgess, +Leonardo Godinho, Thomas Harning Jr., and Gary NG. The originals are part of and +copyrighted by the Kepler project. + +Here we reload a slightly reworked version of these .lua files. We keep the same +(documented) interface but streamlined some fo the code. No more modules, no more +pre 5.2 Lua, etc. Also, as it loads into the ConTeXt ecosystem, we plug in some +logging. (and maybe tracing in the future). As we don't support serial ports in +LuaTeX, related code has been dropped. + +The files are reformatted so that we can more easilly add additional features +and/or tracing options. Any error introduced there is our fault! The url module +might be replaced by the one in ConTeXt. When we need mbox a suitable variant +will be provided. + +--]]-- + local format = string.format local smtp = require("socket.smtp") diff --git a/tex/context/base/mkiv/util-str.lua b/tex/context/base/mkiv/util-str.lua index 3ad30757b..29305f3bb 100644 --- a/tex/context/base/mkiv/util-str.lua +++ b/tex/context/base/mkiv/util-str.lua @@ -918,14 +918,33 @@ end -- return format("tostring(tonumber(a%s) or a%s)",n,n) -- end -local format_N = function(f) -- strips leading and trailing zeros (also accepts string) +-- local format_N = function(f) -- strips leading and trailing zeros +-- n = n + 1 +-- -- stripzero (singular) as we only have a number +-- if not f or f == "" then +-- return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or ((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%.9f',a%s)))",n,n,n,n,n) +-- else +-- return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) +-- end +-- end + +-- local format_N = function(f) -- strips leading and trailing zeros +-- n = n + 1 +-- -- stripzero (singular) as we only have a number +-- if not f or f == "" then +-- return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or ((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or lpegmatch(stripzero,format('%%.9f',a%s)))",n,n,n,n,n) +-- else +-- return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) +-- end +-- end + +local format_N = function(f) -- strips leading and trailing zeros n = n + 1 -- stripzero (singular) as we only have a number if not f or f == "" then - return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or ((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%.9f',a%s)))",n,n,n,n,n) - else - return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) - end + f = ".9" + end -- always a leading number ! + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) end local format_a = function(f) diff --git a/tex/context/base/mkiv/util-you.lua b/tex/context/base/mkiv/util-you.lua index 32a7e07d4..5802e7d7a 100644 --- a/tex/context/base/mkiv/util-you.lua +++ b/tex/context/base/mkiv/util-you.lua @@ -30,7 +30,6 @@ utilities.youless = youless local lpegmatch = lpeg.match local formatters = string.formatters -local sortedhash = table.sortedhash local tonumber, type, next = tonumber, type, next diff --git a/tex/context/interface/mkiv/i-context.pdf b/tex/context/interface/mkiv/i-context.pdf Binary files differindex 96e7b9c5a..28123182c 100644 --- a/tex/context/interface/mkiv/i-context.pdf +++ b/tex/context/interface/mkiv/i-context.pdf diff --git a/tex/context/interface/mkiv/i-readme.pdf b/tex/context/interface/mkiv/i-readme.pdf Binary files differindex afe48ba5a..4bd42a9cd 100644 --- a/tex/context/interface/mkiv/i-readme.pdf +++ b/tex/context/interface/mkiv/i-readme.pdf diff --git a/tex/context/modules/mkiv/s-languages-system.lua b/tex/context/modules/mkiv/s-languages-system.lua index 3b422db9f..d18050577 100644 --- a/tex/context/modules/mkiv/s-languages-system.lua +++ b/tex/context/modules/mkiv/s-languages-system.lua @@ -19,7 +19,7 @@ local ctx_bold = context.bold function moduledata.languages.system.loadinstalled() context.start() - for k, v in table.sortedhash(registered) do + for k, v in sortedhash(registered) do context.language{ k } end context.stop() diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 2cbb670ce..4336f8a25 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 08/10/18 16:51:00 +-- merge date : 08/14/18 23:10:05 do -- begin closure to overcome local limits and interference @@ -880,7 +880,7 @@ do local nonzero=digit-zero local trailingzeros=zero^1*endofstring local stripper=Cs((1-period)^0*( - (period*trailingzeros/"")+period*(nonzero^1+(trailingzeros/"")+zero^1)^0 + period*trailingzeros/""+period*(nonzero^1+(trailingzeros/"")+zero^1)^0+endofstring )) lpeg.patterns.stripzero=stripper end @@ -4375,10 +4375,9 @@ end local format_N=function(f) n=n+1 if not f or f=="" then - return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or ((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%.9f',a%s)))",n,n,n,n,n) - else - return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) - end + f=".9" + end + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripzero,format('%%%sf',a%s)))",n,n,f,n) end local format_a=function(f) n=n+1 @@ -10737,7 +10736,16 @@ local function tounicode16sequence(unicodes) return concat(t) end local unknown=f_single(0xFFFD) -local hash=table.setmetatableindex(function(t,k) +local hash={} +local conc={} +table.setmetatableindex(hash,function(t,k) + if type(k)=="table" then + local n=#k + for l=1,n do + conc[l]=hash[k[l]] + end + return concat(conc,"",1,n) + end local v if k>=0x00E000 and k<=0x00F8FF then v=unknown @@ -10754,17 +10762,8 @@ local hash=table.setmetatableindex(function(t,k) t[k]=v return v end) -table.makeweak(hash) -local function tounicode(unicode,name) - if type(unicode)=="table" then - local t={} - for l=1,#unicode do - t[l]=hash[unicode[l]] - end - return concat(t) - else - return hash[unicode] - end +local function tounicode(unicode) + return hash[unicode] end local function fromunicode16(str) if #str==4 then |