diff options
author | Hans Hagen <pragma@wxs.nl> | 2018-06-22 16:42:14 +0200 |
---|---|---|
committer | Context Git Mirror Bot <phg42.2a@gmail.com> | 2018-06-22 16:42:14 +0200 |
commit | da8162d4e816cf49d9790a1c81556b499f442bed (patch) | |
tree | 5c631b1fb8282670a6cea1087afa766a8b94db43 /tex/context/base/mkiv | |
parent | 204057fdc5023b5f4ff55a69bc6593388ea427e7 (diff) | |
download | context-da8162d4e816cf49d9790a1c81556b499f442bed.tar.gz |
2018-06-22 16:02:00
Diffstat (limited to 'tex/context/base/mkiv')
33 files changed, 1770 insertions, 327 deletions
diff --git a/tex/context/base/mkiv/back-pdf.mkiv b/tex/context/base/mkiv/back-pdf.mkiv index 3b0dd7852..6d2cb4c7a 100644 --- a/tex/context/base/mkiv/back-pdf.mkiv +++ b/tex/context/base/mkiv/back-pdf.mkiv @@ -32,7 +32,11 @@ \registerctxluafile{lpdf-swf}{} % this will become a module \registerctxluafile{lpdf-tag}{} \registerctxluafile{lpdf-fmt}{} -\registerctxluafile{lpdf-epd}{} +\ifnum\texenginefunctionality<6802 + \registerctxluafile{lpdf-epd}{} +\else + \registerctxluafile{lpdf-pde}{} +\fi \registerctxluafile{lpdf-epa}{} \registerctxluafile{back-pdf}{} % some code will move to lpdf-* diff --git a/tex/context/base/mkiv/buff-ini.mkiv b/tex/context/base/mkiv/buff-ini.mkiv index b139e2b51..e487fc298 100644 --- a/tex/context/base/mkiv/buff-ini.mkiv +++ b/tex/context/base/mkiv/buff-ini.mkiv @@ -49,7 +49,13 @@ \def\buff_start_indeed#1#2#3#4% {\edef\p_strip{\namedbufferparameter{#1}\c!strip}% for aditya - \normalexpanded{\buff_pickup{#2}{#3}{#4}{}{\buff_stop{#4}}\ifx\p_strip\v!no\zerocount\else\plusone\fi}} + \normalexpanded{\buff_pickup + {#2}% + {#3}% + {#4}% + {}% + {\buff_stop{#4}}% + \ifx\p_strip\v!no\zerocount\else\plusone\fi}} \unexpanded\def\grabbufferdata % was: \dostartbuffer {\begingroup % (4) diff --git a/tex/context/base/mkiv/buff-ver.lua b/tex/context/base/mkiv/buff-ver.lua index 886b62b9e..bfc9ef89c 100644 --- a/tex/context/base/mkiv/buff-ver.lua +++ b/tex/context/base/mkiv/buff-ver.lua @@ -43,6 +43,7 @@ local findfile = resolvers.findfile local addsuffix = file.addsuffix local v_yes = variables.yes +local v_no = variables.no local v_last = variables.last local v_all = variables.all local v_absolute = variables.absolute @@ -736,7 +737,8 @@ end local function filter(lines,settings) -- todo: inline or display in settings local strip = settings.strip - if strip and strip ~= "" then + -- if strip and strip == "" then + if strip ~= v_no and strip ~= false then lines = realign(lines,strip) end local line, n = 0, 0 diff --git a/tex/context/base/mkiv/buff-ver.mkiv b/tex/context/base/mkiv/buff-ver.mkiv index 558049dcc..47902ced7 100644 --- a/tex/context/base/mkiv/buff-ver.mkiv +++ b/tex/context/base/mkiv/buff-ver.mkiv @@ -514,9 +514,16 @@ \normalexpanded{\buff_verbatim_type_block{\e!start\currenttyping}{\e!stop\currenttyping}}} \unexpanded\def\buff_verbatim_type_block#1#2% - {\buff_pickup{_typing_}{#1}{#2}{}{\buff_verbatim_type_block_verbatim_indeed{#1}{#2}}\plusone} % was dowithbuffer - -\def\buff_verbatim_type_block_verbatim_indeed#1#2% + {\edef\p_strip{\typingparameter\c!strip}% + \normalexpanded{\buff_pickup + {_typing_}% + {#1}% + {#2}% + {}% + {\buff_verbatim_type_block_verbatim_indeed{#1}{#2}}% + \ifx\p_strip\v!no\zerocount\else\plusone\fi}} + +\unexpanded\def\buff_verbatim_type_block_verbatim_indeed#1#2% {\buff_verbatim_initialize_typing_two \dostarttaggedchained\t!verbatimblock\currenttyping\??typing \beginofverbatimlines diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 3a9acfb13..95d4d9604 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2018.06.12 21:48} +\newcontextversion{2018.06.22 15:55} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index 1e90175c8..db855db63 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -42,7 +42,7 @@ %D has to match \type {YYYY.MM.DD HH:MM} format. \edef\contextformat {\jobname} -\edef\contextversion{2018.06.12 21:48} +\edef\contextversion{2018.06.22 15:55} \edef\contextkind {beta} %D For those who want to use this: diff --git a/tex/context/base/mkiv/core-env.lua b/tex/context/base/mkiv/core-env.lua index ea7bbcb8e..50759dd19 100644 --- a/tex/context/base/mkiv/core-env.lua +++ b/tex/context/base/mkiv/core-env.lua @@ -26,6 +26,7 @@ local setmetatablenewindex = table.setmetatablenewindex local setmetatablecall = table.setmetatablecall local createtoken = token.create +local isdefined = tokens.isdefined texmodes = allocate { } tex.modes = texmodes texsystemmodes = allocate { } tex.systemmodes = texsystemmodes @@ -68,7 +69,7 @@ setmetatableindex(texmodes, function(t,k) return m() elseif k then local n = "mode>" .. k - if is_defined then + if isdefined(n) then rawset(modes,k, function() return texgetcount(n) == 1 end) return texgetcount(n) == 1 -- 2 is prevented else @@ -89,7 +90,7 @@ setmetatableindex(texsystemmodes, function(t,k) return m() else local n = "mode>*" .. k - if is_defined(n) then + if isdefined(n) then rawset(systemmodes,k,function() return texgetcount(n) == 1 end) return texgetcount(n) == 1 -- 2 is prevented else @@ -122,15 +123,7 @@ setmetatablenewindex(texifs, function(t,k) -- just ignore end) -setmetatableindex(texisdefined, function(t,k) - return k and cache[k].mode ~= 0 -end) -setmetatablecall(texisdefined, function(t,k) - return k and cache[k].mode ~= 0 -end) -setmetatablenewindex(texisdefined, function(t,k) - -- just ignore -end) +tex.isdefined = isdefined function tex.isdimen(name) local hit = cache[name] diff --git a/tex/context/base/mkiv/core-uti.lua b/tex/context/base/mkiv/core-uti.lua index b281b81a4..9ba2c945f 100644 --- a/tex/context/base/mkiv/core-uti.lua +++ b/tex/context/base/mkiv/core-uti.lua @@ -195,6 +195,7 @@ end local packlist = { "numbers", + "ownnumbers", "metadata", "sectiondata", "prefixdata", @@ -209,6 +210,7 @@ local packlist = { local skiplist = { "datasets", "userdata", + "positions", } -- not ok as we can have arbitrary keys in userdata and dataset so some day we diff --git a/tex/context/base/mkiv/font-imp-italics.lua b/tex/context/base/mkiv/font-imp-italics.lua index aace899c5..83c785d38 100644 --- a/tex/context/base/mkiv/font-imp-italics.lua +++ b/tex/context/base/mkiv/font-imp-italics.lua @@ -83,25 +83,25 @@ end -- no longer used -if context then - - -- local function initializemathitalics(tfmdata,value) -- yes no delay - -- tfmdata.properties.mathitalics = toboolean(value) - -- end - -- - -- local specification = { - -- name = "mathitalics", - -- description = "use alternative math italic correction", - -- initializers = { - -- base = initializemathitalics, - -- node = initializemathitalics, - -- } - -- } - -- - -- registerotffeature(specification) - -- registerafmfeature(specification) - -end +-- if context then +-- +-- -- local function initializemathitalics(tfmdata,value) -- yes no delay +-- -- tfmdata.properties.mathitalics = toboolean(value) +-- -- end +-- -- +-- -- local specification = { +-- -- name = "mathitalics", +-- -- description = "use alternative math italic correction", +-- -- initializers = { +-- -- base = initializemathitalics, +-- -- node = initializemathitalics, +-- -- } +-- -- } +-- -- +-- -- registerotffeature(specification) +-- -- registerafmfeature(specification) +-- +-- end -- -- also not used, only when testing diff --git a/tex/context/base/mkiv/font-ocl.lua b/tex/context/base/mkiv/font-ocl.lua index 032df0cf3..fcafe99ed 100644 --- a/tex/context/base/mkiv/font-ocl.lua +++ b/tex/context/base/mkiv/font-ocl.lua @@ -323,16 +323,39 @@ do local hashed = { } local cache = { } - function otf.storepdfdata(pdf) - local done = hashed[pdf] - if not done then - nofstreams = nofstreams + 1 - local o, n = epdf.openMemStream(pdf,#pdf,f_name(nofstreams)) - cache[n] = o -- we need to keep in mem - done = f_used(n) - hashed[pdf] = done + if epdf then + + local openpdf = epdf.openMemStream + + function otf.storepdfdata(pdf) + local done = hashed[pdf] + if not done then + nofstreams = nofstreams + 1 + local o, n = openpdf(pdf,#pdf,f_name(nofstreams)) + cache[n] = o -- we need to keep in mem + done = f_used(n) + hashed[pdf] = done + end + return nil, done, nil end - return nil, done, nil + + else + + -- todo: bypass backend and directly inject xforms + + local openpdf = pdfe.new + + function otf.storepdfdata(pdf) + local done = hashed[pdf] + if not done then + nofstreams = nofstreams + 1 + local n = openpdf(pdf,#pdf,f_name(nofstreams)) + done = f_used(n) + hashed[pdf] = done + end + return nil, done, nil + end + end -- maybe more efficient but much slower (and we hash already) diff --git a/tex/context/base/mkiv/grph-inc.lua b/tex/context/base/mkiv/grph-inc.lua index 87afb8683..37805e8aa 100644 --- a/tex/context/base/mkiv/grph-inc.lua +++ b/tex/context/base/mkiv/grph-inc.lua @@ -120,7 +120,7 @@ function checkimage(figure) local width = figure.width local height = figure.height if width <= 0 or height <= 0 then - report_inclusion("image %a has bad dimensions (%p,%p), discarding",figure.filename,width,height) + report_inclusion("image %a has bad dimensions (%p,%p), discarding",figure.filename or "?",width,height) return false, "bad dimensions" end local xres = figure.xres @@ -1362,16 +1362,18 @@ end local pagecount = { } function checkers.generic(data) - local dr, du, ds = data.request, data.used, data.status - local name = du.fullname or "unknown generic" - local page = du.page or dr.page - local size = dr.size or "crop" - local color = dr.color or "natural" - local mask = dr.mask or "none" - local conversion = dr.conversion - local resolution = dr.resolution - local arguments = dr.arguments - local scanimage = dr.scanimage or scanimage + local dr, du, ds = data.request, data.used, data.status + local name = du.fullname or "unknown generic" + local page = du.page or dr.page + local size = dr.size or "crop" + local color = dr.color or "natural" + local mask = dr.mask or "none" + local conversion = dr.conversion + local resolution = dr.resolution + local arguments = dr.arguments + local scanimage = dr.scanimage or scanimage + local userpassword = dr.userpassword + local ownerpassword = dr.ownerpassword if not conversion or conversion == "" then conversion = "default" end @@ -1398,6 +1400,8 @@ function checkers.generic(data) page = page, pagebox = dr.size, keepopen = dr.keepopen or false, + userpassword = userpassword, + ownerpassword = ownerpassword, -- visiblefilename = "", -- this prohibits the full filename ending up in the file } codeinjections.setfigurecolorspace(data,figure) @@ -1405,7 +1409,11 @@ function checkers.generic(data) if figure then -- new, bonus check if page and page > 1 then - local f = scanimage{ filename = name } + local f = scanimage{ + filename = name, + userpassword = userpassword, + ownerpassword = ownerpassword, + } if f.page and f.pages < page then report_inclusion("no page %i in %a, using page 1",page,name) page = 1 @@ -1966,6 +1974,8 @@ implement { { "transform" }, { "width", "dimen" }, { "height", "dimen" }, + { "userpassword" }, + { "ownerpassword" }, } } } @@ -2055,36 +2065,49 @@ local function pdf_checker(data) local request = data.request local used = data.used if request and used and not request.scanimage then - local openpdf = lpdf.epdf.image.open - ----- closepdf = lpdf.epdf.image.close - local querypdf = lpdf.epdf.image.query - local copypage = lpdf.epdf.image.copy + local image = lpdf.epdf.image + local openpdf = image.open + local closepdf = image.close + local querypdf = image.query + local copypage = image.copy + local pdfdoc = nil request.scanimage = function(t) - local pdfdoc = openpdf(t.filename) + pdfdoc = openpdf(t.filename,request.userpassword,request.ownerpassword) + -- if pdfdoc then + -- used.pdfdoc = pdfdoc + -- -- nofpages + -- end if pdfdoc then - used.pdfdoc = pdfdoc - -- nofpages + local info = querypdf(pdfdoc,request.page) + local bbox = info and info.boundingbox or { 0, 0, 0, 0 } + return { + filename = filename, + -- page = 1, + pages = pdfdoc.nofpages, + width = bbox[3] - bbox[1], + height = bbox[4] - bbox[2], + depth = 0, + colordepth = 0, + xres = 0, + yres = 0, + xsize = 0, + ysize = 0, + rotation = 0, + orientation = 0, + } end - local info = querypdf(pdfdoc,request.page) - local bbox = info.boundingbox - return { - filename = filename, - -- page = 1, - pages = pdfdoc.nofpages, - width = bbox[3] - bbox[1], - height = bbox[4] - bbox[2], - depth = 0, - colordepth = 0, - xres = 0, - yres = 0, - xsize = 0, - ysize = 0, - rotation = 0, - orientation = 0, - } end request.copyimage = function(t) - return copypage(used.pdfdoc,request.page) + if pdfdoc then + -- local pdfdoc = used.pdfdoc + local result = copypage(pdfdoc,request.page) + if pdfdoc.nofpages == 1 then -- and object usage + closepdf(pdfdoc) + -- used.pdfdoc = nil + pdfdoc = nil + end + return result + end end end return checkers.generic(data) diff --git a/tex/context/base/mkiv/grph-inc.mkiv b/tex/context/base/mkiv/grph-inc.mkiv index 677883fbb..2bcac8539 100644 --- a/tex/context/base/mkiv/grph-inc.mkiv +++ b/tex/context/base/mkiv/grph-inc.mkiv @@ -105,6 +105,8 @@ \c!align =\v!none, % New, for Tacos extremely large graphics. \c!crossreference =\v!no, \c!transform =\v!auto, + \c!userpassword =, + \c!ownerpassword =, ] %D Defining figures. @@ -317,27 +319,29 @@ % \dostarttagged\t!image\empty \clf_figure_push { - name {\p_grph_include_name}% - label {\ifx\p_label\empty\p_grph_include_label\else\p_label\fi}% - page {\externalfigureparameter\c!page}% - file {\externalfigureparameter\c!file}% - size {\externalfigureparameter\c!size}% - object {\externalfigureparameter\c!object}% - prefix {\externalfigureparameter\c!prefix}% - cache {\externalfigureparameter\c!cache}% - format {\externalfigureparameter\c!method}% - preset {\externalfigureparameter\c!prefix}% - controls {\externalfigureparameter\c!controls}% - resources {\externalfigureparameter\c!resources}% - preview {\externalfigureparameter\c!preview}% - display {\externalfigureparameter\c!display}% - mask {\externalfigureparameter\c!mask}% - conversion {\externalfigureparameter\c!conversion}% - resolution {\externalfigureparameter\c!resolution}% - color {\externalfigureparameter\c!color}% unprocessed raw key - arguments {\externalfigureparameter\c!arguments}% used for converters - repeat {\externalfigureparameter\c!repeat}% - transform {\externalfigureparameter\c!transform}% + name {\p_grph_include_name}% + label {\ifx\p_label\empty\p_grph_include_label\else\p_label\fi}% + page {\externalfigureparameter\c!page}% + file {\externalfigureparameter\c!file}% + size {\externalfigureparameter\c!size}% + object {\externalfigureparameter\c!object}% + prefix {\externalfigureparameter\c!prefix}% + cache {\externalfigureparameter\c!cache}% + format {\externalfigureparameter\c!method}% + preset {\externalfigureparameter\c!prefix}% + controls {\externalfigureparameter\c!controls}% + resources {\externalfigureparameter\c!resources}% + preview {\externalfigureparameter\c!preview}% + display {\externalfigureparameter\c!display}% + mask {\externalfigureparameter\c!mask}% + conversion {\externalfigureparameter\c!conversion}% + resolution {\externalfigureparameter\c!resolution}% + color {\externalfigureparameter\c!color}% unprocessed raw key + arguments {\externalfigureparameter\c!arguments}% used for converters + repeat {\externalfigureparameter\c!repeat}% + transform {\externalfigureparameter\c!transform}% + userpassword {\externalfigureparameter\c!userpassword}% + ownerpassword{\externalfigureparameter\c!ownerpassword}% \ifx\p_width\empty \else width \dimexpr\p_width\relax \fi diff --git a/tex/context/base/mkiv/l-lpeg.lua b/tex/context/base/mkiv/l-lpeg.lua index 5101a2e8d..45d254724 100644 --- a/tex/context/base/mkiv/l-lpeg.lua +++ b/tex/context/base/mkiv/l-lpeg.lua @@ -270,7 +270,7 @@ patterns.cpfloat = sign^-1 * patterns.cpunsigned patterns.number = patterns.float + patterns.integer patterns.cnumber = patterns.cfloat + patterns.integer patterns.cpnumber = patterns.cpfloat + patterns.integer -patterns.oct = zero * octdigits +patterns.oct = zero * octdigits -- hm is this ok patterns.octal = patterns.oct patterns.HEX = zero * P("X") * (digit+uppercase)^1 patterns.hex = zero * P("x") * (digit+lowercase)^1 diff --git a/tex/context/base/mkiv/lpdf-ano.lua b/tex/context/base/mkiv/lpdf-ano.lua index 9fdb0913c..fdbd55d55 100644 --- a/tex/context/base/mkiv/lpdf-ano.lua +++ b/tex/context/base/mkiv/lpdf-ano.lua @@ -683,7 +683,7 @@ local function pdfaction(actions) return nil end end - return first, actions.n + return first, actions.n or #actions end end end diff --git a/tex/context/base/mkiv/lpdf-aux.lua b/tex/context/base/mkiv/lpdf-aux.lua new file mode 100644 index 000000000..0d7cecbb8 --- /dev/null +++ b/tex/context/base/mkiv/lpdf-aux.lua @@ -0,0 +1,152 @@ +if not modules then modules = { } end modules ['lpdf-aux'] = { + version = 1.001, + comment = "companion to lpdf-ini.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local tonumber = tonumber +local format, concat = string.format, table.concat +local utfchar, utfbyte, char = utf.char, utf.byte, string.char +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns +local P, C, R, S, Cc, Cs, V = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cc, lpeg.Cs, lpeg.V +local rshift = bit32.rshift + +lpdf = lpdf or { } + +-- tosixteen -- + +local cache = table.setmetatableindex(function(t,k) -- can be made weak + local v = utfbyte(k) + if v < 0x10000 then + v = format("%04x",v) + else + v = format("%04x%04x",rshift(v,10),v%1024+0xDC00) + end + t[k] = v + return v +end) + +local unified = Cs(Cc("<feff") * (lpegpatterns.utf8character/cache)^1 * Cc(">")) + +function lpdf.tosixteen(str) -- an lpeg might be faster (no table) + if not str or str == "" then + return "<feff>" -- not () as we want an indication that it's unicode + else + return lpegmatch(unified,str) + end +end + +-- fromsixteen -- + +-- local zero = S(" \n\r\t") + P("\\ ") +-- local one = C(4) +-- local two = P("d") * R("89","af") * C(2) * C(4) +-- +-- local pattern = P { "start", +-- start = V("wrapped") + V("unwrapped") + V("original"), +-- original = Cs(P(1)^0), +-- wrapped = P("<") * V("unwrapped") * P(">") * P(-1), +-- unwrapped = P("feff") +-- * Cs( ( +-- zero / "" +-- + two / function(a,b) +-- a = (tonumber(a,16) - 0xD800) * 1024 +-- b = (tonumber(b,16) - 0xDC00) +-- return utfchar(a+b) +-- end +-- + one / function(a) +-- return utfchar(tonumber(a,16)) +-- end +-- )^1 ) * P(-1) +-- } +-- +-- function lpdf.fromsixteen(s) +-- return lpegmatch(pattern,s) or s +-- end + +local more = 0 + +local pattern = C(4) / function(s) -- needs checking ! + local now = tonumber(s,16) + if more > 0 then + now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong + more = 0 + return utfchar(now) + elseif now >= 0xD800 and now <= 0xDBFF then + more = now + return "" -- else the c's end up in the stream + else + return utfchar(now) + end +end + +local pattern = P(true) / function() more = 0 end * Cs(pattern^0) + +function lpdf.fromsixteen(str) + if not str or str == "" then + return "" + else + return lpegmatch(pattern,str) + end +end + +-- frombytes -- + +local b_pattern = Cs((P("\\")/"" * ( + S("()") + + S("nrtbf")/ { n = "\n", r = "\r", t = "\t", b = "\b", f = "\f" } + + lpegpatterns.octdigit^-3 / function(s) return char(tonumber(s,8)) end) ++ P(1))^0) + +local u_pattern = lpegpatterns.utfbom_16_be * lpegpatterns.utf16_to_utf8_be -- official + + lpegpatterns.utfbom_16_le * lpegpatterns.utf16_to_utf8_le -- we've seen these + +local h_pattern = lpegpatterns.hextobytes + +local zero = S(" \n\r\t") + P("\\ ") +local one = C(4) +local two = P("d") * R("89","af") * C(2) * C(4) + +local x_pattern = P { "start", + start = V("wrapped") + V("unwrapped") + V("original"), + original = Cs(P(1)^0), + wrapped = P("<") * V("unwrapped") * P(">") * P(-1), + unwrapped = P("feff") + * Cs( ( + zero / "" + + two / function(a,b) + a = (tonumber(a,16) - 0xD800) * 1024 + b = (tonumber(b,16) - 0xDC00) + return utfchar(a+b) + end + + one / function(a) + return utfchar(tonumber(a,16)) + end + )^1 ) * P(-1) +} + +function lpdf.frombytes(s,hex) + if not s or s == "" then + return "" + end + if hex then + local x = lpegmatch(x_pattern,s) + if x then + return x + end + local h = lpegmatch(h_pattern,s) + if h then + return h + end + else + local u = lpegmatch(u_pattern,s) + if u then + return u + end + end + return lpegmatch(b_pattern,s) +end + +-- done -- diff --git a/tex/context/base/mkiv/lpdf-epa.lua b/tex/context/base/mkiv/lpdf-epa.lua index ee3b6d43d..0792db646 100644 --- a/tex/context/base/mkiv/lpdf-epa.lua +++ b/tex/context/base/mkiv/lpdf-epa.lua @@ -281,10 +281,10 @@ function codeinjections.mergereferences(specification) local annotations = pagedata and pagedata.Annots local namespace = makenamespace(fullname) local reference = namespace .. pagenumber - if annotations and annotations.n > 0 then + if annotations and #annotations > 0 then local llx, lly, urx, ury, width, height, xscale, yscale = getmediasize(specification,pagedata,xscale,yscale) initializelayer(height,width) - for i=1,annotations.n do + for i=1,#annotations do local annotation = annotations[i] if annotation then if annotation.Subtype == "Link" then @@ -333,7 +333,7 @@ function codeinjections.mergeviewerlayers(specification) local namespace = makenamespace(fullname) local layers = document.layers if layers then - for i=1,layers.n do + for i=1,#layers do local layer = layers[i] if layer then local tag = namespace .. gsub(layer," ",":") @@ -538,14 +538,14 @@ function codeinjections.mergecomments(specification) -- local pagenumber = specification.page or 1 -- local pagedata = document.pages[pagenumber] -- local annotations = pagedata and pagedata.Annots - -- if annotations and annotations.n > 0 then + -- if annotations and #annotations > 0 then -- local llx, lly, urx, ury, width, height, xscale, yscale = getmediasize(specification,pagedata,xscale,yscale) -- initializelayer(height,width) -- -- -- local lockflags = specification.lock -- todo: proper parameter -- local references = { } -- local usedpopups = { } - -- for i=1,annotations.n do + -- for i=1,#annotations do -- local annotation = annotations[i] -- if annotation then -- local subtype = annotation.Subtype @@ -559,7 +559,7 @@ function codeinjections.mergecomments(specification) -- end -- end -- -- - -- for i=1,annotations.n do + -- for i=1,#annotations do -- -- we keep the order -- local annotation = annotations[i] -- if annotation then @@ -722,11 +722,11 @@ function codeinjections.mergefields(specification) -- local pagenumber = specification.page or 1 -- local pagedata = document.pages[pagenumber] -- local annotations = pagedata and pagedata.Annots - -- if annotations and annotations.n > 0 then + -- if annotations and #annotations > 0 then -- local llx, lly, urx, ury, width, height, xscale, yscale = getmediasize(specification,pagedata,xscale,yscale) -- initializelayer(height,width) -- -- - -- for i=1,annotations.n do + -- for i=1,#annotations do -- -- we keep the order -- local annotation = annotations[i] -- if annotation then @@ -870,7 +870,7 @@ function codeinjections.getbookmarks(filename) local outlines = document.Catalog.Outlines local pages = document.pages - local nofpages = pages.n -- we need to access once in order to initialize + local nofpages = document.nofpages local destinations = document.destinations -- I need to check this destination analyzer with the one in annotations .. best share @@ -892,7 +892,7 @@ function codeinjections.getbookmarks(filename) entry.realpage = pagedata.number end elseif kind == "table" then - local pageref = destination.n + local pageref = #destination if pageref then local pagedata = pages[pageref] if pagedata then @@ -1024,7 +1024,7 @@ function codeinjections.getinfo(specification) local catalog = pdffile.Catalog local info = pdffile.Info local pages = pdffile.pages - local nofpages = pages.n + local nofpages = pdffile.nofpages if metadata then local m = catalog.Metadata if m then @@ -1048,10 +1048,8 @@ function codeinjections.getinfo(specification) local media = nobox local page = pages[pagenumber] if page then - crop = page.CropBox or nobox - media = page.MediaBox or crop or nobox - crop.n = nil -- nicer - media.n = nil -- nicer + crop = page.CropBox or nobox + media = page.MediaBox or crop or nobox end local bbox = crop or media or nobox return { diff --git a/tex/context/base/mkiv/lpdf-epd.lua b/tex/context/base/mkiv/lpdf-epd.lua index 7bfdf57bc..f6e28cdec 100644 --- a/tex/context/base/mkiv/lpdf-epd.lua +++ b/tex/context/base/mkiv/lpdf-epd.lua @@ -149,21 +149,21 @@ end local report_epdf = logs.reporter("epdf") local typenames = { [0] = - "boolean", - "integer", - "real", - "string", - "name", - "null", - "array", - "dictionary", - "stream", - "ref", - "cmd", - "error", - "eof", - "none", - "integer64", + "boolean", + "integer", + "real", + "string", + "name", + "null", + "array", + "dictionary", + "stream", + "ref", + "cmd", + "error", + "eof", + "none", + "integer64", } local typenumbers = table.swapped(typenames) diff --git a/tex/context/base/mkiv/lpdf-ini.lua b/tex/context/base/mkiv/lpdf-ini.lua index abe597810..d3550a756 100644 --- a/tex/context/base/mkiv/lpdf-ini.lua +++ b/tex/context/base/mkiv/lpdf-ini.lua @@ -13,7 +13,8 @@ local char, byte, format, gsub, concat, match, sub, gmatch = string.char, string local utfchar, utfbyte, utfvalues = utf.char, utf.byte, utf.values local sind, cosd, max, min = math.sind, math.cosd, math.max, math.min local sort = table.sort -local lpegmatch, P, C, R, S, Cc, Cs = lpeg.match, lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cc, lpeg.Cs +local P, C, R, S, Cc, Cs, V = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cc, lpeg.Cs, lpeg.V +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns local formatters = string.formatters local isboolean = string.is_boolean local rshift = bit32.rshift @@ -276,7 +277,7 @@ local cache = table.setmetatableindex(function(t,k) -- can be made weak return v end) -local escaped = Cs(Cc("(") * (S("\\()")/"\\%0" + P(1))^0 * Cc(")")) +local escaped = Cs(Cc("(") * (S("\\()\n\r\t\b\f")/"\\%0" + P(1))^0 * Cc(")")) local unified = Cs(Cc("<feff") * (lpeg.patterns.utf8character/cache)^1 * Cc(">")) local function tosixteen(str) -- an lpeg might be faster (no table) @@ -343,12 +344,80 @@ local function toeight(str) end end +local b_pattern = Cs((P("\\")/"" * ( + S("()") + + S("nrtbf") / { n = "\n", r = "\r", t = "\t", b = "\b", f = "\f" } + + lpegpatterns.octdigit^-3 / function(s) return char(tonumber(s,8)) end) ++ P(1))^0) + +local function fromeight(str) + if not str or str == "" then + return "" + else + return lpegmatch(unescape,str) + end +end + lpdf.tosixteen = tosixteen lpdf.toeight = toeight lpdf.topdfdoc = topdfdoc lpdf.fromsixteen = fromsixteen +lpdf.fromeight = fromeight lpdf.frompdfdoc = frompdfdoc +do + + local u_pattern = lpegpatterns.utfbom_16_be * lpegpatterns.utf16_to_utf8_be -- official + + lpegpatterns.utfbom_16_le * lpegpatterns.utf16_to_utf8_le -- we've seen these + + local h_pattern = lpegpatterns.hextobytes + + local zero = S(" \n\r\t") + P("\\ ") + local one = C(4) + local two = P("d") * R("89","af") * C(2) * C(4) + + local x_pattern = P { "start", + start = V("wrapped") + V("unwrapped") + V("original"), + original = Cs(P(1)^0), + wrapped = P("<") * V("unwrapped") * P(">") * P(-1), + unwrapped = P("feff") + * Cs( ( + zero / "" + + two / function(a,b) + a = (tonumber(a,16) - 0xD800) * 1024 + b = (tonumber(b,16) - 0xDC00) + return utfchar(a+b) + end + + one / function(a) + return utfchar(tonumber(a,16)) + end + )^1 ) * P(-1) + } + + function lpdf.frombytes(s,hex) + if not s or s == "" then + return "" + end + if hex then + local x = lpegmatch(x_pattern,s) + if x then + return x + end + local h = lpegmatch(h_pattern,s) + if h then + return h + end + else + local u = lpegmatch(u_pattern,s) + if u then + return u + end + end + return lpegmatch(b_pattern,s) + end + +end + local function merge_t(a,b) local t = { } for k,v in next, a do t[k] = v end @@ -362,10 +431,8 @@ local f_key_dictionary = formatters["/%s << % t >>"] local f_dictionary = formatters["<< % t >>"] local f_key_array = formatters["/%s [ % t ]"] local f_array = formatters["[ % t ]"] ------ f_key_number = formatters["/%s %F"] -local f_key_number = formatters["/%s %n"] ------ f_tonumber = formatters["%F"] -local f_tonumber = formatters["%n"] +local f_key_number = formatters["/%s %N"] +local f_tonumber = formatters["%N"] local tostring_a, tostring_d @@ -385,8 +452,6 @@ tostring_d = function(t,contentonly,key) r[i] = f_key_value(k,toeight(v)) elseif tv == "number" then r[i] = f_key_number(k,v) - -- elseif tv == "unicode" then -- can't happen - -- r[i] = f_key_value(k,tosixteen(v)) elseif tv == "table" then local mv = getmetatable(v) if mv and mv.__lpdftype then @@ -430,8 +495,6 @@ tostring_a = function(t,contentonly,key) r[k] = toeight(v) elseif tv == "number" then r[k] = f_tonumber(v) - -- elseif tv == "unicode" then - -- r[k] = tosixteen(v) elseif tv == "table" then local mv = getmetatable(v) local mt = mv and mv.__lpdftype @@ -486,6 +549,17 @@ local tostring_v = function(t) end end +local tostring_l = function(t) + local s = t[1] + if not s or s == "" then + return "()" + elseif t[2] then + return "<" .. s .. ">" + else + return "(" .. s .. ")" + end +end + local function value_x(t) return t end local function value_s(t) return t[1] end local function value_p(t) return t[1] end @@ -497,8 +571,9 @@ local function value_a(t) return tostring_a(t,true) end local function value_z() return nil end local function value_t(t) return t.value or true end local function value_f(t) return t.value or false end -local function value_r() return t[1] or 0 end -- null -local function value_v() return t[1] end +local function value_r(t) return t[1] or 0 end -- null +local function value_v(t) return t[1] end +local function value_l(t) return t[1] end local function add_x(t,k,v) rawset(t,k,tostring(v)) end @@ -515,6 +590,7 @@ local mt_t = { __lpdftype = "true", __tostring = tostring_t, __call = valu local mt_f = { __lpdftype = "false", __tostring = tostring_f, __call = value_f } local mt_r = { __lpdftype = "reference", __tostring = tostring_r, __call = value_r } local mt_v = { __lpdftype = "verbose", __tostring = tostring_v, __call = value_v } +local mt_l = { __lpdftype = "literal", __tostring = tostring_l, __call = value_l } local function pdfstream(t) -- we need to add attributes if t then @@ -556,6 +632,10 @@ local function pdfunicode(str,default) return setmetatable({ str or default or "" },mt_u) -- could be a string end +local function pdfliteral(str,hex) -- can also produce a hex <> instead of () literal + return setmetatable({ str, hex },mt_l) +end + local cache = { } -- can be weak local function pdfnumber(n,default) -- 0-10 @@ -574,13 +654,26 @@ for i=-1,9 do cache[i] = pdfnumber(i) end local cache = { } -- can be weak -local forbidden, replacements = "\0\t\n\r\f ()[]{}/%%#\\", { } -- table faster than function - -for s in gmatch(forbidden,".") do - replacements[s] = format("#%02x",byte(s)) -end - -local escaped = Cs(Cc("/") * (S(forbidden)/replacements + P(1))^0) +local replacer = S("\0\t\n\r\f ()[]{}/%%#\\") / { + ["\00"]="#00", + ["\09"]="#09", + ["\10"]="#0a", + ["\12"]="#0c", + ["\13"]="#0d", + [ " " ]="#20", + [ "#" ]="#23", + [ "%" ]="#25", + [ "(" ]="#28", + [ ")" ]="#29", + [ "/" ]="#2f", + [ "[" ]="#5b", + [ "\\"]="#5c", + [ "]" ]="#5d", + [ "{" ]="#7b", + [ "}" ]="#7d", +} + P(1) + +local escaped = Cs(Cc("/") * replacer^0) local function pdfconstant(str,default) if not str then @@ -588,15 +681,13 @@ local function pdfconstant(str,default) end local c = cache[str] if not c then - -- c = setmetatable({ "/" .. str },mt_c) c = setmetatable({ lpegmatch(escaped,str) },mt_c) cache[str] = c end return c end -local escaped = Cs((S(forbidden)/replacements + P(1))^0) ------ escaped = Cs((1-forbidden)^0 * S(forbidden)/replacements * ((S(forbidden)/replacements + P(1))^0) +local escaped = Cs(replacer^0) function lpdf.escaped(str) return lpegmatch(escaped,str) or str @@ -663,6 +754,7 @@ lpdf.null = pdfnull lpdf.boolean = pdfboolean lpdf.reference = pdfreference lpdf.verbose = pdfverbose +lpdf.literal = pdfliteral local names, cache = { }, { } @@ -739,15 +831,22 @@ function lpdf.flushobject(name,data) end end - function lpdf.flushstreamobject(data,dict,compressed,objnum) -- default compressed if trace_objects then report_objects("flushing stream object of %s bytes",#data) end - local dtype = type(dict) + local dtype = type(dict) + local kind = compressed == "raw" and "raw" or "stream" + local nolength = nil + if compressed == "raw" then + compressed = nil + nolength = true + -- data = string.formatters["<< %s >>stream\n%s\nendstream"](attr,data) + end return pdfdeferredobject { objnum = objnum, immediate = true, + nolength = nolength, compresslevel = compressed == false and 0 or nil, type = "stream", string = data, @@ -1251,7 +1350,6 @@ end -- end, -- }) - -- The next variant of ActualText is what Taco and I could come up with -- eventually. As of September 2013 Acrobat copies okay, Sumatra copies a -- question mark, pdftotext injects an extra space and Okular adds a @@ -1370,11 +1468,9 @@ end function lpdf.copyarray(a) if a then local t = pdfarray() - local k = a.__kind for i=1,#a do t[i] = a(i) end --- inspect(t) return t end end @@ -1385,7 +1481,6 @@ function lpdf.copydictionary(d) for k, v in next, d do t[k] = d(k) end --- inspect(t) return t end end diff --git a/tex/context/base/mkiv/lpdf-pde.lua b/tex/context/base/mkiv/lpdf-pde.lua new file mode 100644 index 000000000..3f12edd90 --- /dev/null +++ b/tex/context/base/mkiv/lpdf-pde.lua @@ -0,0 +1,1005 @@ +if not modules then modules = { } end modules ['lpdf-epd'] = { + version = 1.001, + comment = "companion to lpdf-epa.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files", + history = "this one replaces the poppler/pdfe binding", +} + +-- maximum integer : +2^32 +-- maximum real : +2^15 +-- minimum real : 1/(2^16) + +-- get_flagged : does that still work + +-- ppdoc_permissions (ppdoc *pdf); + +-- PPSTRING_ENCODED 1 << 0 +-- PPSTRING_DECODED 1 << 1 +-- PPSTRING_EXEC 1 << 2 postscript only +-- PPSTRING_PLAIN 0 +-- PPSTRING_BASE16 1 << 3 +-- PPSTRING_BASE85 1 << 4 +-- PPSTRING_UTF16BE 1 << 5 +-- PPSTRING_UTF16LE 1 << 6 + +-- PPDOC_ALLOW_PRINT 1 << 2 printing +-- PPDOC_ALLOW_MODIFY 1 << 3 filling form fields, signing, creating template pages +-- PPDOC_ALLOW_COPY 1 << 4 copying, copying for accessibility +-- PPDOC_ALLOW_ANNOTS 1 << 5 filling form fields, copying, signing +-- PPDOC_ALLOW_EXTRACT 1 << 9 contents copying for accessibility +-- PPDOC_ALLOW_ASSEMBLY 1 << 10 no effect +-- PPDOC_ALLOW_PRINT_HIRES 1 << 11 no effect + +-- PPCRYPT_NONE 0 no encryption, go ahead +-- PPCRYPT_DONE 1 encryption present but password succeeded, go ahead +-- PPCRYPT_PASS -1 encryption present, need non-empty password +-- PPCRYPT_FAIL -2 invalid or unsupported encryption (eg. undocumented in pdf spec) + +local setmetatable, rawset, rawget, type, next = setmetatable, rawset, rawget, type, next +local tostring, tonumber, unpack = tostring, tonumber, unpack +local char, byte, find = string.char, string.byte, string.find +local abs = math.abs +local concat, swapped = table.concat, table.swapped +local utfchar = string.char +local setmetatableindex = table.setmetatableindex + +local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns +local P, C, S, R, Ct, Cc, V, Carg, Cs, Cf, Cg = lpeg.P, lpeg.C, lpeg.S, lpeg.R, lpeg.Ct, lpeg.Cc, lpeg.V, lpeg.Carg, lpeg.Cs, lpeg.Cf, lpeg.Cg + +if not lpdf then require("lpdf-aux") end + +local epdf = pdfe + lpdf = lpdf or { } +local lpdf = lpdf +local lpdf_epdf = { } +lpdf.epdf = lpdf_epdf + +local openPDF = epdf.open +local closePDF = epdf.close + +local getcatalog = epdf.getcatalog +local getinfo = epdf.getinfo +local gettrailer = epdf.gettrailer +local getnofpages = epdf.getnofpages +local getversion = epdf.getversion +local getbox = epdf.getbox +local getstatus = epdf.getstatus +local unencrypt = epdf.unencrypt + +local dictionarytotable = epdf.dictionarytotable +local arraytotable = epdf.arraytotable +local pagestotable = epdf.pagestotable +local readwholestream = epdf.readwholestream + +local getfromreference = pdfe.getfromreference + +local report_epdf = logs.reporter("epdf") + +local allocate = utilities.storage.allocate + +local objectcodes = { + [0] = "none", + "null", + "bool", + "integer", + "number", + "name", + "string", + "array", + "dictionary", + "stream", + "reference", +} + +local encryptioncodes = { + [0] = "notencrypted", + [1] = "unencrypted", + [-1] = "protected", + [-2] = "failure", +} + +objectcodes = allocate(swapped(objectcodes,objectcodes)) +encryptioncodes = allocate(swapped(encryptioncodes,encryptioncodes)) + +pdfe.objectcodes = objectcodes +pdfe.encryptioncodes = encryptioncodes + +local null_code = objectcodes.null +local reference_code = objectcodes.reference + +local none_code = objectcodes.none +local null_code = objectcodes.null +local bool_code = objectcodes.bool +local integer_code = objectcodes.integer +local number_code = objectcodes.number +local name_code = objectcodes.name +local string_code = objectcodes.string +local array_code = objectcodes.array +local dictionary_code = objectcodes.dictionary +local stream_code = objectcodes.stream +local reference_code = objectcodes.reference + +local checked_access +local get_flagged -- from pdfe -> lpdf + +if lpdf.dictionary then + + -- we're in context + + local pdfdictionary = lpdf.dictionary + local pdfarray = lpdf.array + local pdfconstant = lpdf.constant + local pdfstring = lpdf.string + local pdfunicode = lpdf.unicode + + get_flagged = function(t,f,k) + local tk = t[k] -- triggers resolve + local fk = f[k] + if not fk then + return tk + elseif fk == "name" then + return pdfconstant(tk) + elseif fk == "array" then + return pdfarray(tk) + elseif fk == "dictionary" then + return pdfarray(tk) + elseif fk == "rawtext" then + return pdfstring(tk) + elseif fk == "unicode" then + return pdfunicode(tk) + else + return tk + end + end + +else + + get_flagged = function(t,f,k) + return t[k] + end + +end + +-- We need to convert the string from utf16 although there is no way to +-- check if we have a regular string starting with a bom. So, we have +-- na dilemma here: a pdf doc encoded string can be invalid utf. + +-- <hex encoded> : implicit 0 appended if odd +-- (byte encoded) : \( \) \\ escaped +-- +-- <FE><FF> : utf16be +-- +-- \r \r \t \b \f \( \) \\ \NNN and \<newline> : append next line +-- +-- the getString function gives back bytes so we don't need to worry about +-- the hex aspect. + +local some_dictionary +local some_array +local some_stream +local some_reference + +local some_string = lpdf.frombytes + +local function get_value(document,t,key) + if not key then + return + end + local value = t[key] + if not value then + return + end + if type(value) ~= "table" then + return value + end + -- we can assume names to be simple and strings to be tables + local kind = value[1] + if kind == name_code then + return value[2] + elseif kind == string_code then + return some_string(value[2],value[3]) + elseif kind == array_code then + return some_array(value[2],document) + elseif kind == dictionary_code then + return some_dictionary(value[2],document) + elseif kind == stream_code then + return some_stream(value,document) + elseif kind == reference_code then + return some_reference(value,document) + end + return value +end + +some_dictionary = function (d,document) + local f = dictionarytotable(d,true) + local t = setmetatable({ __raw__ = f, __type__ = dictionary_code }, { + __index = function(t,k) + return get_value(document,f,k) + end, + __call = function(t,k) + return get_flagged(t,f,k) + end, + } ) + return t, "dictionary" +end + +some_array = function (a,document) + local f = arraytotable(a,true) + local n = #f + local t = setmetatable({ __raw__ = f, __type__ = array_code, n = n }, { + __index = function(t,k) + return get_value(document,f,k) + end, + __call = function(t,k) + return get_flagged(t,f,k) + end, + __len = function(t,k) + return n + end, + } ) + return t, "array" +end + +some_stream = function(s,d,document) + local f = dictionarytotable(d,true) + local t = setmetatable({ __raw__ = f, __type__ = stream_code }, { + __index = function(t,k) + return get_value(document,f,k) + end, + __call = function(t,raw) + if raw == false then + return readwholestream(s,false) -- original + else + return readwholestream(s,true) -- uncompressed + end + end, + } ) + return t, "stream" +end + +some_reference = function(r,document) + local objnum = r[3] + local cached = document.__cache__[objnum] + if not cached then + local kind, object, b, c = getfromreference(r[2]) + if kind == dictionary_code then + cached = some_dictionary(object,document) + elseif kind == array_code then + cached = some_array(object,document) + elseif kind == stream_code then + cached = some_stream(object,b,document) + else + cached = { kind, object, b, c } + -- really cache this? + end + document.__cache__[objnum] = cached + document.__xrefs__[cached] = objnum + end + return cached +end + +local resolvers = { } +lpdf_epdf.resolvers = resolvers + +local function resolve(document,k) + local resolver = resolvers[k] + if resolver then + local entry = resolver(document) + document[k] = entry + return entry + end +end + +local function getnames(document,n,target) -- direct + if n then + local Names = n.Names + if Names then + if not target then + target = { } + end + for i=1,#Names,2 do + target[Names[i]] = Names[i+1] + end + else + local Kids = n.Kids + if Kids then + for i=1,#Kids do + target = getnames(document,Kids[i],target) + end + end + end + return target + end +end + +local function getkids(document,n,target) -- direct + if n then + local Kids = n.Kids + if Kids then + for i=1,#Kids do + target = getkids(document,Kids[i],target) + end + elseif target then + target[#target+1] = n + else + target = { n } + end + return target + end +end + +function resolvers.destinations(document) + local Names = document.Catalog.Names + return getnames(document,Names and Names.Dests) +end + +function resolvers.javascripts(document) + local Names = document.Catalog.Names + return getnames(document,Names and Names.JS) +end + +function resolvers.widgets(document) + local Names = document.Catalog.Names + return getnames(document,Names and Names.AcroForm) +end + +function resolvers.embeddedfiles(document) + local Names = document.Catalog.Names + return getnames(document,Names and Names.EmbeddedFiles) +end + +-- /OCProperties << +-- /OCGs [ 15 0 R 17 0 R 19 0 R 21 0 R 23 0 R 25 0 R 27 0 R ] +-- /D << +-- /Order [ 15 0 R 17 0 R 19 0 R 21 0 R 23 0 R 25 0 R 27 0 R ] +-- /ON [ 15 0 R 17 0 R 19 0 R 21 0 R 23 0 R 25 0 R 27 0 R ] +-- /OFF [ ] +-- >> +-- >> + +function resolvers.layers(document) + local properties = document.Catalog.OCProperties + if properties then + local layers = properties.OCGs + if layers then + local t = { } + for i=1,#layers do + local layer = layers[i] + t[i] = layer.Name + end + -- t.n = n + return t + end + end +end + +function resolvers.structure(document) + -- this might become a tree + return document.Catalog.StructTreeRoot +end + +function resolvers.pages(document) + local __data__ = document.__data__ + local __xrefs__ = document.__xrefs__ + local __cache__ = document.__cache__ + -- + local nofpages = document.nofpages + local pages = { } + local rawpages = pagestotable(__data__) + document.pages = pages + -- + for pagenumber=1,nofpages do + local rawpagedata = rawpages[pagenumber] + local pagereference = rawpagedata[3] + local pageobject = rawpagedata[1] + local pagedata = some_dictionary(pageobject,document) + if pagedata and pageobject then + pagedata.number = pagenumber + pagedata.MediaBox = getbox(pageobject,"MediaBox") + pagedata.CropBox = getbox(pageobject,"CropBox") + pagedata.BleedBox = getbox(pageobject,"BleedBox") + pagedata.ArtBox = getbox(pageobject,"ArtBox") + pagedata.TrimBox = getbox(pageobject,"TrimBox") + pages[pagenumber] = pagedata + __xrefs__[pagedata] = pagereference + __cache__[pagereference] = pagedata + else + report_epdf("missing pagedata for page %i",i) + end + end + -- + -- pages.n = nofpages + -- + return pages +end + +local loaded = { } +local nofloaded = 0 + +function lpdf_epdf.load(filename,userpassword,ownerpassword) + local document = loaded[filename] + if not document then + statistics.starttiming(lpdf_epdf) + local __data__ = openPDF(filename) -- maybe resolvers.find_file + if __data__ then +-- nofloaded = nofloaded + 1 +-- report_epdf("%04i opened: %s",nofloaded,filename) + if userpassword and getstatus(__data__) < 0 then + unencrypt(__data__,userpassword,nil) + end + if ownerpassword and getstatus(__data__) < 0 then + unencrypt(__data__,nil,ownerpassword) + end + if getstatus(__data__) < 0 then + report_epdf("the document is encrypted, provide proper passwords",getstatus(__data__)) + end + if __data__ then + document = { + filename = filename, + __cache__ = { }, + __xrefs__ = { }, + __fonts__ = { }, + __copied__ = { }, + __data__ = __data__, + } + document.Catalog = some_dictionary(getcatalog(__data__),document) + document.Info = some_dictionary(getinfo(__data__),document) + document.Trailer = some_dictionary(gettrailer(__data__),document) + -- + setmetatableindex(document,resolve) + -- + document.majorversion, document.minorversion = getversion(__data__) + -- + document.nofpages = getnofpages(__data__) + else + document = false + end + else + document = false + end + loaded[filename] = document + loaded[document] = document + statistics.stoptiming(lpdf_epdf) + -- print(statistics.elapsedtime(lpdf_epdf)) + end + return document or nil +end + +function lpdf_epdf.unload(filename) + if type(filename) == "table" then + filename = filename.filename + end + if type(filename) == "string" then + local document = loaded[filename] + if document then +-- report_epdf("%04i closed: %s",nofloaded,filename) +-- nofloaded = nofloaded - 1 + loaded[document] = nil + loaded[filename] = nil + end + end +end + +-- for k, v in expanded(t) do + +local function expanded(t) + local function iterator(raw,k) + local k, v = next(raw,k) + if v then + return k, t[k] + end + end + return iterator, t.__raw__, nil +end + +---------.expand = expand +lpdf_epdf.expanded = expanded + +-- we could resolve the text stream in one pass if we directly handle the +-- font but why should we complicate things + +local hexdigit = R("09","AF") +local numchar = ( P("\\") * ( (R("09")^3/tonumber) + C(1) ) ) + C(1) +local number = lpegpatterns.number / tonumber +local spaces = lpegpatterns.whitespace^1 +local optspaces = lpegpatterns.whitespace^0 +local keyword = P("/") * C(R("AZ","az","09")^1) +local operator = C((R("AZ","az")+P("'")+P('"'))^1) + +local grammar = P { "start", + start = (keyword + number + V("dictionary") + V("unicode") + V("string") + V("unicode")+ V("array") + spaces)^1, + -- keyvalue = (keyword * spaces * V("start") + spaces)^1, + keyvalue = optspaces * Cf(Ct("") * Cg(keyword * optspaces * V("start") * optspaces)^1,rawset), + array = P("[") * Ct(V("start")^1) * P("]"), + dictionary = P("<<") * V("keyvalue") * P(">>"), + unicode = P("<") * Ct(Cc("hex") * C((1-P(">"))^1)) * P(">"), + string = P("(") * Ct(Cc("dec") * C((V("string")+numchar)^1)) * P(")"), -- untested +} + +local operation = Ct(grammar^1 * operator) +local parser = Ct((operation + P(1))^1) + +-- beginbfrange : <start> <stop> <firstcode> +-- <start> <stop> [ <firstsequence> <firstsequence> <firstsequence> ] +-- beginbfchar : <code> <newcodes> + +local fromsixteen = lpdf.fromsixteen -- maybe inline the lpeg ... but not worth it + +local function f_bfchar(t,a,b) + t[tonumber(a,16)] = fromsixteen(b) +end + +local function f_bfrange_1(t,a,b,c) + print("todo 1",a,b,c) + -- c is string + -- todo t[tonumber(a,16)] = fromsixteen(b) +end + +local function f_bfrange_2(t,a,b,c) + print("todo 2",a,b,c) + -- c is table + -- todo t[tonumber(a,16)] = fromsixteen(b) +end + +local optionals = spaces^0 +local hexstring = optionals * P("<") * C((1-P(">"))^1) * P(">") +local bfchar = Carg(1) * hexstring * hexstring / f_bfchar +local bfrange = Carg(1) * hexstring * hexstring * hexstring / f_bfrange_1 + + Carg(1) * hexstring * hexstring * optionals * P("[") * Ct(hexstring^1) * optionals * P("]") / f_bfrange_2 +local fromunicode = ( + P("beginbfchar" ) * bfchar ^1 * optionals * P("endbfchar" ) + + P("beginbfrange") * bfrange^1 * optionals * P("endbfrange") + + spaces + + P(1) +)^1 * Carg(1) + +local function analyzefonts(document,resources) -- unfinished, see mtx-pdf for better code + local fonts = document.__fonts__ + if resources then + local fontlist = resources.Font + if fontlist then + for id, data in expanded(fontlist) do + if not fonts[id] then + -- a quick hack ... I will look into it more detail if I find a real + -- -application for it + local tounicode = data.ToUnicode() + if tounicode then + tounicode = lpegmatch(fromunicode,tounicode,1,{}) + end + fonts[id] = { + tounicode = type(tounicode) == "table" and tounicode or { } + } + setmetatableindex(fonts[id],"self") + end + end + end + end + return fonts +end + +local more = 0 +local unic = nil -- cheaper than passing each time as Carg(1) + +local p_hex_to_utf = C(4) / function(s) -- needs checking ! + local now = tonumber(s,16) + if more > 0 then + now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong + more = 0 + return unic[now] or utfchar(now) + elseif now >= 0xD800 and now <= 0xDBFF then + more = now + -- return "" + else + return unic[now] or utfchar(now) + end +end + +local p_dec_to_utf = C(1) / function(s) -- needs checking ! + local now = byte(s) + return unic[now] or utfchar(now) +end + +local p_hex_to_utf = P(true) / function() more = 0 end * Cs(p_hex_to_utf^1) +local p_dec_to_utf = P(true) / function() more = 0 end * Cs(p_dec_to_utf^1) + +function lpdf_epdf.getpagecontent(document,pagenumber) + + local page = document.pages[pagenumber] + + if not page then + return + end + + local fonts = analyzefonts(document,page.Resources) + + local content = page.Contents() or "" + local list = lpegmatch(parser,content) + local font = nil + -- local unic = nil + + for i=1,#list do + local entry = list[i] + local size = #entry + local operator = entry[size] + if operator == "Tf" then + font = fonts[entry[1]] + unic = font.tounicode + elseif operator == "TJ" then -- { array, TJ } + local list = entry[1] + for i=1,#list do + local li = list[i] + if type(li) == "table" then + if li[1] == "hex" then + list[i] = lpegmatch(p_hex_to_utf,li[2]) + else + list[i] = lpegmatch(p_dec_to_utf,li[2]) + end + else + -- kern + end + end + elseif operator == "Tj" or operator == "'" or operator == '"' then -- { string, Tj } { string, ' } { n, m, string, " } + local list = entry[size-1] + if list[1] == "hex" then + list[2] = lpegmatch(p_hex_to_utf,li[2]) + else + list[2] = lpegmatch(p_dec_to_utf,li[2]) + end + end + end + + unic = nil -- can be collected + + return list + +end + +-- This is also an experiment. When I really need it I can improve it, for instance +-- with proper position calculating. It might be usefull for some search or so. + +local softhyphen = utfchar(0xAD) .. "$" +local linefactor = 1.3 + +function lpdf_epdf.contenttotext(document,list) -- maybe signal fonts + local last_y = 0 + local last_f = 0 + local text = { } + local last = 0 + + for i=1,#list do + local entry = list[i] + local size = #entry + local operator = entry[size] + if operator == "Tf" then + last_f = entry[2] + elseif operator == "TJ" then + local list = entry[1] + for i=1,#list do + local li = list[i] + if type(li) == "string" then + last = last + 1 + text[last] = li + elseif li < -50 then + last = last + 1 + text[last] = " " + end + end + line = concat(list) + elseif operator == "Tj" then + last = last + 1 + text[last] = entry[size-1] + elseif operator == "cm" or operator == "Tm" then + local ty = entry[6] + local dy = abs(last_y - ty) + if dy > linefactor*last_f then + if last > 0 then + if find(text[last],softhyphen,1,true) then + -- ignore + else + last = last + 1 + text[last] = "\n" + end + end + end + last_y = ty + end + end + + return concat(text) +end + +function lpdf_epdf.getstructure(document,list) -- just a test + local depth = 0 + for i=1,#list do + local entry = list[i] + local size = #entry + local operator = entry[size] + if operator == "BDC" then + report_epdf("%w%s : %s",depth,entry[1] or "?",entry[2] and entry[2].MCID or "?") + depth = depth + 1 + elseif operator == "EMC" then + depth = depth - 1 + elseif operator == "TJ" then + local list = entry[1] + for i=1,#list do + local li = list[i] + if type(li) == "string" then + report_epdf("%w > %s",depth,li) + elseif li < -50 then + report_epdf("%w >",depth,li) + end + end + elseif operator == "Tj" then + report_epdf("%w > %s",depth,entry[size-1]) + end + end +end + +if img then do + + -- This can be made a bit faster (just get raw data and pass it) but I will + -- do that later. In the end the benefit is probably neglectable. + + local recompress = true + local recompress = false + + local copydictionary = nil + local copyarray = nil + + local pdfreserveobject = lpdf.reserveobject + local shareobjectreference = lpdf.shareobjectreference + local pdfflushobject = lpdf.flushobject + local pdfflushstreamobject = lpdf.flushstreamobject + local pdfreference = lpdf.reference + local pdfconstant = lpdf.constant + local pdfarray = lpdf.array + local pdfdictionary = lpdf.dictionary + local pdfnull = lpdf.null + local pdfliteral = lpdf.literal + + local report = logs.reporter("backend","xobjects") + + local factor = 65536 / (7200/7227) -- 1/number.dimenfactors.bp + + local newimage = img.new + + local function scaledbbox(b) + return { b[1]*factor, b[2]*factor, b[3]*factor, b[4]*factor } + end + + local function deepcopyobject(xref,copied,value) + -- no need for tables, just nested loop with obj + local objnum = xref[value] + if objnum then + local usednum = copied[objnum] + if usednum then + -- report("%s object %i is reused",kind,objnum) + else + usednum = pdfreserveobject() + copied[objnum] = usednum + local entry = value + local kind = entry.__type__ + if kind == array_code then + local a = copyarray(xref,copied,entry) + pdfflushobject(usednum,tostring(a)) + elseif kind == dictionary_code then + local d = copydictionary(xref,copied,entry) + pdfflushobject(usednum,tostring(d)) + elseif kind == stream_code then + local d = copydictionary(xref,copied,entry) + if recompress then + -- recompress + d.Filter = nil + d.Length = nil + d.DecodeParms = nil + d.DL = nil + local s = entry() -- get uncompressed stream + pdfflushstreamobject(s,d,true,usednum) -- compress stream + else + -- keep as-is, even Length which indicates the + -- decompressed length + local s = entry(false) -- get compressed stream + -- pdfflushstreamobject(s,d,false,usednum,true) -- don't compress stream + pdfflushstreamobject(s,d,"raw",usednum) -- don't compress stream + end + else + local t = type(value) + if t == "string" then + value = pdfconstant(value) + elseif t == "table" then + local kind = value[1] + local entry = value[2] + if kind == name_code then + value = pdfconstant(entry) + elseif kind == string_code then + value = pdfliteral(entry,value[3]) + elseif kind == null_code then + value = pdfnull() + elseif kind == reference_code then + value = deepcopyobject(xref,copied,entry) + else + value = tostring(entry) + end + end + pdfflushobject(usednum,value) + end + end + return pdfreference(usednum) + elseif kind == stream_code then + report("stream not done: %s", objectcodes[kind] or "?") + else + report("object not done: %s", objectcodes[kind] or "?") + end + end + + local function copyobject(xref,copied,object,key,value) + local t = type(value) + if t == "string" then + return pdfconstant(value) + elseif t ~= "table" then + return value + end + local kind = value[1] + if kind == name_code then + return pdfconstant(value[2]) + elseif kind == string_code then + return pdfliteral(value[2],value[3]) + elseif kind == array_code then + return copyarray(xref,copied,object[key]) + elseif kind == dictionary_code then + return copydictionary(xref,copied,object[key]) + elseif kind == null_code then + return pdfnull() + elseif kind == reference_code then + -- expand + return deepcopyobject(xref,copied,object[key]) + else + report("weird: %s", objecttypes[kind] or "?") + end + end + + copyarray = function (xref,copied,object) + local target = pdfarray() + local source = object.__raw__ + for i=1,#source do + target[i] = copyobject(xref,copied,object,i,source[i]) + end + return target + end + + copydictionary = function (xref,copied,object) + local target = pdfdictionary() + local source = object.__raw__ + for key, value in next, source do + target[key] = copyobject(xref,copied,object,key,value) + end + return target + end + + -- local function copyresources(pdfdoc,xref,copied,pagedata) + -- local Resources = pagedata.Resources + -- if Resources then + -- local r = pdfreserveobject() + -- local d = copydictionary(xref,copied,Resources) + -- pdfflushobject(r,tostring(d)) + -- return pdfreference(r) + -- end + -- end + + local function copyresources(pdfdoc,xref,copied,pagedata) + local Resources = pagedata.Resources + -- + -- -- This needs testing: + -- + -- if not Resources then + -- local Parent = page.Parent + -- while (Parent and (Parent.__type__ == dictionary_code or Parent.__type__ == reference_code) do + -- Resources = Parent.Resources + -- if Resources then + -- break + -- end + -- Parent = Parent.Parent + -- end + -- end + if Resources then + local d = copydictionary(xref,copied,Resources) + return shareobjectreference(d) + end + end + + local openpdf = lpdf_epdf.load + local closepdf = lpdf_epdf.unload + + local function querypdf(pdfdoc,pagenumber) + if pdfdoc then + if not pagenumber then + pagenumber = 1 + end + local root = pdfdoc.Catalog + local page = pdfdoc.pages[pagenumber] + if page then + -- todo + local mediabox = page.MediaBox or { 0, 0, 0, 0 } + local cropbox = page.CropBox or mediabox + return { + filename = pdfdoc.filename, + pagenumber = pagenumber, + nofpages = pdfdoc.nofpages, + boundingbox = scaledbbox(cropbox), + cropbox = cropbox, + mediabox = mediabox, + bleedbox = page.BleedBox or cropbox, + trimbox = page.TrimBox or cropbox, + artbox = page.ArtBox or cropbox, + } + end + end + end + + local function copypage(pdfdoc,pagenumber,attributes) + if pdfdoc then + local root = pdfdoc.Catalog + local page = pdfdoc.pages[pagenumber or 1] + local pageinfo = querypdf(pdfdoc,pagenumber) + local contents = page.Contents + local xref = pdfdoc.__xrefs__ + local copied = pdfdoc.__copied__ + local xobject = pdfdictionary { + Group = copyobject(xref,copied,page,"Group"), + LastModified = copyobject(xref,copied,page,"LastModified"), + MetaData = copyobject(xref,copied,root,"MetaData"), + Metadata = copyobject(xref,copied,page,"Metadata"), + PieceInfo = copyobject(xref,copied,page,"PieceInfo"), + Resources = copyresources(pdfdoc,xref,copied,page), + SeparationInfo = copyobject(xref,copied,page,"SeparationInfo"), + } + if attributes then + for k, v in expanded(attributes) do + page[k] = v -- maybe nested + end + end + local content = "" + local ctype = contents.__type__ + -- we always recompress because image object streams can not be + -- influenced (yet) + if ctype == stream_code then + content = contents() -- uncompressed + elseif ctype == array_code then + content = { } + for i=1,#contents do + content[i] = contents[i]() -- uncompressed + end + content = concat(content," ") + end + -- still not nice: we double wrap now + return newimage { + bbox = pageinfo.boundingbox, + stream = content, + attr = xobject(), + } + end + end + + lpdf_epdf.image = { + open = openpdf, + close = closepdf, + query = querypdf, + copy = copypage, + } + +end end + +-- local d = lpdf_epdf.load("e:/tmp/oeps.pdf") +-- inspect(d) +-- inspect(d.Catalog.Lang) +-- inspect(d.Catalog.OCProperties.D.AS[1].Event) +-- inspect(d.Catalog.Metadata()) +-- inspect(d.Catalog.Pages.Kids[1]) +-- inspect(d.layers) +-- inspect(d.pages) +-- inspect(d.destinations) +-- inspect(lpdf_epdf.getpagecontent(d,1)) +-- inspect(lpdf_epdf.contenttotext(document,lpdf_epdf.getpagecontent(d,1))) +-- inspect(lpdf_epdf.getstructure(document,lpdf_epdf.getpagecontent(d,1))) diff --git a/tex/context/base/mkiv/mlib-lua.lua b/tex/context/base/mkiv/mlib-lua.lua index 543d04697..eca17b751 100644 --- a/tex/context/base/mkiv/mlib-lua.lua +++ b/tex/context/base/mkiv/mlib-lua.lua @@ -74,11 +74,13 @@ local f_integer = formatters["%i"] local f_numeric = formatters["%n"] local f_pair = formatters["(%n,%n)"] +local f_ctrl = formatters["(%n,%n) .. controls (%n,%n) and (%n,%n)"] local f_triplet = formatters["(%n,%n,%n)"] local f_quadruple = formatters["(%n,%n,%n,%n)"] local f_points = formatters["%p"] local f_pair_pt = formatters["(%p,%p)"] +local f_ctrl_pt = formatters["(%p,%p) .. controls (%p,%p) and (%p,%p)"] local f_triplet_pt = formatters["(%p,%p,%p)"] local f_quadruple_pt = formatters["(%p,%p,%p,%p)"] @@ -231,7 +233,7 @@ function mp.quadruplepoints(w,x,y,z) end end -local function mppath(f,t,connector,cycle) +local function mppath(f2,f6,t,connector,cycle) if type(t) == "table" then local tn = #t if tn > 0 then @@ -242,11 +244,23 @@ local function mppath(f,t,connector,cycle) connector = "--" end local ti = t[1] - n = n + 1 ; buffer[n] = f(ti[1],ti[2]) + n = n + 1 ; + if #ti == 6 then + local tn = t[2] or t[1] + buffer[n] = f6(ti[1],ti[2],ti[5],ti[6],tn[3],tn[4]) + else + buffer[n] = f2(ti[1],ti[2]) + end for i=2,tn do local ti = t[i] n = n + 1 ; buffer[n] = connector - n = n + 1 ; buffer[n] = f(ti[1],ti[2]) + n = n + 1 ; + if #ti == 6 then + local tn = t[i+1] or t[1] + buffer[n] = f6(ti[1],ti[2],ti[5],ti[6],tn[3],tn[4]) + else + buffer[n] = f2(ti[1],ti[2]) + end end if cycle then n = n + 1 ; buffer[n] = connector @@ -257,11 +271,47 @@ local function mppath(f,t,connector,cycle) end function mp.path(...) - mppath(f_pair,...) + mppath(f_pair,f_ctrl,...) end function mp.pathpoints(...) - mppath(f_pair_pt,...) + mppath(f_pair_pt,f_ctrl_pt,...) +end + + +function mp.pathpoints(t,connector,cycle) + if type(t) == "table" then + local tn = #t + if tn > 0 then + if connector == true then + connector = "--" + cycle = true + elseif not connector then + connector = "--" + end + local ti = t[1] + n = n + 1 ; + if #ti == 6 then + buffer[n] = f_pair_pt_ctrl(ti[1],ti[2],ti[3],ti[4],ti[5],ti[6]) + else + buffer[n] = f_pair_pt(ti[1],ti[2]) + end + for i=2,tn do + local ti = t[i] + n = n + 1 ; buffer[n] = connector + n = n + 1 ; + if #ti == 6 then + buffer[n] = f_pair_pt_ctrl(ti[1],ti[2],ti[3],ti[4],ti[5],ti[6]) + else + buffer[n] = f_pair_pt(ti[1],ti[2]) + end + end + if cycle then + n = n + 1 ; buffer[n] = connector + n = n + 1 ; buffer[n] = "cycle" + end + end + end end function mp.size(t) diff --git a/tex/context/base/mkiv/mtx-context-listing.tex b/tex/context/base/mkiv/mtx-context-listing.tex index 29c4999ae..1053e80b9 100644 --- a/tex/context/base/mkiv/mtx-context-listing.tex +++ b/tex/context/base/mkiv/mtx-context-listing.tex @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -%D This is a \TEXEXEC\ features that has been moved to \MKIV. +%D This is a \TEXEXEC\ feature that has been moved to \MKIV. % begin help % diff --git a/tex/context/base/mkiv/mult-def.lua b/tex/context/base/mkiv/mult-def.lua index 803d6c3e1..f0da0e38f 100644 --- a/tex/context/base/mkiv/mult-def.lua +++ b/tex/context/base/mkiv/mult-def.lua @@ -11552,6 +11552,12 @@ return { ["pe"]="گاموای", ["ro"]="ystep", }, + ["ownerpassword"]={ + ["en"]="ownerpassword", + }, + ["userpassword"]={ + ["en"]="userpassword", + }, }, ["elements"]={ ["answerlines"]={ diff --git a/tex/context/base/mkiv/mult-prm.mkiv b/tex/context/base/mkiv/mult-prm.mkiv index 1b9195f41..d6131bc38 100644 --- a/tex/context/base/mkiv/mult-prm.mkiv +++ b/tex/context/base/mkiv/mult-prm.mkiv @@ -55,7 +55,7 @@ "pdflastobj", "pdflastxform", "pdflastximage", "pdflastximagepages", "pdflastxpos", "pdflastypos", "pdflinkmargin", "pdfliteral", "pdfmapfile", "pdfmapline", "pdfmajorversion", "pdfminorversion", "pdfnames", - "pdfnoligatures", "pdfnormaldeviate", "pdfobj", + "pdfnoligatures", "pdfnormaldeviate", "pdfobj", "pdfrecompress", "pdfobjcompresslevel", "pdfoutline", "pdfoutput", "pdfpageattr", "pdfpagebox", "pdfpageheight", "pdfpageref", "pdfpageresources", "pdfpagesattr", "pdfpagewidth", "pdfpkfixeddpi", "pdfpkmode", diff --git a/tex/context/base/mkiv/page-sid.mkiv b/tex/context/base/mkiv/page-sid.mkiv index 32dc94f6a..9ac95472a 100644 --- a/tex/context/base/mkiv/page-sid.mkiv +++ b/tex/context/base/mkiv/page-sid.mkiv @@ -541,13 +541,18 @@ \def\page_sides_inject_dummy_lines {\par \nointerlineskip + % \ifnum\lastpenalty>\zerocount + % \penalty\plustenthousand + % \fi \dontleavehmode \iftracesidefloats \page_sides_inject_dummy_line_traced \else \page_sides_inject_dummy_line_normal \fi - \vskip-\dimexpr\lineheight+\strutdp\relax + \par + \ignoreparskip + \kern-\dimexpr\lineheight+\strutdp\relax \ignoreparskip \blank[\v!samepage] \blank[\v!disable]} @@ -716,6 +721,15 @@ %D As we have no clear end of one or more paragraphs we only have pre float %D skips. +\newconstant\c_page_sides_page_method % will be: \c_page_sides_page_method\plusone + +\def\page_otr_force_new_page_one + {\vskip\d_page_sides_height + \penalty\outputpenalty + \vskip-\dimexpr\d_page_sides_height-\strutdp\relax + \prevdepth\strutdp} + %\ignoreparskip} + \def\page_sides_handle_float#1% grid (4) is rather experimental {\page_sides_check_horizontal_skips \page_sides_check_vertical_skips @@ -726,8 +740,16 @@ \page_sides_relocate_float{#1}% \page_sides_apply_vertical_shift \page_sides_analyse_space - \ifconditional\c_page_floats_room \else - \page_otr_fill_and_eject_page + \ifconditional\c_page_floats_room + % we're ok + \else + \ifcase\c_page_sides_page_method + \page_otr_fill_and_eject_page + \or + \page_otr_force_new_page_one + \else + \page_otr_fill_and_eject_page + \fi \page_sides_analyse_space %\page_sides_inject_before \page_sides_inject_dummy_lines diff --git a/tex/context/base/mkiv/publ-dat.lua b/tex/context/base/mkiv/publ-dat.lua index 310df82f3..c0682613a 100644 --- a/tex/context/base/mkiv/publ-dat.lua +++ b/tex/context/base/mkiv/publ-dat.lua @@ -22,6 +22,10 @@ if not characters then dofile(resolvers.findfile("char-tex.lua")) end +if not utilities.sequencers then + dofile(resolvers.findfile("util-seq.lua")) +end + local lower, find, sub = string.lower, string.find, string.sub local concat, copy, tohash = table.concat, table.copy, table.tohash local next, type, rawget, tonumber = next, type, rawget, tonumber @@ -1248,3 +1252,5 @@ do writers.pagenumber = writers.range end + +-- inspect(publications.load { filename = "e:/tmp/oeps.bib" }) diff --git a/tex/context/base/mkiv/publ-ini.lua b/tex/context/base/mkiv/publ-ini.lua index 9dfeda168..58a1d8f5e 100644 --- a/tex/context/base/mkiv/publ-ini.lua +++ b/tex/context/base/mkiv/publ-ini.lua @@ -229,9 +229,9 @@ logs.registerfinalactions(function() logs.startfilelogging(report,"used btx commands") done = true end - if isdefined[command] then + if isdefined(command) then report("%-20s %-20s % 5i %s",name,command,n,"known") - elseif isdefined[upper(command)] then + elseif isdefined(upper(command)) then report("%-20s %-20s % 5i %s",name,command,n,"KNOWN") else report("%-20s %-20s % 5i %s",name,command,n,"unknown") @@ -246,7 +246,7 @@ logs.registerfinalactions(function() logs.starterrorlogging(report,"unknown btx commands") for name, dataset in sortedhash(datasets) do for command, n in sortedhash(dataset.commands) do - if not isdefined[command] and not isdefined[upper(command)] then + if not isdefined(command) and not isdefined(upper(command)) then report("%-20s %-20s % 5i %s",name,command,n,"unknown") end end diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf Binary files differindex 4a95efec4..bc20d5d59 100644 --- a/tex/context/base/mkiv/status-files.pdf +++ b/tex/context/base/mkiv/status-files.pdf diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf Binary files differindex bd1efee16..ac101317c 100644 --- a/tex/context/base/mkiv/status-lua.pdf +++ b/tex/context/base/mkiv/status-lua.pdf diff --git a/tex/context/base/mkiv/strc-doc.lua b/tex/context/base/mkiv/strc-doc.lua index 80e41451e..f2674ea5a 100644 --- a/tex/context/base/mkiv/strc-doc.lua +++ b/tex/context/base/mkiv/strc-doc.lua @@ -124,12 +124,16 @@ local tobesaved = allocate() sections.collected = collected sections.tobesaved = tobesaved --- local function initializer() --- collected = sections.collected --- tobesaved = sections.tobesaved --- end --- --- job.register('structures.sections.collected', tobesaved, initializer) +-- We have to save this mostly redundant list because we can have (rare) +-- cases with own numbers that don't end up in the list so we get out of +-- sync when we use (*). + +local function initializer() + collected = sections.collected + tobesaved = sections.tobesaved +end + +job.register('structures.sections.collected', tobesaved, initializer) local registered = sections.registered or allocate() sections.registered = registered @@ -160,7 +164,7 @@ end local lastsaved = 0 function sections.save(sectiondata) --- local sectionnumber = helpers.simplify(section.sectiondata) -- maybe done earlier +local sectiondata = helpers.simplify(sectiondata) -- maybe done earlier local numberdata = sectiondata.numberdata local ntobesaved = #tobesaved if not numberdata or sectiondata.metadata.nolist then @@ -180,28 +184,28 @@ function sections.currentsectionindex() return lastsaved -- only for special controlled situations end -function sections.load() - setmetatableindex(collected,nil) - local lists = lists.collected - for i=1,#lists do - local list = lists[i] - local metadata = list.metadata - if metadata and metadata.kind == "section" and not metadata.nolist then - local numberdata = list.numberdata - if numberdata then - collected[#collected+1] = numberdata - end - end - end - sections.load = functions.dummy -end - -table.setmetatableindex(collected, function(t,i) - sections.load() - return collected[i] or { } -end) - +-- See comment above (*). We cannot use the following space optimization: +-- +-- function sections.load() +-- setmetatableindex(collected,nil) +-- local lists = lists.collected +-- for i=1,#lists do +-- local list = lists[i] +-- local metadata = list.metadata +-- if metadata and metadata.kind == "section" and not metadata.nolist then +-- local numberdata = list.numberdata +-- if numberdata then +-- collected[#collected+1] = numberdata +-- end +-- end +-- end +-- sections.load = functions.dummy +-- end -- +-- table.setmetatableindex(collected, function(t,i) +-- sections.load() +-- return collected[i] or { } +-- end) sections.verbose = true diff --git a/tex/context/base/mkiv/strc-reg.lua b/tex/context/base/mkiv/strc-reg.lua index b4d660c2b..b51106e6a 100644 --- a/tex/context/base/mkiv/strc-reg.lua +++ b/tex/context/base/mkiv/strc-reg.lua @@ -7,7 +7,7 @@ if not modules then modules = { } end modules ['strc-reg'] = { } local next, type, tonumber = next, type, tonumber -local format, gmatch = string.format, string.gmatch +local char, format, gmatch = string.char, string.format, string.gmatch local equal, concat, remove = table.are_equal, table.concat, table.remove local lpegmatch, P, C, Ct = lpeg.match, lpeg.P, lpeg.C, lpeg.Ct local allocate = utilities.storage.allocate @@ -782,24 +782,28 @@ local function crosslinkseewords(result) -- all words local seeparent = seeparents[text] if seeparent then local seeindex = seewords[text] - local s, ns, d, w, l = { }, 0, data.split, seeparent.split, data.list - -- trick: we influence sorting by adding fake subentries - for i=1,#d do - ns = ns + 1 - s[ns] = d[i] -- parent - end - for i=1,#w do - ns = ns + 1 - s[ns] = w[i] -- see - end - data.split = s - -- we also register a fake extra list entry so that the - -- collapser works okay - l[#l+1] = { text, "" } +-- local s, ns, d, w, l = { }, 0, data.split, seeparent.split, data.list +-- -- trick: we influence sorting by adding fake subentries +-- for i=1,#d do +-- ns = ns + 1 +-- s[ns] = d[i] -- parent +-- end +-- for i=1,#w do +-- ns = ns + 1 +-- s[ns] = w[i] -- see +-- end +-- data.split = s +-- -- we also register a fake extra list entry so that the +-- -- collapser works okay +-- l[#l+1] = { text, "" } data.references.seeindex = seeindex if trace_registers then report_registers("see crosslink %03i: %s",seeindex,text) end + seeword.valid = true + else + report_registers("invalid crosslink : %s",text) + seeword.valid = false end end end @@ -829,11 +833,16 @@ function registers.prepare(data) local splitter = sorters.splitters.utf local result = data.result if result then + local seeprefix = char(0) for i=1, #result do local entry = result[i] local split = { } local list = entry.list if list then +if entry.seeword then + -- we can have multiple seewords, only two levels supported + list[#list+1] = { seeprefix .. strip(entry.seeword.text) } +end for l=1,#list do local ll = list[l] local word = ll[1] @@ -1125,6 +1134,7 @@ function registers.flush(data,options,prefixspec,pagespec) end -- ok, this is tricky: we use e[i] delayed so we need it to be local -- but we don't want to allocate too many entries so there we go + while d < #data do d = d + 1 local entry = data[d] @@ -1163,8 +1173,8 @@ function registers.flush(data,options,prefixspec,pagespec) started = false end if n == i then --- ctx_stopregisterentries() --- ctx_startregisterentries(n) + -- ctx_stopregisterentries() + -- ctx_startregisterentries(n) else while n > i do n = n - 1 @@ -1192,120 +1202,123 @@ function registers.flush(data,options,prefixspec,pagespec) end end end - if kind == 'entry' then - if show_page_number then - ctx_startregisterpages() - if collapse_singles or collapse_ranges then - -- we collapse ranges and keep existing ranges as they are - -- so we get prebuilt as well as built ranges - local first, last, prev, pages, dd, nofpages = entry, nil, entry, { }, d, 0 - while dd < #data do - dd = dd + 1 - local next = data[dd] - if next and next.metadata.kind == "see" then - dd = dd - 1 - break - else - local el, nl = entry.list, next.list - if not equal(el,nl) then - dd = dd - 1 - --~ first = nil - break - elseif next.references.lastrealpage then - nofpages = nofpages + 1 - pages[nofpages] = first and { first, last or first } or { entry, entry } - nofpages = nofpages + 1 - pages[nofpages] = { next, next } - first, last, prev = nil, nil, nil - elseif not first then - first, prev = next, next - elseif next.references.realpage - prev.references.realpage == 1 then -- 1 ? - last, prev = next, next - else - nofpages = nofpages + 1 - pages[nofpages] = { first, last or first } - first, last, prev = next, nil, next - end - end - end - if first then + + local function case_1() + -- we collapse ranges and keep existing ranges as they are + -- so we get prebuilt as well as built ranges + local first, last, prev, pages, dd, nofpages = entry, nil, entry, { }, d, 0 + while dd < #data do + dd = dd + 1 + local next = data[dd] + if next and next.metadata.kind == "see" then + dd = dd - 1 + break + else + local el, nl = entry.list, next.list + if not equal(el,nl) then + dd = dd - 1 + --~ first = nil + break + elseif next.references.lastrealpage then + nofpages = nofpages + 1 + pages[nofpages] = first and { first, last or first } or { entry, entry } + nofpages = nofpages + 1 + pages[nofpages] = { next, next } + first, last, prev = nil, nil, nil + elseif not first then + first, prev = next, next + elseif next.references.realpage - prev.references.realpage == 1 then -- 1 ? + last, prev = next, next + else nofpages = nofpages + 1 pages[nofpages] = { first, last or first } + first, last, prev = next, nil, next end - if collapse_ranges and nofpages > 1 then - nofpages = collapsepages(pages) - end - if nofpages > 0 then -- or 0 - d = dd - for p=1,nofpages do - local first, last = pages[p][1], pages[p][2] - if first == last then - if first.references.lastrealpage then - pagerange(first,first,true,prefixspec,pagespec) - else - pagenumber(first,prefixspec,pagespec) - end - elseif last.references.lastrealpage then - pagerange(first,last,true,prefixspec,pagespec) - else - pagerange(first,last,false,prefixspec,pagespec) - end + end + end + if first then + nofpages = nofpages + 1 + pages[nofpages] = { first, last or first } + end + if collapse_ranges and nofpages > 1 then + nofpages = collapsepages(pages) + end + if nofpages > 0 then -- or 0 + d = dd + for p=1,nofpages do + local first, last = pages[p][1], pages[p][2] + if first == last then + if first.references.lastrealpage then + pagerange(first,first,true,prefixspec,pagespec) + else + pagenumber(first,prefixspec,pagespec) end - elseif entry.references.lastrealpage then - pagerange(entry,entry,true,prefixspec,pagespec) + elseif last.references.lastrealpage then + pagerange(first,last,true,prefixspec,pagespec) else - pagenumber(entry,prefixspec,pagespec) + pagerange(first,last,false,prefixspec,pagespec) end - elseif collapse_packed then - local first = nil - local last = nil - while true do - if not first then - first = entry - end - last = entry - if d == #data then - break - else - d = d + 1 - local next = data[d] - if next.metadata.kind == "see" or not equal(entry.list,next.list) then - d = d - 1 - break - else - entry = next - end - end + end + elseif entry.references.lastrealpage then + pagerange(entry,entry,true,prefixspec,pagespec) + else + pagenumber(entry,prefixspec,pagespec) + end + end + + local function case_2() + local first = nil + local last = nil + while true do + if not first then + first = entry + end + last = entry + if d == #data then + break + else + d = d + 1 + local next = data[d] + if next.metadata.kind == "see" or not equal(entry.list,next.list) then + d = d - 1 + break + else + entry = next end - packed(first,last) -- returns internals + end + end + packed(first,last) -- returns internals + end + + local function case_3() + while true do + if entry.references.lastrealpage then + pagerange(entry,entry,true,prefixspec,pagespec) else - while true do - if entry.references.lastrealpage then - pagerange(entry,entry,true,prefixspec,pagespec) - else - pagenumber(entry,prefixspec,pagespec) - end - if d == #data then - break - else - d = d + 1 - local next = data[d] - if next.metadata.kind == "see" or not equal(entry.list,next.list) then - d = d - 1 - break - else - entry = next - end - end + pagenumber(entry,prefixspec,pagespec) + end + if d == #data then + break + else + d = d + 1 + local next = data[d] + if next.metadata.kind == "see" or not equal(entry.list,next.list) then + d = d - 1 + break + else + entry = next end end - ctx_stopregisterpages() end - elseif kind == 'see' then + end + + local function case_4() local t, nt = { }, 0 while true do +if entry.seeword and entry.seeword.valid then nt = nt + 1 t[nt] = entry +end if d == #data then break else @@ -1319,19 +1332,36 @@ function registers.flush(data,options,prefixspec,pagespec) end end end - ctx_startregisterseewords() for i=1,nt do local entry = t[i] local seeword = entry.seeword local seetext = seeword.text or "" local processor = seeword.processor or (entry.processors and entry.processors[1]) or "" local seeindex = entry.references.seeindex or "" - -- ctx_registerseeword(i,nt,processor,0,seeindex,seetext) ctx_registerseeword(i,nt,processor,0,seeindex,function() h_title(seetext,metadata) end) end + end + + if kind == "entry" then + if show_page_number then + ctx_startregisterpages() + if collapse_singles or collapse_ranges then + case_1() + elseif collapse_packed then + case_2() + else + case_3() + end + ctx_stopregisterpages() + end + elseif kind == "see" then + ctx_startregisterseewords() + case_4() ctx_stopregisterseewords() end + end + if started then ctx_stopregisterentry() started = false diff --git a/tex/context/base/mkiv/strc-reg.mkiv b/tex/context/base/mkiv/strc-reg.mkiv index 3f401bbac..e5adf5e2f 100644 --- a/tex/context/base/mkiv/strc-reg.mkiv +++ b/tex/context/base/mkiv/strc-reg.mkiv @@ -844,8 +844,7 @@ \endgroup} \unexpanded\def\startregisterseewords - {%\par % \ifhmode\crlf\fi % otherwise wrong level - \begingroup + {\begingroup \dostarttagged\t!registerpage\empty \useregisterstyleandcolor\c!pagestyle\c!pagecolor} diff --git a/tex/context/base/mkiv/syst-ini.mkiv b/tex/context/base/mkiv/syst-ini.mkiv index 485a87ecb..799fccc7a 100644 --- a/tex/context/base/mkiv/syst-ini.mkiv +++ b/tex/context/base/mkiv/syst-ini.mkiv @@ -1095,6 +1095,7 @@ \edef\pdfcompresslevel {\pdfvariable compresslevel} \pdfcompresslevel \plusnine \edef\pdfobjcompresslevel {\pdfvariable objcompresslevel} \pdfobjcompresslevel \plusone +%edef\pdfrecompress {\pdfvariable recompress} \pdfrecompress \zerocount \edef\pdfdecimaldigits {\pdfvariable decimaldigits} \pdfdecimaldigits \plusfive \edef\pdfgamma {\pdfvariable gamma} \pdfgamma \plusthousand \edef\pdfimageresolution {\pdfvariable imageresolution} \pdfimageresolution 300 diff --git a/tex/context/base/mkiv/util-str.lua b/tex/context/base/mkiv/util-str.lua index 5317da423..f575050ff 100644 --- a/tex/context/base/mkiv/util-str.lua +++ b/tex/context/base/mkiv/util-str.lua @@ -612,6 +612,7 @@ local sequenced=table.sequenced local formattednumber=number.formatted local sparseexponent=number.sparseexponent local formattedfloat=number.formattedfloat +local stripper=lpeg.patterns.stripzeros ]] else @@ -639,6 +640,7 @@ else formattednumber = number.formatted, sparseexponent = number.sparseexponent, formattedfloat = number.formattedfloat, + stripper = lpeg.patterns.stripzeros, } end @@ -907,9 +909,18 @@ local format_n = function() -- strips leading and trailing zeros and removes .0 return format("((a%s %% 1 == 0) and format('%%i',a%s) or tostring(a%s))",n,n,n) end +-- local format_N = function() -- strips leading and trailing zeros (also accepts string) +-- n = n + 1 +-- return format("tostring(tonumber(a%s) or a%s)",n,n) +-- end + local format_N = function() -- strips leading and trailing zeros (also accepts string) n = n + 1 - return format("tostring(tonumber(a%s) or a%s)",n,n) + if not f or f == "" then + return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or ((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripper,format('%%.9f',a%s)))",n,n,n,n,n) + else + return format("(((a%s %% 1 == 0) and format('%%i',a%s)) or lpegmatch(stripper,format('%%%sf',a%s)))",n,n,f,n) + end end local format_a = function(f) |