diff options
Diffstat (limited to 'tex')
37 files changed, 775 insertions, 258 deletions
diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index 8e8704bd1..6fe3bf184 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2017.08.11 14:00} +\newcontextversion{2017.08.13 16:37} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index f52e5b989..728a2e252 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2017.08.11 14:00} +\edef\contextversion{2017.08.13 16:37} %D For those who want to use this: diff --git a/tex/context/base/mkiv/buff-ini.lua b/tex/context/base/mkiv/buff-ini.lua index e2f0be0dc..600f9c959 100644 --- a/tex/context/base/mkiv/buff-ini.lua +++ b/tex/context/base/mkiv/buff-ini.lua @@ -68,6 +68,13 @@ local txtcatcodes = catcodenumbers.txtcatcodes local setdata = job.datasets.setdata local getdata = job.datasets.getdata +local ctx_viafile = context.viafile +local ctx_getbuffer = context.getbuffer +local ctx_pushcatcodetable = context.pushcatcodetable +local ctx_popcatcodetable = context.popcatcodetable +local ctx_setcatcodetable = context.setcatcodetable +local ctx_printlines = context.printlines + buffers = buffers or { } local buffers = buffers @@ -609,30 +616,32 @@ local function runbuffer(name,encapsulate,runnername,suffixes) return resultname -- first result end +local f_getbuffer = formatters["buffer.%s"] + local function getbuffer(name) local str = getcontent(name) if str ~= "" then -- characters.showstring(str) - context.viafile(str,formatters["buffer.%s"](validstring(name,"noname"))) + ctx_viafile(str,f_getbuffer(validstring(name,"noname"))) end end local function getbuffermkvi(name) -- rather direct ! - context.viafile(resolvers.macros.preprocessed(getcontent(name)),formatters["buffer.%s.mkiv"](validstring(name,"noname"))) + ctx_viafile(resolvers.macros.preprocessed(getcontent(name)),formatters["buffer.%s.mkiv"](validstring(name,"noname"))) end local function gettexbuffer(name) local buffer = name and cache[name] if buffer and buffer.data ~= "" then - context.pushcatcodetable() + ctx_pushcatcodetable() if buffer.catcodes == txtcatcodes then - context.setcatcodetable(txtcatcodes) + ctx_setcatcodetable(txtcatcodes) else - context.setcatcodetable(ctxcatcodes) + ctx_setcatcodetable(ctxcatcodes) end - -- context(function() context.viafile(buffer.data) end) - context.getbuffer { name } -- viafile flushes too soon - context.popcatcodetable() + -- context(function() ctx_viafile(buffer.data) end) + ctx_getbuffer { name } -- viafile flushes too soon + ctx_popcatcodetable() end end @@ -643,6 +652,12 @@ implement { name = "getbuffer", actions = getbuffer, arguments = "stri implement { name = "getbuffermkvi", actions = getbuffermkvi, arguments = "string" } implement { name = "gettexbuffer", actions = gettexbuffer, arguments = "string" } +interfaces.implement { + name = "getbuffercontent", + arguments = "string", + actions = { getcontent, context }, +} + implement { name = "typesetbuffer", actions = { runbuffer, context }, @@ -667,29 +682,29 @@ implement { implement { name = "feedback", -- bad name, maybe rename to injectbuffercontent - actions = { collectcontent, context.printlines }, + actions = { collectcontent, ctx_printlines }, arguments = "string" } do - local context = context - local ctxcore = context.core + local context = context + local ctxcore = context.core - local startbuffer = ctxcore.startbuffer - local stopbuffer = ctxcore.stopbuffer + local ctx_startbuffer = ctxcore.startbuffer + local ctx_stopbuffer = ctxcore.stopbuffer - local startcollecting = context.startcollecting - local stopcollecting = context.stopcollecting + local ctx_startcollecting = context.startcollecting + local ctx_stopcollecting = context.stopcollecting function ctxcore.startbuffer(...) - startcollecting() - startbuffer(...) + ctx_startcollecting() + ctx_startbuffer(...) end function ctxcore.stopbuffer() - stopbuffer() - stopcollecting() + ctx_stopbuffer() + ctx_stopcollecting() end end diff --git a/tex/context/base/mkiv/buff-ver.lua b/tex/context/base/mkiv/buff-ver.lua index 6af56b327..db3db5aea 100644 --- a/tex/context/base/mkiv/buff-ver.lua +++ b/tex/context/base/mkiv/buff-ver.lua @@ -791,8 +791,8 @@ local nospace = space^1/"" local endstring = P(-1) local compactors = { - [v_all] = Cs((backslash * (1-backslash-space)^1 * nospace * (endstring + fences) + 1)^0), - [v_absolute] = Cs((backslash * (1-symbols -space)^1 * nospace * (symbols+backslash) + 1) ^0), + [v_all] = Cs((backslash * (1-backslash-space)^1 * nospace * (endstring+fences) + 1)^0), + [v_absolute] = Cs((backslash * (1-symbols -space)^1 * nospace * (symbols +backslash) + 1) ^0), [v_last] = Cs((space^1 * endstring/"" + 1)^0), } diff --git a/tex/context/base/mkiv/char-utf.lua b/tex/context/base/mkiv/char-utf.lua index 6d6551226..2d557d4af 100644 --- a/tex/context/base/mkiv/char-utf.lua +++ b/tex/context/base/mkiv/char-utf.lua @@ -106,7 +106,7 @@ if not graphemes then end end - local function setlist(unicode,list,start) + local function setlist(unicode,list,start,category) if list[start] ~= 0x20 then local t = mathlists for i=start,#list do @@ -120,10 +120,12 @@ if not graphemes then t = f end end - t.ligature = unicode + t[category] = unicode end end + local mlists = { } + for unicode, v in next, data do local vs = v.specials if vs then @@ -147,15 +149,23 @@ if not graphemes then -- end if (kind == "char" or kind == "compat") and (size > 2) and (v.mathclass or v.mathspec) then - setlist(unicode,vs,2) + setlist(unicode,vs,2,"specials") end end local ml = v.mathlist if ml then - setlist(unicode,ml,1) + mlists[unicode] = ml end end + -- these win: + + for unicode, ml in next, mlists do + setlist(unicode,ml,1,"mathlist") + end + + mlists = nil + if storage then storage.register("characters/graphemes", characters.graphemes, "characters.graphemes") storage.register("characters/collapsed", characters.collapsed, "characters.collapsed") diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index 1cf30f5bb..4611e5c2f 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2017.08.11 14:00} +\newcontextversion{2017.08.13 16:37} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index d9e2e3fc3..7e80b760a 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -41,7 +41,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2017.08.11 14:00} +\edef\contextversion{2017.08.13 16:37} \edef\contextkind {beta} %D For those who want to use this: diff --git a/tex/context/base/mkiv/font-dsp.lua b/tex/context/base/mkiv/font-dsp.lua index dc1b95e14..dee115a6b 100644 --- a/tex/context/base/mkiv/font-dsp.lua +++ b/tex/context/base/mkiv/font-dsp.lua @@ -1465,7 +1465,7 @@ function gposhandlers.single(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofg local value = readposition(f,format,tableoffset,getdelta) local coverage = readcoverage(f,tableoffset+coverage) for index, newindex in next, coverage do - coverage[index] = value + coverage[index] = value -- will be packed and shared anyway end return { format = "single", diff --git a/tex/context/base/mkiv/font-ext.lua b/tex/context/base/mkiv/font-ext.lua index 46309c324..6020d6374 100644 --- a/tex/context/base/mkiv/font-ext.lua +++ b/tex/context/base/mkiv/font-ext.lua @@ -1463,10 +1463,10 @@ do vectors.arab = { gsub = { ccmp = 1, - init = 2, - medi = 3, - fina = 4, - isol = 5, + isol = 2, + fina = 3, + medi = 4, + init = 5, rlig = 6, rclt = 7, calt = 8, diff --git a/tex/context/base/mkiv/font-otj.lua b/tex/context/base/mkiv/font-otj.lua index 1fbf61197..74e052408 100644 --- a/tex/context/base/mkiv/font-otj.lua +++ b/tex/context/base/mkiv/font-otj.lua @@ -26,6 +26,9 @@ if not modules then modules = { } end modules ['font-otj'] = { -- if needed we can flag a kern node as immutable +-- The thing with these positioning options is that it is not clear what Uniscribe does with +-- the 2rl flag and we keep oscillating a between experiments. + if not nodes.properties then return end local next, rawget, tonumber = next, rawget, tonumber @@ -220,7 +223,15 @@ function injections.getligaindex(n,default) return default end -function injections.setcursive(start,nxt,factor,rlmode,exit,entry,tfmstart,tfmnext) -- hm: nuts or nodes +function injections.setcursive(start,nxt,factor,rlmode,exit,entry,tfmstart,tfmnext,r2lflag) + + -- The standard says something about the r2lflag related to the first in a series: + -- + -- When this bit is set, the last glyph in a given sequence to which the cursive + -- attachment lookup is applied, will be positioned on the baseline. + -- + -- But it looks like we don't need to consider it. + local dx = factor*(exit[1]-entry[1]) local dy = -factor*(exit[2]-entry[2]) local ws = tfmstart.width @@ -276,7 +287,9 @@ function injections.setcursive(start,nxt,factor,rlmode,exit,entry,tfmstart,tfmne return dx, dy, nofregisteredcursives end -function injections.setposition(current,factor,rlmode,r2lflag,spec,injection) -- r2lflag not used +-- kind: 0=single 1=first of pair, 2=second of pair + +function injections.setposition(kind,current,factor,rlmode,spec,injection) local x = factor*spec[1] local y = factor*spec[2] local w = factor*spec[3] @@ -384,6 +397,26 @@ function injections.setmove(current,factor,rlmode,x,injection) if not injection then injection = "injections" end +if rlmode and rlmode < 0 then + -- we need to swap with a single so then we also need to to it here + -- as move is just a simple single + if p then + local i = p[injection] + if i then + i.rightkern = dx + (i.rightkern or 0) + else + p[injection] = { + rightkern = dx, + } + end + else + properties[current] = { + [injection] = { + rightkern = dx, + }, + } + end +else if p then local i = p[injection] if i then @@ -400,6 +433,7 @@ function injections.setmove(current,factor,rlmode,x,injection) }, } end +end return dx, nofregisteredkerns else return 0, 0 @@ -755,11 +789,16 @@ local function inject_positions_only(head,where) if yoffset and yoffset ~= 0 then setoffsets(current,false,yoffset) end - local leftkern = i.leftkern + local leftkern = i.leftkern + local rightkern = i.rightkern if leftkern and leftkern ~= 0 then +if rightkern and leftkern == -rightkern then + setoffsets(current,leftkern,false) + rightkern = 0 +else head = insert_node_before(head,current,fontkern(leftkern)) +end end - local rightkern = i.rightkern if rightkern and rightkern ~= 0 then insert_node_after(head,current,fontkern(rightkern)) end @@ -969,8 +1008,6 @@ local function inject_everything(head,where) -- local current = head local last = nil - local font = font - local markdata = nil local prev = nil local next = nil local prevdisc = nil @@ -1145,11 +1182,16 @@ local function inject_everything(head,where) end end -- left|glyph|right - local leftkern = i.leftkern + local leftkern = i.leftkern + local rightkern = i.rightkern if leftkern and leftkern ~= 0 then +if rightkern and leftkern == -rightkern then + setoffsets(current,leftkern,false) + rightkern = 0 +else head = insert_node_before(head,current,fontkern(leftkern)) +end end - local rightkern = i.rightkern if rightkern and rightkern ~= 0 then insert_node_after(head,current,fontkern(rightkern)) end diff --git a/tex/context/base/mkiv/font-otl.lua b/tex/context/base/mkiv/font-otl.lua index 4625c5aaf..6f71f837f 100644 --- a/tex/context/base/mkiv/font-otl.lua +++ b/tex/context/base/mkiv/font-otl.lua @@ -101,6 +101,33 @@ registerdirective("fonts.otf.loader.forcenotdef", function(v) forcenotdef = registerotfenhancer("check extra features", function() end) -- placeholder +-- Kai has memory problems on osx so here is an experiment (I only tested on windows as +-- my test mac is old and gets no updates and is therefore rather useless.): + +local checkmemory = utilities.lua and utilities.lua.checkmemory +local threshold = 100 -- MB +local tracememory = false + +registertracker("fonts.otf.loader.memory",function(v) tracememory = v end) + +if not checkmemory then -- we need a generic plug (this code might move): + + local collectgarbage = collectgarbage + + checkmemory = function(previous,threshold) -- threshold in MB + local current = collectgarbage("count") + if previous then + local checked = (threshold or 64)*1024 + if current - previous > checked then + collectgarbage("collect") + current = collectgarbage("count") + end + end + return current + end + +end + function otf.load(filename,sub,instance) local base = file.basename(file.removesuffix(filename)) local name = file.removesuffix(base) -- already no suffix @@ -132,9 +159,13 @@ function otf.load(filename,sub,instance) data = otfreaders.loadfont(filename,sub or 1,instance) -- we can pass the number instead (if it comes from a name search) if data then -- todo: make this a plugin + local used = checkmemory() local resources = data.resources local svgshapes = resources.svgshapes local sbixshapes = resources.sbixshapes + if cleanup == 0 then + checkmemory(used,threshold,tracememory) + end if svgshapes then resources.svgshapes = nil if otf.svgenabled then @@ -149,6 +180,11 @@ function otf.load(filename,sub,instance) timestamp = timestamp, } end + if cleanup > 1 then + collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) + end end if sbixshapes then resources.sbixshapes = nil @@ -164,18 +200,31 @@ function otf.load(filename,sub,instance) timestamp = timestamp, } end + if cleanup > 1 then + collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) + end end -- otfreaders.compact(data) + if cleanup == 0 then + checkmemory(used,threshold,tracememory) + end otfreaders.rehash(data,"unicodes") otfreaders.addunicodetable(data) otfreaders.extend(data) + if cleanup == 0 then + checkmemory(used,threshold,tracememory) + end otfreaders.pack(data) report_otf("loading done") report_otf("saving %a in cache",filename) data = containers.write(otf.cache, hash, data) if cleanup > 1 then collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) end stoptiming(otfreaders) if elapsedtime then @@ -183,10 +232,14 @@ function otf.load(filename,sub,instance) end if cleanup > 3 then collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) end data = containers.read(otf.cache,hash) -- this frees the old table and load the sparse one if cleanup > 2 then collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) end else data = nil diff --git a/tex/context/base/mkiv/font-ots.lua b/tex/context/base/mkiv/font-ots.lua index d8754da32..6a893a67d 100644 --- a/tex/context/base/mkiv/font-ots.lua +++ b/tex/context/base/mkiv/font-ots.lua @@ -206,6 +206,7 @@ local usesfont = nuts.uses_font local insert_node_after = nuts.insert_after local copy_node = nuts.copy local copy_node_list = nuts.copy_list +local remove_node = nuts.remove local find_node_tail = nuts.tail local flush_node_list = nuts.flush_list local flush_node = nuts.flush_node @@ -494,7 +495,15 @@ local function markstoligature(head,start,stop,char) end end -local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfound) -- brr head +-- Remark for Kai: (some arabic fonts do mark + mark = other mark and such) +-- +-- The hasmarks is needed for ligatures of marks that are part of a ligature in +-- which case we assume that we can delete the marks anyway (we can always become +-- more clever if needed) .. in fact the whole logic here should be redone. We're +-- in the not discfound branch then. We now have skiphash too so we can be more +-- selective if needed (todo). + +local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfound,hasmarks) -- brr head if getattr(start,a_noligature) == 1 then -- so we can do: e\noligature{ff}e e\noligature{f}fie (we only look at the first) return head, start @@ -519,19 +528,21 @@ local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfou set_components(base,comp) setlink(prev,base,next) if not discfound then - local deletemarks = not skiphash + local deletemarks = not skiphash or hasmarks local components = start local baseindex = 0 local componentindex = 0 local head = base local current = base - -- first we loop over the glyphs in start .. stop + -- first we loop over the glyphs in start ... stop while start do local char = getchar(start) if not marks[char] then baseindex = baseindex + componentindex componentindex = count_components(start,marks) - elseif not deletemarks then -- quite fishy + -- we can be more clever here: "not deletemarks or (skiphash and not skiphash[char])" + -- and such: + elseif not deletemarks then setligaindex(start,baseindex + getligaindex(start,componentindex)) if trace_marks then logwarning("%s: keep mark %s, gets index %s",pref(dataset,sequence),gref(char),getligaindex(start)) @@ -549,6 +560,7 @@ local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfou while start do local char = ischar(start) if char then + -- also something skiphash here? if marks[char] then setligaindex(start,baseindex + getligaindex(start,componentindex)) if trace_marks then @@ -724,7 +736,8 @@ function handlers.gsub_multiple(head,start,dataset,sequence,multiple,rlmode,skip end -- Don't we deal with disc otherwise now? I need to check if the next one can be --- simplified. +-- simplified. Anyway, it can be way messier: marks that get removed as well as +-- marks that are kept. function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skiphash) local current = getnext(start) @@ -764,14 +777,16 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip -- ok, goto next lookup end end - else + else -- is the check for disc still valid here ? and why only replace then local discfound = false local lastdisc = nil + local hasmarks = marks[startchar] while current do local char, id = ischar(current,currentfont) if char then if skiphash and skiphash[char] then current = getnext(current) + -- if stop then stop = current end -- ? else -- ligature is a tree local lg = ligature[char] -- can there be multiple in a row? maybe in a bad font if lg then @@ -779,6 +794,9 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip discfound = lastdisc lastdisc = nil end + if marks[char] then + hasmarks = true + end stop = current -- needed for fake so outside then ligature = lg current = getnext(current) @@ -790,8 +808,20 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip -- kind of weird break elseif id == disc_code then - -- tricky .. we also need to do pre here - local replace = getfield(current,"replace") + -- + -- Kai: see chainprocs, we probably could do the same here or was there a reason + -- why we kept the replace check here. + -- + -- if not discfound then + -- discfound = current + -- end + -- if current == stop then + -- break -- okay? or before the disc + -- else + -- current = getnext(current) + -- end + -- + local replace = getfield(current,"replace") -- hm: pre and post if replace then -- of{f-}{}{f}e o{f-}{}{f}fe o{-}{}{ff}e (oe and ff ligature) -- we can end up here when we have a start run .. testruns start at a disc but @@ -801,6 +831,9 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip if char then local lg = ligature[char] -- can there be multiple in a row? maybe in a bad font if lg then + if marks[char] then + hasmarks = true -- very unlikely + end ligature = lg replace = getnext(replace) else @@ -823,10 +856,10 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip if stop then if trace_ligatures then local stopchar = getchar(stop) - head, start = toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound) + head, start = toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound,hasmarks) logprocess("%s: replacing %s upto %s by ligature %s case 2",pref(dataset,sequence),gref(startchar),gref(stopchar),gref(lig)) else - head, start = toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound) + head, start = toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound,hasmarks) end else -- weird but happens (in some arabic font) @@ -848,7 +881,7 @@ function handlers.gpos_single(head,start,dataset,sequence,kerns,rlmode,skiphash, local startchar = getchar(start) local format = step.format if format == "single" or type(kerns) == "table" then -- the table check can go - local dx, dy, w, h = setposition(start,factor,rlmode,sequence.flags[4],kerns,injection) + local dx, dy, w, h = setposition(0,start,factor,rlmode,kerns,injection) if trace_kerns then logprocess("%s: shifting single %s by %s xy (%p,%p) and wh (%p,%p)",pref(dataset,sequence),gref(startchar),format,dx,dy,w,h) end @@ -858,7 +891,7 @@ function handlers.gpos_single(head,start,dataset,sequence,kerns,rlmode,skiphash, logprocess("%s: shifting single %s by %s %p",pref(dataset,sequence),gref(startchar),format,k) end end - return head, start, false + return head, start, true end function handlers.gpos_pair(head,start,dataset,sequence,kerns,rlmode,skiphash,step,injection) @@ -884,7 +917,7 @@ function handlers.gpos_pair(head,start,dataset,sequence,kerns,rlmode,skiphash,st if a == true then -- zero elseif a then -- #a > 0 - local x, y, w, h = setposition(start,factor,rlmode,sequence.flags[4],a,injection) + local x, y, w, h = setposition(1,start,factor,rlmode,a,injection) if trace_kerns then local startchar = getchar(start) logprocess("%s: shifting first of pair %s and %s by xy (%p,%p) and wh (%p,%p) as %s",pref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h,injection or "injections") @@ -894,7 +927,7 @@ function handlers.gpos_pair(head,start,dataset,sequence,kerns,rlmode,skiphash,st -- zero start = snext -- cf spec elseif b then -- #b > 0 - local x, y, w, h = setposition(snext,factor,rlmode,sequence.flags[4],b,injection) + local x, y, w, h = setposition(2,snext,factor,rlmode,b,injection) if trace_kerns then local startchar = getchar(snext) logprocess("%s: shifting second of pair %s and %s by xy (%p,%p) and wh (%p,%p) as %s",pref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h,injection or "injections") @@ -1104,7 +1137,8 @@ function handlers.gpos_cursive(head,start,dataset,sequence,exitanchors,rlmode,sk if entry then entry = entry[2] if entry then - local dx, dy, bound = setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar]) + local r2lflag = sequence.flags[4] -- mentioned in the standard + local dx, dy, bound = setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar],r2lflag) if trace_cursive then logprocess("%s: moving %s to %s cursive (%p,%p) using bound %s in %s mode",pref(dataset,sequence),gref(startchar),gref(nextchar),dx,dy,bound,mref(rlmode)) end @@ -1364,6 +1398,7 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup logwarning("%s: no ligatures starting with %s",cref(dataset,sequence,chainindex),gref(startchar)) end else + local hasmarks = marks[startchar] local current = getnext(start) local discfound = false local last = stop @@ -1394,6 +1429,9 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup ligatures = lg last = current nofreplacements = nofreplacements + 1 + if marks[char] then + hasmarks = true + end if current == stop then break else @@ -1417,7 +1455,7 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup logprocess("%s: replacing character %s upto %s by ligature %s case 4",cref(dataset,sequence,chainindex),gref(startchar),gref(getchar(stop)),gref(ligature)) end end - head, start = toligature(head,start,stop,ligature,dataset,sequence,skiphash,discfound) + head, start = toligature(head,start,stop,ligature,dataset,sequence,skiphash,discfound,hasmarks) return head, start, true, nofreplacements, discfound elseif trace_bugs then if start == stop then @@ -1442,7 +1480,7 @@ function chainprocs.gpos_single(head,start,stop,dataset,sequence,currentlookup,r if kerns then local format = currentlookup.format if format == "single" then - local dx, dy, w, h = setposition(start,factor,rlmode,sequence.flags[4],kerns) -- currentlookup.flags ? + local dx, dy, w, h = setposition(0,start,factor,rlmode,kerns) -- currentlookup.flags ? if trace_kerns then logprocess("%s: shifting single %s by %s (%p,%p) and correction (%p,%p)",cref(dataset,sequence),gref(startchar),format,dx,dy,w,h) end @@ -1452,6 +1490,7 @@ function chainprocs.gpos_single(head,start,stop,dataset,sequence,currentlookup,r logprocess("%s: shifting single %s by %s %p",cref(dataset,sequence),gref(startchar),format,k) end end + return head, start, true end end return head, start, false @@ -1488,7 +1527,7 @@ function chainprocs.gpos_pair(head,start,stop,dataset,sequence,currentlookup,rlm if a == true then -- zero elseif a then - local x, y, w, h = setposition(start,factor,rlmode,sequence.flags[4],a,"injections") -- currentlookups flags? + local x, y, w, h = setposition(1,start,factor,rlmode,a,"injections") -- currentlookups flags? if trace_kerns then local startchar = getchar(start) logprocess("%s: shifting first of pair %s and %s by (%p,%p) and correction (%p,%p)",cref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h) @@ -1498,7 +1537,7 @@ function chainprocs.gpos_pair(head,start,stop,dataset,sequence,currentlookup,rlm -- zero start = snext -- cf spec elseif b then -- #b > 0 - local x, y, w, h = setposition(snext,factor,rlmode,sequence.flags[4],b,"injections") + local x, y, w, h = setposition(2,snext,factor,rlmode,b,"injections") if trace_kerns then local startchar = getchar(start) logprocess("%s: shifting second of pair %s and %s by (%p,%p) and correction (%p,%p)",cref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h) @@ -1741,7 +1780,8 @@ function chainprocs.gpos_cursive(head,start,stop,dataset,sequence,currentlookup, if entry then entry = entry[2] if entry then - local dx, dy, bound = setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar]) + local r2lflag = sequence.flags[4] -- mentioned in the standard + local dx, dy, bound = setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar],r2lflag) if trace_cursive then logprocess("%s: moving %s to %s cursive (%p,%p) using bound %s in %s mode",pref(dataset,sequence),gref(startchar),gref(nextchar),dx,dy,bound,mref(rlmode)) end @@ -3808,11 +3848,11 @@ do while start do local char, id = ischar(start,font) if char then - local m = merged[char] - if m then - if skiphash and skiphash[char] then -- we never needed it here but let's try - start = getnext(start) - else + if skiphash and skiphash[char] then -- we never needed it here but let's try + start = getnext(start) + else + local m = merged[char] + if m then local a -- happens often so no assignment is faster if attr then if getattr(start,0) == attr and (not attribute or getprop(start,a_state) == attribute) then @@ -3847,9 +3887,9 @@ do else start = getnext(start) end + else + start = getnext(start) end - else - start = getnext(start) end elseif char == false or id == glue_code then -- a different font|state or glue (happens often) diff --git a/tex/context/base/mkiv/font-oup.lua b/tex/context/base/mkiv/font-oup.lua index ea66e141c..0ebd88c32 100644 --- a/tex/context/base/mkiv/font-oup.lua +++ b/tex/context/base/mkiv/font-oup.lua @@ -2487,6 +2487,7 @@ function readers.compact(data) lookup.merged = true end elseif nofsteps == 1 then + local kern = kerned if kind == "gpos_single" then if compact_singles then kerned = kerned + checkkerns(lookup) @@ -2496,6 +2497,9 @@ function readers.compact(data) kerned = kerned + checkpairs(lookup) end end + if kern ~= kerned then + -- lookup.kerned = true + end end end else @@ -2557,7 +2561,8 @@ local function checkflags(sequence,resources) or (markclass and c == "mark" and not markclass[k]) or c == skipligature or c == skipbase - t[k] = v or false + or false + t[k] = v return v end) else diff --git a/tex/context/base/mkiv/math-ini.mkiv b/tex/context/base/mkiv/math-ini.mkiv index 6fdf32531..a180464bf 100644 --- a/tex/context/base/mkiv/math-ini.mkiv +++ b/tex/context/base/mkiv/math-ini.mkiv @@ -122,6 +122,7 @@ \definesystemattribute[mathkernpairs] [public] \definesystemattribute[mathbidi] [public] \definesystemattribute[mathdomain] [public] +\definesystemattribute[mathcollapsing] [public] \definesystemattribute[displaymath] [public] @@ -1335,6 +1336,39 @@ \s!lcgreek=\v!italic, \s!ucgreek=\v!normal] % was: none +%D Math collapsing (ligatures) + +\installcorenamespace{mathcollapsing} + +\setnewconstant\c_math_collapsing_attribute\attributeunsetvalue + +\letvalue{\??mathcollapsing 1}\plusone % specials +\letvalue{\??mathcollapsing 2}\plustwo % specials + mathlist +\letvalue{\??mathcollapsing 3}\plusthree % mathlist + specials +\letvalue{\??mathcollapsing\v!none }\attributeunsetvalue +\letvalue{\??mathcollapsing\v!reset}\attributeunsetvalue + +\def\math_collapsing_initialize + {\ifnum\c_math_collapsing_attribute=\attributeunsetvalue \else + \clf_initializemathcollapsing % one time + \global\let\math_collapsing_initialize\relax + \fi} + +\appendtoks + \edef\p_collapsing{\mathematicsparameter\s!collapsing}% + \c_math_collapsing_attribute + \ifcsname\??mathcollapsing\p_collapsing\endcsname\lastnamedcs\else\attributeunsetvalue\fi + \relax +\to \everyswitchmathematics % only in mathematics + +\appendtoks + \math_collapsing_initialize + \attribute\mathcollapsingattribute\c_math_collapsing_attribute +\to \everymathematics + +\setupmathematics + [\s!collapsing=3] % mathlist wins over specials + %D Math italics (experiment) %D We need keys but what names to use and because we have hardcoded solution @@ -1357,14 +1391,6 @@ \global\let\math_italics_initialize\relax \fi} -% \appendtoks -% \edef\p_italics{\mathematicsparameter\s!italics}% -% \c_math_italics_attribute\csname\??mathitalics -% \ifcsname\??mathitalics\p_italics\endcsname\p_italics\else\v!none\fi -% \endcsname\relax -% % \math_italics_initialize -% \to \everyswitchmathematics % only in mathematics - \appendtoks \edef\p_italics{\mathematicsparameter\s!italics}% \c_math_italics_attribute diff --git a/tex/context/base/mkiv/math-noa.lua b/tex/context/base/mkiv/math-noa.lua index 317b2b6f1..acd2c0dc6 100644 --- a/tex/context/base/mkiv/math-noa.lua +++ b/tex/context/base/mkiv/math-noa.lua @@ -62,7 +62,7 @@ local trace_domains = false registertracker("math.domains", function local trace_families = false registertracker("math.families", function(v) trace_families = v end) local trace_fences = false registertracker("math.fences", function(v) trace_fences = v end) -local check_coverage = true registerdirective("math.checkcoverage", function(v) check_coverage = v end) +local check_coverage = true registerdirective("math.checkcoverage", function(v) check_coverage = v end) local report_processing = logreporter("mathematics","processing") local report_remapping = logreporter("mathematics","remapping") @@ -1399,9 +1399,7 @@ do return true end - local enable - - enable = function() + local enable = function() enableaction("math", "noads.handlers.italics") if trace_italics then report_italics("enabling math italics") @@ -1538,9 +1536,10 @@ do -- is validpair stil needed? - local collapse = { } - local mathlists = characters.mathlists - local validpair = { + local a_mathcollapsing = privateattribute("mathcollapsing") + local collapse = { } + local mathlists = characters.mathlists + local validpair = { [noad_ord] = true, [noad_rel] = true, [noad_bin] = true, -- new @@ -1557,7 +1556,7 @@ do collapse[math_char] = function(pointer,what,n,parent) if parent and mathlists[getchar(pointer)] then - local found, last, lucleus, lsup, lsub + local found, last, lucleus, lsup, lsub, category local tree = mathlists local current = parent while current and validpair[getsubtype(current)] do @@ -1568,19 +1567,34 @@ do if char then local match = tree[char] if match then - local ligature = match.ligature - if ligature then - found = ligature - last = current - lucleus = nucleus - lsup = sup - lsub = sub - end - tree = match - if sub or sup then - break + local method = getattr(current,a_mathcollapsing) + if method and method > 0 and method <= 3 then + local specials = match.specials + local mathlist = match.mathlist + local ligature + if method == 1 then + ligature = specials + elseif method == 2 then + ligature = specials or mathlist + else -- 3 + ligature = mathlist or specials + end + if ligature then + category = mathlist and "mathlist" or "specials" + found = ligature + last = current + lucleus = nucleus + lsup = sup + lsub = sub + end + tree = match + if sub or sup then + break + else + current = getnext(current) + end else - current = getnext(current) + break end else break @@ -1596,10 +1610,10 @@ do if not replace then if not reported[id][found] then reported[id][found] = true - report_collapsing("missing ligature %C",found) + report_collapsing("%s ligature %C from %s","ignoring",found,category) end elseif trace_collapsing then - report_collapsing("creating ligature %C",found) + report_collapsing("%s ligature %C from %s","creating",found,category) end setchar(pointer,found) local l = getnext(last) @@ -1627,6 +1641,20 @@ do return true end + local enable = function() + enableaction("math", "noads.handlers.collapse") + if trace_collapsing then + report_collapsing("enabling math collapsing") + end + enable = false + end + + implement { + name = "initializemathcollapsing", + actions = enable, + onlyonce = true, + } + end do diff --git a/tex/context/base/mkiv/mult-low.lua b/tex/context/base/mkiv/mult-low.lua index 29d7adc44..bc9bcaf71 100644 --- a/tex/context/base/mkiv/mult-low.lua +++ b/tex/context/base/mkiv/mult-low.lua @@ -427,5 +427,7 @@ return { "frule", -- "compoundhyphenpenalty", + -- + "start", "stop", } } diff --git a/tex/context/base/mkiv/mult-sys.mkiv b/tex/context/base/mkiv/mult-sys.mkiv index bd3ff9b3d..038d5e5af 100644 --- a/tex/context/base/mkiv/mult-sys.mkiv +++ b/tex/context/base/mkiv/mult-sys.mkiv @@ -302,6 +302,8 @@ \definesystemconstant {ucgreek} \definesystemconstant {sygreek} \definesystemconstant {italics} +\definesystemconstant {ligatures} +\definesystemconstant {collapsing} \definesystemconstant {run} diff --git a/tex/context/base/mkiv/publ-imp-apa.lua b/tex/context/base/mkiv/publ-imp-apa.lua index 8a1cc1b9c..79686447d 100644 --- a/tex/context/base/mkiv/publ-imp-apa.lua +++ b/tex/context/base/mkiv/publ-imp-apa.lua @@ -123,7 +123,7 @@ categories.newspaper = categories.magazine categories.periodical = { sets = { - author = { "editor", "publisher" }, + author = { "editor", "publisher", "organization", }, doi = generic.doi, }, required = { @@ -437,6 +437,7 @@ categories.unpublished = { categories.electronic = { sets = { doi = generic.doi, + author = { "author", "organization", }, }, required = { "title" diff --git a/tex/context/base/mkiv/publ-imp-apa.mkvi b/tex/context/base/mkiv/publ-imp-apa.mkvi index 182802331..5576b47f8 100644 --- a/tex/context/base/mkiv/publ-imp-apa.mkvi +++ b/tex/context/base/mkiv/publ-imp-apa.mkvi @@ -310,7 +310,8 @@ \c!etallimit=5, \c!etaldisplay=1, % TODO: when 2-4, show all first time, etaldisplay subsequently... \c!authorconversion=\v!name, - \c!sorttype=normal, + \c!sorttype=\v!normal, + \c!style=, \c!compress=\v!yes, % note that cite sorts only work with compress=yes. \c!separator:names:3={\btxcomma\btxlabeltext{and}\space}, % not \textampersand \c!separator:names:4={\btxspace\btxlabeltext{and}\space}] % not \textampersand @@ -345,6 +346,10 @@ [apa:\s!cite] \definebtx + [apa:\s!cite:organization] + [apa:\s!cite] + +\definebtx [apa:\s!cite:authoryear] [apa:\s!cite:author] [\c!left={(}, @@ -950,6 +955,16 @@ } \stoptexdefinition +\starttexdefinition unexpanded btx:apa:organization-if-not-author + \btxdoif {organization} { + \doifnot {\btxfoundname{author}} {organization} { + \btxspace + \btxflush{organization} + \btxcomma + } + } +\stoptexdefinition + % TODO: The title is terminated with period. However, % we probably don't want this before the parenthesis. @@ -1280,11 +1295,7 @@ \texdefinition{btx:apa:title-if-not-placed} \texdefinition{btx:apa:editor-in} \texdefinition{btx:apa:editionset} - \btxdoif {organization} { - \btxspace - \btxflush{organization} - \btxcomma - } + \texdefinition{btx:apa:organization-if-not-author} \texdefinition{btx:apa:wherefrom-publisher} \texdefinition{btx:apa:url-doi-note} \stopsetups @@ -1411,11 +1422,7 @@ \startsetups btx:apa:list:electronic \texdefinition{btx:apa:authoryear} \texdefinition{btx:apa:title-if-not-placed} - \btxdoif {organization} { - \btxspace - \btxflush{organization} - \btxperiod - } + \texdefinition{btx:apa:organization-if-not-author} \btxdoif {howpublished} { \btxspace \btxflush{howpublished} diff --git a/tex/context/base/mkiv/publ-imp-aps.mkvi b/tex/context/base/mkiv/publ-imp-aps.mkvi index 90d418ca0..4180b6491 100644 --- a/tex/context/base/mkiv/publ-imp-aps.mkvi +++ b/tex/context/base/mkiv/publ-imp-aps.mkvi @@ -283,6 +283,10 @@ [aps:\s!cite] \definebtx + [aps:\s!cite:organization] + [aps:\s!cite] + +\definebtx [aps:\s!cite:authoryear] [aps:\s!cite:author] [\c!left={(}, @@ -651,6 +655,16 @@ \btxcomma \stoptexdefinition +\starttexdefinition unexpanded btx:aps:organization-if-not-author + \btxdoif {organization} { + \doifnot {\btxfoundname{author}} {organization} { + \btxspace + \btxflush{organization} + \btxcomma + } + } +\stoptexdefinition + \starttexdefinition unexpanded btx:aps:editor-in \btxdoif {booktitle} { \btxlabeltext{In} @@ -903,6 +917,7 @@ \texdefinition{btx:aps:title} \texdefinition{btx:aps:editor-in} \texdefinition{btx:aps:editionset} + %\texdefination{btx:aps:organization-if-not-author}% first check publ-imp-aps.lua \btxdoif {organization} { \btxspace \btxflush{organization} @@ -1035,13 +1050,7 @@ \startsetups btx:aps:list:electronic \texdefinition{btx:aps:author} \texdefinition{btx:aps:title} - \btxdoif {organization} { - \doifnot {\btxfoundname{author}} {organization} { - \btxspace - \btxflush{organization} - \btxcomma - } - } + \texdefinition{btx:aps:organization-if-not-author} \btxdoif {howpublished} { \texdefinition{btx:aps:doi-url} {\btxspace\btxflush{howpublished}} } diff --git a/tex/context/base/mkiv/publ-imp-cite.mkvi b/tex/context/base/mkiv/publ-imp-cite.mkvi index 56af83a1b..8fe96429d 100644 --- a/tex/context/base/mkiv/publ-imp-cite.mkvi +++ b/tex/context/base/mkiv/publ-imp-cite.mkvi @@ -232,6 +232,9 @@ \startsetups \s!btx:\s!cite:pages \fastsetup{\s!btx:\s!cite:range} \stopsetups +\startsetups \s!btx:\s!cite:organization + \fastsetup{\s!btx:\s!cite:range} +\stopsetups % is the next one used? % Yes, bibtex is a mess and one can have pages or sometimes page diff --git a/tex/context/base/mkiv/publ-ini.mkiv b/tex/context/base/mkiv/publ-ini.mkiv index 981aa8f36..456919b6a 100644 --- a/tex/context/base/mkiv/publ-ini.mkiv +++ b/tex/context/base/mkiv/publ-ini.mkiv @@ -223,8 +223,7 @@ \installcommandhandler \??btxregister {btxregister} \??btxregister \installcommandhandler \??btxrendering {btxrendering} \??btxrendering -\let\currentbtxcitealternative\empty - +\let\currentbtxcitealternative \empty \let\currentbtxspecificationfallback\empty \unexpanded\def\setbtxparameterset#1#2% @@ -1229,6 +1228,7 @@ \let\p_publ_cite_righttext\empty \let\currentbtxciteuservariables\empty +\let\currentbtxcitealternative \empty \unexpanded\def\btxhybridcite % so one can alias the old {\dontleavehmode @@ -1400,8 +1400,9 @@ \unexpanded\def\citation {\doifelsenextoptionalcs\btxlistcitation \btxdirectlistcite} \unexpanded\def\nocitation{\doifelsenextoptionalcs\btxhiddencitation\btxdirecthiddencite} -\let\cite \citation -\let\nocite\nocitation +\let\cite \citation +\let\nocite \nocitation +\let\usecitation\citation \unexpanded\def\publ_entry_citation {\doifelsenextoptionalcs\btxlistcitation \btxdirectlistcite} \unexpanded\def\publ_entry_nocitation{\doifelsenextoptionalcs\btxhiddencitation\btxdirecthiddencite} diff --git a/tex/context/base/mkiv/spac-chr.lua b/tex/context/base/mkiv/spac-chr.lua index 299f86ad6..27eac1753 100644 --- a/tex/context/base/mkiv/spac-chr.lua +++ b/tex/context/base/mkiv/spac-chr.lua @@ -333,6 +333,10 @@ characters.methods = methods -- for current, char, font in traverse_char_data(head) will save 0.015 on a 300 page doc +-- this also works ok in math as we run over glyphs and these stay glyphs ... not sure +-- about scripts and such but that is not important anyway ... some day we can consider +-- special definitions in math + function characters.handler(head) local head = tonut(head) local wipe = false diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf Binary files differindex 72073241d..fa9c777c4 100644 --- a/tex/context/base/mkiv/status-files.pdf +++ b/tex/context/base/mkiv/status-files.pdf diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf Binary files differindex c94deff60..26c3eb179 100644 --- a/tex/context/base/mkiv/status-lua.pdf +++ b/tex/context/base/mkiv/status-lua.pdf diff --git a/tex/context/base/mkiv/task-ini.lua b/tex/context/base/mkiv/task-ini.lua index f334fcb6a..202b38ede 100644 --- a/tex/context/base/mkiv/task-ini.lua +++ b/tex/context/base/mkiv/task-ini.lua @@ -99,7 +99,7 @@ appendaction("math", "normalizers", "noads.handlers.relocate", nil, "no appendaction("math", "normalizers", "noads.handlers.families", nil, "nohead") -- always on appendaction("math", "normalizers", "noads.handlers.render", nil, "nohead") -- always on -appendaction("math", "normalizers", "noads.handlers.collapse", nil, "nohead") -- * first-- always on +appendaction("math", "normalizers", "noads.handlers.collapse", nil, "nohead") -- disabled appendaction("math", "normalizers", "noads.handlers.fixscripts",nil, "nohead") -- * first-- always on appendaction("math", "normalizers", "noads.handlers.domains", nil, "nohead") -- * last -- disabled appendaction("math", "normalizers", "noads.handlers.autofences",nil, "nohead") -- disabled @@ -218,6 +218,7 @@ disableaction("contributers","nodes.handlers.textbackgrounds") disableaction("math", "noads.handlers.showtree") disableaction("math", "noads.handlers.tags") disableaction("math", "noads.handlers.italics") +disableaction("math", "noads.handlers.collapse") disableaction("math", "noads.handlers.kernpairs") disableaction("math", "noads.handlers.domains") disableaction("math", "noads.handlers.classes") diff --git a/tex/context/base/mkiv/typo-dub.lua b/tex/context/base/mkiv/typo-dub.lua index a914b63ba..d85caa078 100644 --- a/tex/context/base/mkiv/typo-dub.lua +++ b/tex/context/base/mkiv/typo-dub.lua @@ -240,7 +240,7 @@ end -- local space = { char = 0x0020, direction = "ws", original = "ws" } -- local lre = { char = 0x202A, direction = "lre", original = "lre" } --- local lre = { char = 0x202B, direction = "rle", original = "rle" } +-- local rle = { char = 0x202B, direction = "rle", original = "rle" } -- local pdf = { char = 0x202C, direction = "pdf", original = "pdf" } -- local object = { char = 0xFFFC, direction = "on", original = "on" } -- diff --git a/tex/context/base/mkiv/typo-duc.lua b/tex/context/base/mkiv/typo-duc.lua index c1b4d66cb..53be51aea 100644 --- a/tex/context/base/mkiv/typo-duc.lua +++ b/tex/context/base/mkiv/typo-duc.lua @@ -62,8 +62,8 @@ local tonode = nuts.tonode local getnext = nuts.getnext local getid = nuts.getid local getsubtype = nuts.getsubtype -local getchar = nuts.getchar local getlist = nuts.getlist +local getchar = nuts.getchar local getattr = nuts.getattr local getprop = nuts.getprop local getdir = nuts.getdir @@ -73,7 +73,7 @@ local setchar = nuts.setchar local setdir = nuts.setdir local setattrlist = nuts.setattrlist -local properties = nodes.properties +local properties = nodes.properties.data local remove_node = nuts.remove local insert_node_after = nuts.insert_after @@ -244,26 +244,28 @@ end local mt_space = { __index = { char = 0x0020, direction = "ws", original = "ws", level = 0 } } local mt_lre = { __index = { char = 0x202A, direction = "lre", original = "lre", level = 0 } } -local mt_lre = { __index = { char = 0x202B, direction = "rle", original = "rle", level = 0 } } +local mt_rle = { __index = { char = 0x202B, direction = "rle", original = "rle", level = 0 } } local mt_pdf = { __index = { char = 0x202C, direction = "pdf", original = "pdf", level = 0 } } local mt_object = { __index = { char = 0xFFFC, direction = "on", original = "on", level = 0 } } +local stack = table.setmetatableindex("table") -- shared +local list = { } -- shared + local function build_list(head) -- todo: store node pointer ... saves loop -- P1 local current = head - local list = { } local size = 0 while current do size = size + 1 local id = getid(current) local p = properties[current] - local t if p and p.directions then local skip = 0 local last = id current = getnext(current) while current do local id = getid(current) + local p = properties[current] if p and p.directions then skip = skip + 1 last = id @@ -273,35 +275,29 @@ local function build_list(head) -- todo: store node pointer ... saves loop end end if id == last then -- the start id - t = { skip = skip, id = id } + list[size] = setmetatable({ skip = skip, id = id },mt_object) else - t = { skip = skip, id = id, last = last } + list[size] = setmetatable({ skip = skip, id = id, last = last },mt_object) end - setmetatable(t,mt_object) elseif id == glyph_code then - local chr = getchar(current) - local dir = directiondata[chr] - t = { char = chr, direction = dir, original = dir, level = 0 } - current = getnext(current) + local chr = getchar(current) + local dir = directiondata[chr] + list[size] = { char = chr, direction = dir, original = dir, level = 0 } + current = getnext(current) -- if not list[dir] then list[dir] = true end -- not faster when we check for usage elseif id == glue_code then -- and how about kern - t = { } - setmetatable(t,mt_space) - current = getnext(current) + list[size] = setmetatable({ },mt_space) + current = getnext(current) elseif id == dir_code then local dir = getdir(current) if dir == "+TLT" then - t = { } - setmetatable(t,mt_lre) + list[size] = setmetatable({ },mt_lre) elseif dir == "+TRT" then - t = { } - setmetatable(t,mt_rle) + list[size] = setmetatable({ },mt_rle) elseif dir == "-TLT" or dir == "-TRT" then - t = { } - setmetatable(t,mt_pdf) + list[size] = setmetatable({ },mt_pdf) else - t = { id = id } - setmetatable(t,mt_object) + list[size] = setmetatable({ id = id },mt_object) end current = getnext(current) elseif id == math_code then @@ -313,8 +309,7 @@ local function build_list(head) -- todo: store node pointer ... saves loop end skip = skip + 1 current = getnext(current) - t = { id = id, skip = skip } - setmetatable(t,mt_object) + list[size] = setmetatable({ id = id, skip = skip },mt_object) else local skip = 0 local last = id @@ -329,16 +324,12 @@ local function build_list(head) -- todo: store node pointer ... saves loop break end end - if skip == 0 then - t = { id = id } - elseif id == last then -- the start id - t = { id = id, skip = skip } + if id == last then -- the start id + list[size] = setmetatable({ id = id, skip = skip },mt_object) else - t = { id = id, skip = skip, last = last } + list[size] = setmetatable({ id = id, skip = skip, last = last },mt_object) end - setmetatable(t,mt_object) end - list[size] = t end return list, size end @@ -359,8 +350,8 @@ end local function resolve_fences(list,size,start,limit) -- N0: funny effects, not always better, so it's an option - local stack = { } - local top = 0 + -- local stack = { } + local nofstack = 0 for i=start,limit do local entry = list[i] if entry.direction == "on" then @@ -371,15 +362,18 @@ local function resolve_fences(list,size,start,limit) entry.mirror = mirror entry.class = class if class == "open" then - top = top + 1 - stack[top] = { mirror, i, false } - elseif top == 0 then + nofstack = nofstack + 1 + local stacktop = stack[nofstack] + stacktop[1] = mirror + stacktop[2] = i + stacktop[3] = false -- not used + elseif nofstack == 0 then -- skip elseif class == "close" then - while top > 0 do - local s = stack[top] - if s[1] == char then - local open = s[2] + while nofstack > 0 do + local stacktop = stack[nofstack] + if stacktop[1] == char then + local open = stacktop[2] local close = i list[open ].paired = close list[close].paired = open @@ -387,7 +381,7 @@ local function resolve_fences(list,size,start,limit) else -- do we mirror or not end - top = top - 1 + nofstack = nofstack - 1 end end end @@ -438,7 +432,7 @@ local function resolve_explicit(list,size,baselevel) -- X1 local level = baselevel local override = "on" - local stack = { } + -- local stack = { } local nofstack = 0 for i=1,size do local entry = list[i] @@ -447,7 +441,9 @@ local function resolve_explicit(list,size,baselevel) if direction == "rle" then if nofstack < maximum_stack then nofstack = nofstack + 1 - stack[nofstack] = { level, override } + local stacktop = stack[nofstack] + stacktop[1] = level + stacktop[2] = override level = level + (level % 2 == 1 and 2 or 1) -- least_greater_odd(level) override = "on" entry.level = level @@ -460,7 +456,9 @@ local function resolve_explicit(list,size,baselevel) elseif direction == "lre" then if nofstack < maximum_stack then nofstack = nofstack + 1 - stack[nofstack] = { level, override } + local stacktop = stack[nofstack] + stacktop[1] = level + stacktop[2] = override level = level + (level % 2 == 1 and 1 or 2) -- least_greater_even(level) override = "on" entry.level = level @@ -473,7 +471,9 @@ local function resolve_explicit(list,size,baselevel) elseif direction == "rlo" then if nofstack < maximum_stack then nofstack = nofstack + 1 - stack[nofstack] = { level, override } + local stacktop = stack[nofstack] + stacktop[1] = level + stacktop[2] = override level = level + (level % 2 == 1 and 2 or 1) -- least_greater_odd(level) override = "r" entry.level = level @@ -486,7 +486,9 @@ local function resolve_explicit(list,size,baselevel) elseif direction == "lro" then if nofstack < maximum_stack then nofstack = nofstack + 1 - stack[nofstack] = { level, override } + local stacktop = stack[nofstack] + stacktop[1] = level + stacktop[2] = override level = level + (level % 2 == 1 and 1 or 2) -- least_greater_even(level) override = "l" entry.level = level @@ -499,9 +501,9 @@ local function resolve_explicit(list,size,baselevel) elseif direction == "pdf" then if nofstack < maximum_stack then local stacktop = stack[nofstack] - nofstack = nofstack - 1 level = stacktop[1] override = stacktop[2] + nofstack = nofstack - 1 entry.level = level entry.direction = "bn" entry.remove = true @@ -568,6 +570,7 @@ local function resolve_weak(list,size,start,limit,orderbefore,orderafter) -- W4: make separators number -- if list.es or list.cs then -- skip +-- if false then if false then for i=start+1,limit-1 do local entry = list[i] @@ -1012,7 +1015,7 @@ local function process(head) report_directions("after : %s",show_list(list,size,"direction")) report_directions("result : %s",show_done(list,size)) end - head, done = apply_to_list(list,size,head,pardir) + local head, done = apply_to_list(list,size,head,pardir) return tonode(head), done end diff --git a/tex/context/base/mkiv/util-lua.lua b/tex/context/base/mkiv/util-lua.lua index b3346006c..6481435bd 100644 --- a/tex/context/base/mkiv/util-lua.lua +++ b/tex/context/base/mkiv/util-lua.lua @@ -10,15 +10,17 @@ if not modules then modules = { } end modules ['util-lua'] = { -- we will remove the 5.1 code some day soon local rep, sub, byte, dump, format = string.rep, string.sub, string.byte, string.dump, string.format -local load, loadfile, type = load, loadfile, type +local load, loadfile, type, collectgarbage = load, loadfile, type, collectgarbage utilities = utilities or {} utilities.lua = utilities.lua or { } local luautilities = utilities.lua local report_lua = logs.reporter("system","lua") +local report_mem = logs.reporter("system","lua memory") local tracestripping = false +local tracememory = false local forcestupidcompile = true -- use internal bytecode compiler luautilities.stripcode = true -- support stripping when asked for luautilities.alwaysstripcode = false -- saves 1 meg on 7 meg compressed format file (2012.08.12) @@ -176,3 +178,24 @@ setmetatable(finalizers, { function luautilities.registerfinalizer(f) finalizers[#finalizers+1] = f end + +function luautilities.checkmemory(previous,threshold,trace) -- threshold in MB + local current = collectgarbage("count") + if previous then + local checked = (threshold or 64)*1024 + local delta = current - previous + if current - previous > checked then + collectgarbage("collect") + local afterwards = collectgarbage("count") + if trace or tracememory then + report_mem("previous %i MB, current %i MB, delta %i MB, threshold %i MB, afterwards %i MB", + previous/1024,current/1024,delta/1024,threshold,afterwards) + end + return afterwards + elseif trace or tracememory then + report_mem("previous %i MB, current %i MB, delta %i MB, threshold %i MB", + previous/1024,current/1024,delta/1024,threshold) + end + end + return current +end diff --git a/tex/context/interface/mkiv/context-en.xml b/tex/context/interface/mkiv/context-en.xml index a236b893f..b5137bb50 100644 --- a/tex/context/interface/mkiv/context-en.xml +++ b/tex/context/interface/mkiv/context-en.xml @@ -29578,6 +29578,9 @@ <cd:parameter name="interaction"> <cd:constant default="yes" type="start"/> <cd:constant type="stop"/> + <cd:constant type="number"/> + <cd:constant type="text"/> + <cd:constant type="page"/> <cd:constant type="all"/> </cd:parameter> <cd:parameter name="alternative"> @@ -31260,6 +31263,20 @@ </cd:keywords> </cd:arguments> </cd:command> + <cd:command category="bibliography" file="publ-ini.mkiv" level="document" name="usecitation"> + <cd:arguments> + <cd:keywords> + <cd:constant type="cd:reference"/> + </cd:keywords> + </cd:arguments> + </cd:command> + <cd:command category="bibliography" file="publ-ini.mkiv" level="document" name="usecitation" variant="direct"> + <cd:arguments> + <cd:keywords delimiters="braces"> + <cd:constant type="cd:reference"/> + </cd:keywords> + </cd:arguments> + </cd:command> <cd:command category="bibliography" file="publ-ini.mkiv" level="document" name="nocite"> <cd:arguments> <cd:keywords> diff --git a/tex/context/interface/mkiv/i-context.pdf b/tex/context/interface/mkiv/i-context.pdf Binary files differindex 65537fd65..29515f75c 100644 --- a/tex/context/interface/mkiv/i-context.pdf +++ b/tex/context/interface/mkiv/i-context.pdf diff --git a/tex/context/interface/mkiv/i-publication.xml b/tex/context/interface/mkiv/i-publication.xml index d981c4acd..4da08d270 100644 --- a/tex/context/interface/mkiv/i-publication.xml +++ b/tex/context/interface/mkiv/i-publication.xml @@ -77,6 +77,9 @@ <cd:parameter name="interaction"> <cd:constant type="start" default="yes"/> <cd:constant type="stop"/> + <cd:constant type="number"/> + <cd:constant type="text"/> + <cd:constant type="page"/> <cd:constant type="all"/> </cd:parameter> <cd:parameter name="alternative"> @@ -670,6 +673,18 @@ </cd:arguments> </cd:command> + <cd:command name="usecitation" category="bibliography" level="document" file="publ-ini.mkiv"> + <cd:arguments> + <cd:resolve name="keyword-reference"/> + </cd:arguments> + </cd:command> + + <cd:command name="usecitation" variant="direct" category="bibliography" level="document" file="publ-ini.mkiv"> + <cd:arguments> + <cd:resolve name="argument-reference"/> + </cd:arguments> + </cd:command> + <cd:command name="nocite" category="bibliography" level="document" file="publ-ini.mkiv"> <cd:arguments> <cd:resolve name="keyword-reference"/> diff --git a/tex/context/interface/mkiv/i-readme.pdf b/tex/context/interface/mkiv/i-readme.pdf Binary files differindex 32bd1a8b9..fdb9f9f2c 100644 --- a/tex/context/interface/mkiv/i-readme.pdf +++ b/tex/context/interface/mkiv/i-readme.pdf diff --git a/tex/context/modules/mkiv/m-scite.mkiv b/tex/context/modules/mkiv/m-scite.mkiv index 516f3ce54..c381475d0 100644 --- a/tex/context/modules/mkiv/m-scite.mkiv +++ b/tex/context/modules/mkiv/m-scite.mkiv @@ -45,7 +45,7 @@ -- todo: hook into the pretty print code -- todo: a simple catcode regime with only \ { } -local gsub, sub, find = string.gsub, string.sub, string.find +local gsub, sub, find, lower = string.gsub, string.sub, string.find, string.lower local concat = table.concat local formatters = string.formatters local lpegmatch = lpeg.match @@ -167,53 +167,126 @@ function scite.installcommands() context(exportcolors()) end -local p1 = lpeg.tsplitat(lpeg.patterns.newline) -local p2 = lpeg.P("\\slxS ")^1 -local p3 = lpeg.Cs((lpeg.Cp() * lpeg.P(" ") * lpeg.Cp() / function(b,e) return "\\slxF{" .. (e-b-1) .. "}" end - + (1-lpeg.P(" "))^1)^0) +local function slxF(b,e) + local d = (e - b)/6 + if d > 0 then + return "\\slxF{" .. d .. "}" + else + return " " + end +end --- slxb could have a \hskip so then we can replace the slxS's +local p1 = lpeg.tsplitat(lpeg.patterns.newline) +local p2 = lpeg.P("\\slxS ") +local p3 = p2^1 +local p4 = lpeg.Cs( ( (lpeg.Cp() * p2 * p2^1 * lpeg.Cp()) / slxF + lpeg.P(1) )^0 ) local function indent(str) local l = lpegmatch(p1,str) for i=1,#l do local s = l[i] if #s > 0 then - local n = lpegmatch(p2,s) + local n = lpegmatch(p3,s) if n then - n = (n-1)/6 + n = (n-1)/6 -- length of "\\slxS " else n = 0 end -s = lpegmatch(p3,s) -- can be combined + if n > 0 then + s = sub(s,n*6+1) + end + s = lpegmatch(p4,s) -- can be combined l[i] = f_hanging(n,s) -- "\\slxb{%s}%s\\slxe " --- print(">",s) --- print("<",l[i]) end end return concat(l,"\n") end +local assignbuffer = buffers.assign +local getcontent = buffers.getcontent +local loaddata = io.loaddata +local loadedlexers = scite.loadedlexers + local function lexdata(data,lexname) - local styled = exportstyled(scite.loadedlexers[lexname],data or "") + local styled = exportstyled(loadedlexers[lexname],data or "") styled = indent(styled) - buffers.assign("lex",styled) + assignbuffer("lex",styled) end scite.lexdata = lexdata function scite.lexbuffer(name,lexname) - lexdata(buffers.getcontent(name) or "",lexname or "tex") + lexdata(getcontent(name) or "",lexname or "tex") end function scite.lexfile(filename,lexname) - lexdata(io.loaddata(filename) or "",lexname or file.suffix(filename)) + lexdata(loaddata(filename) or "",lexname or file.suffix(filename)) end --- html output +interfaces.implement { + name = "scitelexfile", + arguments = "string", + actions = scite.lexfile, +} + +interfaces.implement { + name = "scitelexbuffer", + arguments = { "string", "string" }, + actions = scite.lexbuffer, +} + +interfaces.implement { + name = "sciteinstallcommands", + actions = scite.installcommands, +} + +local startInlineScite = context.startInlineScite +local stopInlineScite = context.stopInlineScite +local startDisplayScite = context.startDisplayScite +local stopDisplayScite = context.stopDisplayScite + +local lexdata = buffers.scite.lexdata + +local handler = visualizers.newhandler { + startinline = function(settings) startInlineScite(settings.name or "") end, + stopinline = function(settings) stopInlineScite() end, + startdisplay = function(settings) startDisplayScite(settings.name or "") end, + stopdisplay = function(settings) stopDisplayScite() end, +} + +local knownlexers = scite.knownlexers + +local parser = function(content,specification) + local method = lower(specification.method) + lexdata(content,knownlexers[method] or method) +end + +local visualizer = { + handler = handler, + parser = parser, +} + +visualizers.register("cld", visualizer) +visualizers.register("tex", visualizer) +visualizers.register("lua", visualizer) +visualizers.register("mps", visualizer) +visualizers.register("xml", visualizer) +visualizers.register("bibtex",visualizer) +visualizers.register("btx", visualizer) +visualizers.register("web", visualizer) +visualizers.register("cpp", visualizer) +-----------.register("sql", visualizer) \stopluacode +\definetyping[TEX][option=cld] +\definetyping[LUA][option=lua] +\definetyping[BTX][option=bibtex] +\definetyping[MPS][option=mps] +\definetyping[MP] [option=mps] +\definetyping[CPP][option=web] +\definetyping[WEB][option=web] + % This is a preliminary interface. \unprotect @@ -221,22 +294,34 @@ end \newdimen\scitespaceskip \unexpanded\def\buff_scite_slxb#1% - {%\begingroup - \hangindent\numexpr#1+2\relax\scitespaceskip + {\hangindent\numexpr#1+2\relax\scitespaceskip + \hskip#1\scitespaceskip \hangafter 1\relax} \unexpanded\def\buff_scite_slxe - {\par - }%\endgroup} - -\unexpanded\def\installscitecommands - {\ctxlua{buffers.scite.installcommands()}% - \let\installscitecommands\relax - \scitespaceskip\fontcharwd\font`0\relax - \unexpanded\def\slxS{\hskip\scitespaceskip}% - \unexpanded\def\slxF##1{\hskip##1\scitespaceskip\relax}% + {\par} + +\unexpanded\def\buff_scite_slxs {\hskip\scitespaceskip} +\unexpanded\def\buff_scite_slxf#1{\hskip#1\scitespaceskip\relax} + +\unexpanded\def\installscitecommandsinline + {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr + \let\slxb\gobbleoneargument + \let\slxe\relax + \let\installscitecommandsinline\relax} + +\unexpanded\def\installscitecommandsdisplay + {\scitespaceskip\interwordspace % \fontcharwd\font`0\relax % brrrrr \let\slxb\buff_scite_slxb - \let\slxe\buff_scite_slxe} + \let\slxe\buff_scite_slxe + \let\installscitecommandsdisplay\relax} + +\clf_sciteinstallcommands + +\installscitecommandsinline + +\let\slxS\buff_scite_slxs +\let\slxF\buff_scite_slxf \unexpanded\def\startscite {\begingroup @@ -255,9 +340,10 @@ end \unexpanded\def\module_scite_file[#1]% {\begingroup - \ctxlua{buffers.scite.lexfile("#1")}% - \installscitecommands + \setcatcodetable\ctxcatcodes % needed in xml + \clf_scitelexfile{#1}% \tt + \installscitecommandsdisplay \dontcomplain \raggedright \startcontextcode @@ -272,16 +358,16 @@ end \unexpanded\def\module_scite_buffer[#1][#2]% {\begingroup + \setcatcodetable\ctxcatcodes % needed in xml \ifsecondargument - \ctxlua{buffers.scite.lexbuffer("#2","#1")}% + \clf_scitelexbuffer{#2}{#1}% \else - \ctxlua{buffers.scite.lexbuffer("#1","tex")}% + \clf_scitelexbuffer{#1}{tex}% \fi - \installscitecommands \tt + \installscitecommandsdisplay \dontcomplain \raggedright - \setcatcodetable\ctxcatcodes % needed in xml \startlines \getbuffer[lex]% \stoplines @@ -290,21 +376,49 @@ end \unexpanded\def\sciteinlinebuffer {\dodoubleargument\module_scite_buffer_inline} -\unexpanded\def\module_scite_buffer[#1][#2]% +\unexpanded\def\module_scite_buffer_inline[#1][#2]% {\dontleavehmode \begingroup + \setcatcodetable\ctxcatcodes % needed in xml \ifsecondargument - \ctxlua{buffers.scite.lexbuffer("#2","#1")}% + \clf_scitelexbuffer{#2}{#1}% \else - \ctxlua{buffers.scite.lexbuffer("#1","tex")}% + \clf_scitelexbuffer{#1}{tex}% \fi - \installscitecommands \tt + \installscitecommandsinline \dontcomplain - \setcatcodetable\ctxcatcodes % needed in xml \getbuffer[lex]% \endgroup} +\unexpanded\def\startInlineScite#1% + {\dontleavehmode + \begingroup + \setcatcodetable\ctxcatcodes % needed in xml + \tt + \installscitecommandsinline + \dontcomplain + \ignorespaces + \clf_getbuffercontent{lex}% much faster than getbuffer + \removeunwantedspaces} + +\unexpanded\def\stopInlineScite + {\endgroup} + +\unexpanded\def\startDisplayScite#1% + {\begingroup + \setcatcodetable\ctxcatcodes % needed in xml + \tt + \installscitecommandsdisplay + \dontcomplain + \raggedright + \startlines + \clf_getbuffer{lex}} + +\unexpanded\def\stopDisplayScite + {\stoplines + \endgroup} + \protect \continueifinputfile{m-scite.mkiv} diff --git a/tex/context/modules/mkiv/s-math-ligatures.lua b/tex/context/modules/mkiv/s-math-ligatures.lua index 16fe0c4bc..5c888c088 100644 --- a/tex/context/modules/mkiv/s-math-ligatures.lua +++ b/tex/context/modules/mkiv/s-math-ligatures.lua @@ -17,7 +17,7 @@ local uformat = string.formatters["%U"] function moduledata.math.ligatures.showlist(specification) -- specification = interfaces.checkedspecification(specification) - local function setlist(unicode,list,start,v) + local function setlist(unicode,list,start,v,how) if list[start] ~= 0x20 then local t, u = { }, { } for i=start,#list do @@ -25,6 +25,7 @@ function moduledata.math.ligatures.showlist(specification) t[#t+1] = utfchar(li) u[#u+1] = uformat(li) end + context.NC() context(how) context.NC() context("%U",unicode) context.NC() context("%c",unicode) context.NC() context("% t",u) @@ -53,18 +54,18 @@ function moduledata.math.ligatures.showlist(specification) end end - context.starttabulate { "|T|m|T|T|m|pl|" } + context.starttabulate { "|T|T|m|T|T|m|pl|" } for unicode, v in table.sortedhash(characters.data) do local vs = v.specials if vs and #vs > 2 then local kind = vs[1] if (v.mathclass or v.mathspec) and (kind == "char" or kind == "compat") then - setlist(unicode,vs,2,v) + setlist(unicode,vs,2,v,"sp") end end local ml = v.mathlist if ml then - setlist(unicode,ml,1,v) + setlist(unicode,ml,1,v,"ml") end end context.stoptabulate() diff --git a/tex/context/modules/mkiv/x-setups-basics.mkiv b/tex/context/modules/mkiv/x-setups-basics.mkiv index e3d3d37be..35314bc9c 100644 --- a/tex/context/modules/mkiv/x-setups-basics.mkiv +++ b/tex/context/modules/mkiv/x-setups-basics.mkiv @@ -645,7 +645,8 @@ \setfalse\c_cmd_doing_line \global\c_cmd_current_argument\zerocount %\blank[\v!line] % packed mode (we could do \startunpacked ...) - \godown[.75\lineheight] +% \godown[.75\lineheight] + \blank[\v!samepage,\v!medium] \switchtobodyfont[\v!small] \ignorespaces\xmlfilter{#1}{/arguments/text()}\endgraf \egroup @@ -663,7 +664,8 @@ \startxmlsetups xml:setups:typeset:instances \ifx\m_cmd_instance\empty \xmldoif{#1}{/instances} { - \godown[.75\lineheight] +% \godown[.75\lineheight] + \blank[\v!samepage,\v!medium] {\ttbf instances:} \enspace \xmlfilter{#1}{/instances/(constant|resolve)/command(xml:setups:instance)} @@ -822,7 +824,8 @@ \unknown } \cmd_column_one_stop - \godown[.75\lineheight] +% \godown[.75\lineheight] + \blank[\v!samepage,\v!medium] \ignorespaces \fi \stopxmlsetups @@ -839,7 +842,8 @@ \ignorespaces \xmlflush{#1} \cmd_column_one_stop - \godown[.75\lineheight] +% \godown[.75\lineheight] + \blank[\v!samepage,\v!medium] \ignorespaces \fi \stopxmlsetups @@ -928,7 +932,8 @@ {\cmd_column_one_start \cmd_internal_value{#1}% \cmd_column_one_stop - \godown[.75\lineheight] +% \godown[.75\lineheight] + \blank[\v!samepage,\v!medium] \ignorespaces} %D Auxiliary. @@ -1060,7 +1065,7 @@ \stopxmlsetups \startxmlsetups xml:setups:parameters:values - \blank[\v!big] + \blank[\v!big,\v!samepage] \expanded { \parametercommand { \currentsetupparametercommand diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 8621d19a3..08fbc5974 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 08/11/17 14:00:42 +-- merge date : 08/13/17 16:37:50 do -- begin closure to overcome local limits and interference @@ -15514,7 +15514,7 @@ function gposhandlers.single(f,fontdata,lookupid,lookupoffset,offset,glyphs,nofg local value=readposition(f,format,tableoffset,getdelta) local coverage=readcoverage(f,tableoffset+coverage) for index,newindex in next,coverage do - coverage[index]=value + coverage[index]=value end return { format="single", @@ -19458,6 +19458,7 @@ function readers.compact(data) lookup.merged=true end elseif nofsteps==1 then + local kern=kerned if kind=="gpos_single" then if compact_singles then kerned=kerned+checkkerns(lookup) @@ -19467,6 +19468,8 @@ function readers.compact(data) kerned=kerned+checkpairs(lookup) end end + if kern~=kerned then + end end end else @@ -19523,7 +19526,8 @@ local function checkflags(sequence,resources) or (markclass and c=="mark" and not markclass[k]) or c==skipligature or c==skipbase - t[k]=v or false + or false + t[k]=v return v end) else @@ -19766,7 +19770,25 @@ registerdirective("fonts.otf.loader.cleanup",function(v) cleanup=tonumber(v) or registerdirective("fonts.otf.loader.force",function(v) forceload=v end) registerdirective("fonts.otf.loader.syncspace",function(v) syncspace=v end) registerdirective("fonts.otf.loader.forcenotdef",function(v) forcenotdef=v end) -registerotfenhancer("check extra features",function() end) +registerotfenhancer("check extra features",function() end) +local checkmemory=utilities.lua and utilities.lua.checkmemory +local threshold=100 +local tracememory=false +registertracker("fonts.otf.loader.memory",function(v) tracememory=v end) +if not checkmemory then + local collectgarbage=collectgarbage + checkmemory=function(previous,threshold) + local current=collectgarbage("count") + if previous then + local checked=(threshold or 64)*1024 + if current-previous>checked then + collectgarbage("collect") + current=collectgarbage("count") + end + end + return current + end +end function otf.load(filename,sub,instance) local base=file.basename(file.removesuffix(filename)) local name=file.removesuffix(base) @@ -19795,9 +19817,13 @@ function otf.load(filename,sub,instance) starttiming(otfreaders) data=otfreaders.loadfont(filename,sub or 1,instance) if data then + local used=checkmemory() local resources=data.resources local svgshapes=resources.svgshapes local sbixshapes=resources.sbixshapes + if cleanup==0 then + checkmemory(used,threshold,tracememory) + end if svgshapes then resources.svgshapes=nil if otf.svgenabled then @@ -19811,6 +19837,11 @@ function otf.load(filename,sub,instance) timestamp=timestamp, } end + if cleanup>1 then + collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) + end end if sbixshapes then resources.sbixshapes=nil @@ -19825,17 +19856,30 @@ function otf.load(filename,sub,instance) timestamp=timestamp, } end + if cleanup>1 then + collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) + end end otfreaders.compact(data) + if cleanup==0 then + checkmemory(used,threshold,tracememory) + end otfreaders.rehash(data,"unicodes") otfreaders.addunicodetable(data) otfreaders.extend(data) + if cleanup==0 then + checkmemory(used,threshold,tracememory) + end otfreaders.pack(data) report_otf("loading done") report_otf("saving %a in cache",filename) data=containers.write(otf.cache,hash,data) if cleanup>1 then collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) end stoptiming(otfreaders) if elapsedtime then @@ -19843,10 +19887,14 @@ function otf.load(filename,sub,instance) end if cleanup>3 then collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) end data=containers.read(otf.cache,hash) if cleanup>2 then collectgarbage("collect") + else + checkmemory(used,threshold,tracememory) end else data=nil @@ -20971,7 +21019,7 @@ function injections.getligaindex(n,default) end return default end -function injections.setcursive(start,nxt,factor,rlmode,exit,entry,tfmstart,tfmnext) +function injections.setcursive(start,nxt,factor,rlmode,exit,entry,tfmstart,tfmnext,r2lflag) local dx=factor*(exit[1]-entry[1]) local dy=-factor*(exit[2]-entry[2]) local ws=tfmstart.width @@ -21024,7 +21072,7 @@ function injections.setcursive(start,nxt,factor,rlmode,exit,entry,tfmstart,tfmne end return dx,dy,nofregisteredcursives end -function injections.setposition(current,factor,rlmode,r2lflag,spec,injection) +function injections.setposition(kind,current,factor,rlmode,spec,injection) local x=factor*spec[1] local y=factor*spec[2] local w=factor*spec[3] @@ -21122,6 +21170,24 @@ function injections.setmove(current,factor,rlmode,x,injection) if not injection then injection="injections" end +if rlmode and rlmode<0 then + if p then + local i=p[injection] + if i then + i.rightkern=dx+(i.rightkern or 0) + else + p[injection]={ + rightkern=dx, + } + end + else + properties[current]={ + [injection]={ + rightkern=dx, + }, + } + end +else if p then local i=p[injection] if i then @@ -21138,6 +21204,7 @@ function injections.setmove(current,factor,rlmode,x,injection) }, } end +end return dx,nofregisteredkerns else return 0,0 @@ -21461,10 +21528,15 @@ local function inject_positions_only(head,where) setoffsets(current,false,yoffset) end local leftkern=i.leftkern + local rightkern=i.rightkern if leftkern and leftkern~=0 then +if rightkern and leftkern==-rightkern then + setoffsets(current,leftkern,false) + rightkern=0 +else head=insert_node_before(head,current,fontkern(leftkern)) +end end - local rightkern=i.rightkern if rightkern and rightkern~=0 then insert_node_after(head,current,fontkern(rightkern)) end @@ -21662,8 +21734,6 @@ local function inject_everything(head,where) local hasmarks=nofregisteredmarks>0 local current=head local last=nil - local font=font - local markdata=nil local prev=nil local next=nil local prevdisc=nil @@ -21812,10 +21882,15 @@ local function inject_everything(head,where) end end local leftkern=i.leftkern + local rightkern=i.rightkern if leftkern and leftkern~=0 then +if rightkern and leftkern==-rightkern then + setoffsets(current,leftkern,false) + rightkern=0 +else head=insert_node_before(head,current,fontkern(leftkern)) +end end - local rightkern=i.rightkern if rightkern and rightkern~=0 then insert_node_after(head,current,fontkern(rightkern)) end @@ -22699,6 +22774,7 @@ local usesfont=nuts.uses_font local insert_node_after=nuts.insert_after local copy_node=nuts.copy local copy_node_list=nuts.copy_list +local remove_node=nuts.remove local find_node_tail=nuts.tail local flush_node_list=nuts.flush_list local flush_node=nuts.flush_node @@ -22920,7 +22996,7 @@ local function markstoligature(head,start,stop,char) return head,base end end -local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfound) +local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfound,hasmarks) if getattr(start,a_noligature)==1 then return head,start end @@ -22944,7 +23020,7 @@ local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfou set_components(base,comp) setlink(prev,base,next) if not discfound then - local deletemarks=not skiphash + local deletemarks=not skiphash or hasmarks local components=start local baseindex=0 local componentindex=0 @@ -22955,7 +23031,7 @@ local function toligature(head,start,stop,char,dataset,sequence,skiphash,discfou if not marks[char] then baseindex=baseindex+componentindex componentindex=count_components(start,marks) - elseif not deletemarks then + elseif not deletemarks then setligaindex(start,baseindex+getligaindex(start,componentindex)) if trace_marks then logwarning("%s: keep mark %s, gets index %s",pref(dataset,sequence),gref(char),getligaindex(start)) @@ -23157,9 +23233,10 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip else end end - else + else local discfound=false local lastdisc=nil + local hasmarks=marks[startchar] while current do local char,id=ischar(current,currentfont) if char then @@ -23172,6 +23249,9 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip discfound=lastdisc lastdisc=nil end + if marks[char] then + hasmarks=true + end stop=current ligature=lg current=getnext(current) @@ -23182,13 +23262,16 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip elseif char==false then break elseif id==disc_code then - local replace=getfield(current,"replace") + local replace=getfield(current,"replace") if replace then while replace do local char,id=ischar(replace,currentfont) if char then local lg=ligature[char] if lg then + if marks[char] then + hasmarks=true + end ligature=lg replace=getnext(replace) else @@ -23211,10 +23294,10 @@ function handlers.gsub_ligature(head,start,dataset,sequence,ligature,rlmode,skip if stop then if trace_ligatures then local stopchar=getchar(stop) - head,start=toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound) + head,start=toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound,hasmarks) logprocess("%s: replacing %s upto %s by ligature %s case 2",pref(dataset,sequence),gref(startchar),gref(stopchar),gref(lig)) else - head,start=toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound) + head,start=toligature(head,start,stop,lig,dataset,sequence,skiphash,discfound,hasmarks) end else resetinjection(start) @@ -23233,7 +23316,7 @@ function handlers.gpos_single(head,start,dataset,sequence,kerns,rlmode,skiphash, local startchar=getchar(start) local format=step.format if format=="single" or type(kerns)=="table" then - local dx,dy,w,h=setposition(start,factor,rlmode,sequence.flags[4],kerns,injection) + local dx,dy,w,h=setposition(0,start,factor,rlmode,kerns,injection) if trace_kerns then logprocess("%s: shifting single %s by %s xy (%p,%p) and wh (%p,%p)",pref(dataset,sequence),gref(startchar),format,dx,dy,w,h) end @@ -23243,7 +23326,7 @@ function handlers.gpos_single(head,start,dataset,sequence,kerns,rlmode,skiphash, logprocess("%s: shifting single %s by %s %p",pref(dataset,sequence),gref(startchar),format,k) end end - return head,start,false + return head,start,true end function handlers.gpos_pair(head,start,dataset,sequence,kerns,rlmode,skiphash,step,injection) local snext=getnext(start) @@ -23267,7 +23350,7 @@ function handlers.gpos_pair(head,start,dataset,sequence,kerns,rlmode,skiphash,st local a,b=krn[1],krn[2] if a==true then elseif a then - local x,y,w,h=setposition(start,factor,rlmode,sequence.flags[4],a,injection) + local x,y,w,h=setposition(1,start,factor,rlmode,a,injection) if trace_kerns then local startchar=getchar(start) logprocess("%s: shifting first of pair %s and %s by xy (%p,%p) and wh (%p,%p) as %s",pref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h,injection or "injections") @@ -23276,7 +23359,7 @@ function handlers.gpos_pair(head,start,dataset,sequence,kerns,rlmode,skiphash,st if b==true then start=snext elseif b then - local x,y,w,h=setposition(snext,factor,rlmode,sequence.flags[4],b,injection) + local x,y,w,h=setposition(2,snext,factor,rlmode,b,injection) if trace_kerns then local startchar=getchar(snext) logprocess("%s: shifting second of pair %s and %s by xy (%p,%p) and wh (%p,%p) as %s",pref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h,injection or "injections") @@ -23475,7 +23558,8 @@ function handlers.gpos_cursive(head,start,dataset,sequence,exitanchors,rlmode,sk if entry then entry=entry[2] if entry then - local dx,dy,bound=setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar]) + local r2lflag=sequence.flags[4] + local dx,dy,bound=setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar],r2lflag) if trace_cursive then logprocess("%s: moving %s to %s cursive (%p,%p) using bound %s in %s mode",pref(dataset,sequence),gref(startchar),gref(nextchar),dx,dy,bound,mref(rlmode)) end @@ -23656,6 +23740,7 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup logwarning("%s: no ligatures starting with %s",cref(dataset,sequence,chainindex),gref(startchar)) end else + local hasmarks=marks[startchar] local current=getnext(start) local discfound=false local last=stop @@ -23681,6 +23766,9 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup ligatures=lg last=current nofreplacements=nofreplacements+1 + if marks[char] then + hasmarks=true + end if current==stop then break else @@ -23704,7 +23792,7 @@ function chainprocs.gsub_ligature(head,start,stop,dataset,sequence,currentlookup logprocess("%s: replacing character %s upto %s by ligature %s case 4",cref(dataset,sequence,chainindex),gref(startchar),gref(getchar(stop)),gref(ligature)) end end - head,start=toligature(head,start,stop,ligature,dataset,sequence,skiphash,discfound) + head,start=toligature(head,start,stop,ligature,dataset,sequence,skiphash,discfound,hasmarks) return head,start,true,nofreplacements,discfound elseif trace_bugs then if start==stop then @@ -23728,7 +23816,7 @@ function chainprocs.gpos_single(head,start,stop,dataset,sequence,currentlookup,r if kerns then local format=currentlookup.format if format=="single" then - local dx,dy,w,h=setposition(start,factor,rlmode,sequence.flags[4],kerns) + local dx,dy,w,h=setposition(0,start,factor,rlmode,kerns) if trace_kerns then logprocess("%s: shifting single %s by %s (%p,%p) and correction (%p,%p)",cref(dataset,sequence),gref(startchar),format,dx,dy,w,h) end @@ -23738,6 +23826,7 @@ function chainprocs.gpos_single(head,start,stop,dataset,sequence,currentlookup,r logprocess("%s: shifting single %s by %s %p",cref(dataset,sequence),gref(startchar),format,k) end end + return head,start,true end end return head,start,false @@ -23772,7 +23861,7 @@ function chainprocs.gpos_pair(head,start,stop,dataset,sequence,currentlookup,rlm local a,b=krn[1],krn[2] if a==true then elseif a then - local x,y,w,h=setposition(start,factor,rlmode,sequence.flags[4],a,"injections") + local x,y,w,h=setposition(1,start,factor,rlmode,a,"injections") if trace_kerns then local startchar=getchar(start) logprocess("%s: shifting first of pair %s and %s by (%p,%p) and correction (%p,%p)",cref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h) @@ -23781,7 +23870,7 @@ function chainprocs.gpos_pair(head,start,stop,dataset,sequence,currentlookup,rlm if b==true then start=snext elseif b then - local x,y,w,h=setposition(snext,factor,rlmode,sequence.flags[4],b,"injections") + local x,y,w,h=setposition(2,snext,factor,rlmode,b,"injections") if trace_kerns then local startchar=getchar(start) logprocess("%s: shifting second of pair %s and %s by (%p,%p) and correction (%p,%p)",cref(dataset,sequence),gref(startchar),gref(nextchar),x,y,w,h) @@ -24019,7 +24108,8 @@ function chainprocs.gpos_cursive(head,start,stop,dataset,sequence,currentlookup, if entry then entry=entry[2] if entry then - local dx,dy,bound=setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar]) + local r2lflag=sequence.flags[4] + local dx,dy,bound=setcursive(start,nxt,factor,rlmode,exit,entry,characters[startchar],characters[nextchar],r2lflag) if trace_cursive then logprocess("%s: moving %s to %s cursive (%p,%p) using bound %s in %s mode",pref(dataset,sequence),gref(startchar),gref(nextchar),dx,dy,bound,mref(rlmode)) end @@ -25718,11 +25808,11 @@ do while start do local char,id=ischar(start,font) if char then - local m=merged[char] - if m then - if skiphash and skiphash[char] then - start=getnext(start) - else + if skiphash and skiphash[char] then + start=getnext(start) + else + local m=merged[char] + if m then local a if attr then if getattr(start,0)==attr and (not attribute or getprop(start,a_state)==attribute) then @@ -25753,9 +25843,9 @@ do else start=getnext(start) end + else + start=getnext(start) end - else - start=getnext(start) end elseif char==false or id==glue_code then start=getnext(start) |