diff options
Diffstat (limited to 'tex')
69 files changed, 889 insertions, 771 deletions
diff --git a/tex/context/base/anch-pos.lua b/tex/context/base/anch-pos.lua index 5f46ee5dd..7321d7d99 100644 --- a/tex/context/base/anch-pos.lua +++ b/tex/context/base/anch-pos.lua @@ -269,6 +269,8 @@ commands.setpos = setall -- will become private table (could also become attribute driven but too nasty -- as attributes can bleed e.g. in margin stuff) +-- not much gain in keeping stack (inc/dec instead of insert/remove) + function jobpositions.b_col(tag) tobesaved[tag] = { r = true, @@ -312,7 +314,7 @@ end function jobpositions.b_region(tag) local last = tobesaved[tag] last.x = pdf.h -last.y = pdf.v + last.y = pdf.v last.p = texcount.realpageno insert(regions,tag) region = tag diff --git a/tex/context/base/buff-imp-lua.lua b/tex/context/base/buff-imp-lua.lua index 1147666cc..04e79afba 100644 --- a/tex/context/base/buff-imp-lua.lua +++ b/tex/context/base/buff-imp-lua.lua @@ -6,7 +6,8 @@ if not modules then modules = { } end modules ['buff-imp-lua'] = { license = "see context related readme files" } --- borrowed from scite +-- borrowed from ctx scite lexers +-- add goto/label scanning -- -- depricated: -- @@ -26,9 +27,9 @@ local core = tohash { local base = tohash { "assert", "collectgarbage", "dofile", "error", "loadfile", - "loadstring", "print", "rawget", "rawset", "require", "tonumber", + "loadstring", "load", "print", "rawget", "rawset", "require", "tonumber", "tostring", "type", "_G", "getmetatable", "ipairs", "next", "pairs", - "pcall", "rawequal", "setmetatable", "xpcall", "module", "select", + "pcall", "rawequal", "setmetatable", "xpcall", "module", "select", "goto", } local libraries = { @@ -61,7 +62,7 @@ local libraries = { }, lpeg = tohash{ "print", "match", "locale", "type", "version", "setmaxstack", - "P", "R", "S", "C", "V", "Cs", "Ct", "Cs", "Cp", "Carg", + "P", "R", "S", "C", "V", "Cs", "Ct", "Cs", "Cc", "Cp", "Carg", "Cg", "Cb", "Cmt", "Cf", "B", }, -- bit diff --git a/tex/context/base/buff-imp-tex.lua b/tex/context/base/buff-imp-tex.lua index 29fd8c0c5..9e61de016 100644 --- a/tex/context/base/buff-imp-tex.lua +++ b/tex/context/base/buff-imp-tex.lua @@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['buff-imp-tex'] = { license = "see context related readme files" } +-- needs an update, use mult-low + local P, S, V, patterns = lpeg.P, lpeg.S, lpeg.V, lpeg.patterns local context = context diff --git a/tex/context/base/buff-ini.lua b/tex/context/base/buff-ini.lua index 475d23efe..f76494c43 100644 --- a/tex/context/base/buff-ini.lua +++ b/tex/context/base/buff-ini.lua @@ -6,21 +6,22 @@ if not modules then modules = { } end modules ['buff-ini'] = { license = "see context related readme files" } -local trace_run = false trackers.register("buffers.run", function(v) trace_run = v end) -local trace_grab = false trackers.register("buffers.grab", function(v) trace_grab = v end) -local trace_visualize = false trackers.register("buffers.visualize", function(v) trace_visualize = v end) - -local report_buffers = logs.reporter("buffers","usage") -local report_grabbing = logs.reporter("buffers","grabbing") - -local context, commands = context, commands - local concat = table.concat local type, next, load = type, next, load local sub, format = string.sub, string.format local splitlines, validstring = string.splitlines, string.valid local P, Cs, patterns, lpegmatch = lpeg.P, lpeg.Cs, lpeg.patterns, lpeg.match +local trace_run = false trackers.register("buffers.run", function(v) trace_run = v end) +local trace_grab = false trackers.register("buffers.grab", function(v) trace_grab = v end) +local trace_visualize = false trackers.register("buffers.visualize", function(v) trace_visualize = v end) + +local report_buffers = logs.reporter("buffers","usage") +local report_grabbing = logs.reporter("buffers","grabbing") + +local context = context +local commands = commands + local variables = interfaces.variables local settings_to_array = utilities.parsers.settings_to_array local formatters = string.formatters diff --git a/tex/context/base/buff-ver.lua b/tex/context/base/buff-ver.lua index e327a59dd..e6508d3a5 100644 --- a/tex/context/base/buff-ver.lua +++ b/tex/context/base/buff-ver.lua @@ -10,16 +10,16 @@ if not modules then modules = { } end modules ['buff-ver'] = { -- supposed to use different names for their own variants. -- -- todo: skip=auto +-- +-- todo: update to match context scite lexing -local type, next, rawset, rawget, setmetatable, getmetatable = type, next, rawset, rawget, setmetatable, getmetatable +local type, next, rawset, rawget, setmetatable, getmetatable, tonumber = type, next, rawset, rawget, setmetatable, getmetatable, tonumber local format, lower, upper,match, find, sub = string.format, string.lower, string.upper, string.match, string.find, string.sub local splitlines = string.splitlines local concat = table.concat local C, P, R, S, V, Carg, Cc, Cs = lpeg.C, lpeg.P, lpeg.R, lpeg.S, lpeg.V, lpeg.Carg, lpeg.Cc, lpeg.Cs local patterns, lpegmatch, is_lpeg = lpeg.patterns, lpeg.match, lpeg.is_lpeg -local context, commands = context, commands - local trace_visualize = false trackers.register("buffers.visualize", function(v) trace_visualize = v end) local report_visualizers = logs.reporter("buffers","visualizers") @@ -29,6 +29,9 @@ visualizers = visualizers or { } local specifications = allocate() visualizers.specifications = specifications +local context = context +local commands = commands + local tabtospace = utilities.strings.tabtospace local variables = interfaces.variables local settings_to_array = utilities.parsers.settings_to_array @@ -568,7 +571,7 @@ local function realign(lines,strip) -- "yes", <number> if strip == v_yes then n = math.huge for i=1, #lines do - local spaces = find(lines[i],"%S") + local spaces = find(lines[i],"%S") -- can be lpeg if not spaces then -- empty line elseif spaces == 0 then @@ -592,11 +595,13 @@ local function realign(lines,strip) -- "yes", <number> return lines end +local onlyspaces = S(" \t\f\n\r")^0 * P(-1) + local function getstrip(lines,first,last) local first, last = first or 1, last or #lines for i=first,last do local li = lines[i] - if #li == 0 or find(li,"^%s*$") then + if #li == 0 or lpegmatch(onlyspaces,li) then first = first + 1 else break @@ -604,7 +609,7 @@ local function getstrip(lines,first,last) end for i=last,first,-1 do local li = lines[i] - if #li == 0 or find(li,"^%s*$") then + if #li == 0 or lpegmatch(onlyspaces,li) then last = last - 1 else break diff --git a/tex/context/base/char-utf.lua b/tex/context/base/char-utf.lua index d0e40e664..95ed48279 100644 --- a/tex/context/base/char-utf.lua +++ b/tex/context/base/char-utf.lua @@ -6,14 +6,19 @@ if not modules then modules = { } end modules ['char-utf'] = { license = "see context related readme files" } +-- todo: trackers +-- todo: no longer special characters (high) here, only needed in special cases and +-- these don't go through this file anyway +-- graphemes: basic symbols + --[[ldx-- -<p>When a sequence of <l n='utf'/> characters enters the application, it may -be neccessary to collapse subsequences into their composed variant.</p> +<p>When a sequence of <l n='utf'/> characters enters the application, it may be +neccessary to collapse subsequences into their composed variant.</p> <p>This module implements methods for collapsing and expanding <l n='utf'/> -sequences. We also provide means to deal with characters that are -special to <l n='tex'/> as well as 8-bit characters that need to end up -in special kinds of output (for instance <l n='pdf'/>).</p> +sequences. We also provide means to deal with characters that are special to +<l n='tex'/> as well as 8-bit characters that need to end up in special kinds +of output (for instance <l n='pdf'/>).</p> <p>We implement these manipulations as filters. One can run multiple filters over a string.</p> @@ -26,9 +31,6 @@ local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns local charfromnumber = characters.fromnumber --- todo: trackers --- graphemes: basic symbols - characters = characters or { } local characters = characters @@ -53,8 +55,8 @@ local utffilters = characters.filters.utf -- is characters.combined cached? --[[ldx-- -<p>It only makes sense to collapse at runtime, since we don't expect -source code to depend on collapsing.</p> +<p>It only makes sense to collapse at runtime, since we don't expect source code +to depend on collapsing.</p> --ldx]]-- -- for the moment, will be entries in char-def.lua @@ -164,27 +166,21 @@ function utffilters.addgrapheme(result,first,second) -- can be U+ 0x string or u end --[[ldx-- -<p>In order to deal with 8-bit output, we need to find a way to -go from <l n='utf'/> to 8-bit. This is handled in the -<l n='luatex'/> engine itself.</p> - -<p>This leaves us problems with characters that are specific to -<l n='tex'/> like <type>{}</type>, <type>$</type> and alike.</p> - -<p>We can remap some chars that tex input files are sensitive for to -a private area (while writing to a utility file) and revert then -to their original slot when we read in such a file. Instead of -reverting, we can (when we resolve characters to glyphs) map them -to their right glyph there.</p> - -<p>For this purpose we can use the private planes 0x0F0000 and -0x100000.</p> +<p>In order to deal with 8-bit output, we need to find a way to go from <l n='utf'/> to +8-bit. This is handled in the <l n='luatex'/> engine itself.</p> + +<p>This leaves us problems with characters that are specific to <l n='tex'/> like +<type>{}</type>, <type>$</type> and alike. We can remap some chars that tex input files +are sensitive for to a private area (while writing to a utility file) and revert then +to their original slot when we read in such a file. Instead of reverting, we can (when +we resolve characters to glyphs) map them to their right glyph there. For this purpose +we can use the private planes 0x0F0000 and 0x100000.</p> --ldx]]-- -local low = allocate({ }) -local high = allocate({ }) -local escapes = allocate({ }) -local special = "~#$%^&_{}\\|" +local low = allocate() +local high = allocate() +local escapes = allocate() +local special = "~#$%^&_{}\\|" -- "~#$%{}\\|" local private = { low = low, @@ -248,128 +244,21 @@ first snippet uses the relocated dollars.</p> </typing> <p>The next variant has lazy token collecting, on a 140 page mk.tex this saves -about .25 seconds, which is understandable because we have no graphmes and +about .25 seconds, which is understandable because we have no graphemes and not collecting tokens is not only faster but also saves garbage collecting. </p> --ldx]]-- --- lpeg variant is not faster --- --- I might use the combined loop at some point for the filter --- some day. - --- function utffilters.collapse(str) -- not really tested (we could preallocate a table) --- if str and str ~= "" then --- local nstr = #str --- if nstr > 1 then --- if initialize then -- saves a call --- initialize() --- end --- local tokens, t, first, done, n = { }, 0, false, false, 0 --- for second in utfcharacters(str) do --- local dec = decomposed[second] --- if dec then --- if not done then --- if n > 0 then --- for s in utfcharacters(str) do --- if n == 1 then --- break --- else --- t = t + 1 --- tokens[t] = s --- n = n - 1 --- end --- end --- end --- done = true --- elseif first then --- t = t + 1 --- tokens[t] = first --- end --- t = t + 1 --- tokens[t] = dec --- first = false --- elseif done then --- local crs = high[second] --- if crs then --- if first then --- t = t + 1 --- tokens[t] = first --- end --- first = crs --- else --- local cgf = graphemes[first] --- if cgf and cgf[second] then --- first = cgf[second] --- elseif first then --- t = t + 1 --- tokens[t] = first --- first = second --- else --- first = second --- end --- end --- else --- local crs = high[second] --- if crs then --- for s in utfcharacters(str) do --- if n == 1 then --- break --- else --- t = t + 1 --- tokens[t] = s --- n = n - 1 --- end --- end --- if first then --- t = t + 1 --- tokens[t] = first --- end --- first = crs --- done = true --- else --- local cgf = graphemes[first] --- if cgf and cgf[second] then --- for s in utfcharacters(str) do --- if n == 1 then --- break --- else --- t = t + 1 --- tokens[t] = s --- n = n - 1 --- end --- end --- first = cgf[second] --- done = true --- else --- first = second --- n = n + 1 --- end --- end --- end --- end --- if done then --- if first then --- t = t + 1 --- tokens[t] = first --- end --- return concat(tokens) -- seldom called --- end --- elseif nstr > 0 then --- return high[str] or str --- end --- end --- return str --- end - local skippable = table.tohash { "mkiv", "mkvi" } local filesuffix = file.suffix --- we could reuse tokens but it's seldom populated anyway - -function utffilters.collapse(str,filename) -- not really tested (we could preallocate a table) +function utffilters.collapse(str,filename) -- we can make high a seperate pass (never needed with collapse) if skippable[filesuffix(filename)] then return str + -- elseif find(filename,"^virtual://") then + -- return str + -- else + -- -- print("\n"..filename) end if str and str ~= "" then local nstr = #str @@ -380,44 +269,60 @@ function utffilters.collapse(str,filename) -- not really tested (we could preall local tokens, t, first, done, n = { }, 0, false, false, 0 for second in utfcharacters(str) do if done then - local crs = high[second] - if crs then - if first then - t = t + 1 - tokens[t] = first - end - first = crs - else - local cgf = graphemes[first] - if cgf and cgf[second] then - first = cgf[second] - elseif first then + if first then + if second == " " then t = t + 1 tokens[t] = first first = second else - first = second + -- local crs = high[second] + -- if crs then + -- t = t + 1 + -- tokens[t] = first + -- first = crs + -- else + local cgf = graphemes[first] + if cgf and cgf[second] then + first = cgf[second] + else + t = t + 1 + tokens[t] = first + first = second + end + -- end end + elseif second == " " then + first = second + else + -- local crs = high[second] + -- if crs then + -- first = crs + -- else + first = second + -- end end + elseif second == " " then + first = nil + n = n + 1 else - local crs = high[second] - if crs then - for s in utfcharacters(str) do - if n == 1 then - break - else - t = t + 1 - tokens[t] = s - n = n - 1 - end - end - if first then - t = t + 1 - tokens[t] = first - end - first = crs - done = true - else + -- local crs = high[second] + -- if crs then + -- for s in utfcharacters(str) do + -- if n == 1 then + -- break + -- else + -- t = t + 1 + -- tokens[t] = s + -- n = n - 1 + -- end + -- end + -- if first then + -- t = t + 1 + -- tokens[t] = first + -- end + -- first = crs + -- done = true + -- else local cgf = graphemes[first] if cgf and cgf[second] then for s in utfcharacters(str) do @@ -435,7 +340,7 @@ function utffilters.collapse(str,filename) -- not really tested (we could preall first = second n = n + 1 end - end + -- end end end if done then @@ -520,34 +425,3 @@ if sequencers then end) end - ---[[ldx-- -<p>Next we implement some commands that are used in the user interface.</p> ---ldx]]-- - --- commands = commands or { } --- --- function commands.uchar(first,second) --- context(utfchar(first*256+second)) --- end - ---[[ldx-- -<p>A few helpers (used to be <t>luat-uni<t/>).</p> ---ldx]]-- - --- obsolete: --- --- function utf.split(str) --- local t, n = { }, 0 --- for snippet in utfcharacters(str) do --- n = n + 1 --- t[n+1] = snippet --- end --- return t --- end --- --- function utf.each(str,fnc) --- for snippet in utfcharacters(str) do --- fnc(snippet) --- end --- end diff --git a/tex/context/base/chem-str.lua b/tex/context/base/chem-str.lua index dfcf0a3e1..8c1bf38fc 100644 --- a/tex/context/base/chem-str.lua +++ b/tex/context/base/chem-str.lua @@ -43,6 +43,7 @@ local lpegmatch = lpeg.match local P, R, S, C, Cs, Ct, Cc, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Ct, lpeg.Cc, lpeg.Cmt local variables = interfaces and interfaces.variables +local commands = commands local context = context local formatters = string.formatters local texcount = tex.count diff --git a/tex/context/base/cldf-int.lua b/tex/context/base/cldf-int.lua index 6cbfd666f..2743e4924 100644 --- a/tex/context/base/cldf-int.lua +++ b/tex/context/base/cldf-int.lua @@ -19,9 +19,10 @@ local catcodenumbers = catcodes.numbers local ctxcatcodes = catcodenumbers.ctxcatcodes local vrbcatcodes = catcodenumbers.vrbcatcodes +local context = context local contextsprint = context.sprint -local trace_define = false trackers.register("context.define", function(v) trace_define = v end) +local trace_define = false trackers.register("context.define", function(v) trace_define = v end) interfaces = interfaces or { } diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv index 24b56997e..ab6ac327b 100644 --- a/tex/context/base/cont-new.mkiv +++ b/tex/context/base/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2013.05.31 21:57} +\newcontextversion{2013.06.04 19:42} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf Binary files differindex f5dd71f5f..0711ba100 100644 --- a/tex/context/base/context-version.pdf +++ b/tex/context/base/context-version.pdf diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv index 168179f8a..63faee3ab 100644 --- a/tex/context/base/context.mkiv +++ b/tex/context/base/context.mkiv @@ -25,7 +25,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2013.05.31 21:57} +\edef\contextversion{2013.06.04 19:42} \edef\contextkind {beta} %D For those who want to use this: diff --git a/tex/context/base/core-env.lua b/tex/context/base/core-env.lua index 025192d4b..de977d0bb 100644 --- a/tex/context/base/core-env.lua +++ b/tex/context/base/core-env.lua @@ -21,6 +21,8 @@ local texsetcount = tex.setcount local allocate = utilities.storage.allocate local setmetatableindex = table.setmetatableindex +local context = context + local undefined = csname_id("*undefined*crap*") local iftrue = create("iftrue")[2] -- inefficient hack diff --git a/tex/context/base/core-uti.lua b/tex/context/base/core-uti.lua index 96ccdca48..53ff891e2 100644 --- a/tex/context/base/core-uti.lua +++ b/tex/context/base/core-uti.lua @@ -95,6 +95,8 @@ job.register('job.variables.checksums', checksums) local rmethod, rvalue +local setxvalue = context.setxvalue + local function initializer() tobesaved = jobvariables.tobesaved collected = jobvariables.collected @@ -110,7 +112,7 @@ local function initializer() end tobesaved.randomseed = rvalue for cs, value in next, collected do - context.setxvalue(cs,value) + setxvalue(cs,value) end end diff --git a/tex/context/base/data-exp.lua b/tex/context/base/data-exp.lua index 8a2fd0320..c67e97bb1 100644 --- a/tex/context/base/data-exp.lua +++ b/tex/context/base/data-exp.lua @@ -9,7 +9,7 @@ if not modules then modules = { } end modules ['data-exp'] = { local format, find, gmatch, lower, char, sub = string.format, string.find, string.gmatch, string.lower, string.char, string.sub local concat, sort = table.concat, table.sort local lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local Ct, Cs, Cc, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.P, lpeg.C, lpeg.S +local Ct, Cs, Cc, Carg, P, C, S = lpeg.Ct, lpeg.Cs, lpeg.Cc, lpeg.Carg, lpeg.P, lpeg.C, lpeg.S local type, next = type, next local ostype = os.type @@ -26,21 +26,21 @@ local resolvers = resolvers -- all, when working on the main resolver code, I don't want to scroll -- past this every time. See data-obs.lua for the gsub variant. -local function f_first(a,b) - local t, n = { }, 0 - for s in gmatch(b,"[^,]+") do - n = n + 1 ; t[n] = a .. s - end - return concat(t,",") -end - -local function f_second(a,b) - local t, n = { }, 0 - for s in gmatch(a,"[^,]+") do - n = n + 1 ; t[n] = s .. b - end - return concat(t,",") -end +-- local function f_first(a,b) +-- local t, n = { }, 0 +-- for s in gmatch(b,"[^,]+") do +-- n = n + 1 ; t[n] = a .. s +-- end +-- return concat(t,",") +-- end +-- +-- local function f_second(a,b) +-- local t, n = { }, 0 +-- for s in gmatch(a,"[^,]+") do +-- n = n + 1 ; t[n] = s .. b +-- end +-- return concat(t,",") +-- end -- kpsewhich --expand-braces '{a,b}{c,d}' -- ac:bc:ad:bd @@ -69,6 +69,21 @@ local function f_both(a,b) return concat(t,",") end +local comma = P(",") +local nocomma = (1-comma)^1 +local docomma = comma^1/"," +local before = Cs((nocomma * Carg(1) + docomma)^0) +local after = Cs((Carg(1) * nocomma + docomma)^0) +local both = Cs(((C(nocomma) * Carg(1))/function(a,b) return lpegmatch(before,b,1,a) end + docomma)^0) + +local function f_first (a,b) return lpegmatch(after, b,1,a) end +local function f_second(a,b) return lpegmatch(before,a,1,b) end +local function f_both (a,b) return lpegmatch(both, b,1,a) end + +-- print(f_first ("a", "x,y,z")) +-- print(f_second("a,b,c","x")) +-- print(f_both ("a,b,c","x,y,z")) + local left = P("{") local right = P("}") local var = P((1 - S("{}" ))^0) diff --git a/tex/context/base/file-mod.lua b/tex/context/base/file-mod.lua index 3659d3089..822f37c86 100644 --- a/tex/context/base/file-mod.lua +++ b/tex/context/base/file-mod.lua @@ -27,6 +27,8 @@ local report_modules = logs.reporter("resolvers","modules") commands = commands or { } local commands = commands +local context = context + local findbyscheme = resolvers.finders.byscheme -- use different one local iterator = utilities.parsers.iterator diff --git a/tex/context/base/font-syn.lua b/tex/context/base/font-syn.lua index 27176dade..eb420e5be 100644 --- a/tex/context/base/font-syn.lua +++ b/tex/context/base/font-syn.lua @@ -257,7 +257,7 @@ function filters.afm(name) local f = io.open(name) if f then local hash = { } - for line in f:lines() do + for line in f:lines() do -- slow local key, value = match(line,"^(.+)%s+(.+)%s*$") if key and #key > 0 then hash[lower(key)] = value diff --git a/tex/context/base/grph-inc.lua b/tex/context/base/grph-inc.lua index 9603419ae..be1b7f45a 100644 --- a/tex/context/base/grph-inc.lua +++ b/tex/context/base/grph-inc.lua @@ -55,6 +55,10 @@ local allocate = utilities.storage.allocate local setmetatableindex = table.setmetatableindex local replacetemplate = utilities.templates.replace +local images = img + +local context = context + local variables = interfaces.variables local codeinjections = backends.codeinjections local nodeinjections = backends.nodeinjections @@ -67,8 +71,6 @@ local trace_inclusion = false trackers.register("graphics.inclusion", functi local report_inclusion = logs.reporter("graphics","inclusion") -local context, img = context, img - local f_hash_part = formatters["%s->%s->%s"] local f_hash_full = formatters["%s->%s->%s->%s->%s->%s->%s"] @@ -82,7 +84,7 @@ local v_default = variables.default local maxdimen = 2^30-1 -function img.check(figure) +function images.check(figure) if figure then local width = figure.width local height = figure.height @@ -103,36 +105,38 @@ end --- some extra img functions --- can become luat-img.lua -local imgkeys = img.keys() +local allimagekeys = images.keys() -function img.totable(imgtable) +local function imagetotable(imgtable) local result = { } - for k=1,#imgkeys do - local key = imgkeys[k] + for k=1,#allimagekeys do + local key = allimagekeys[k] result[key] = imgtable[key] end return result end -function img.serialize(i,...) - return table.serialize(img.totable(i),...) +images.totable = imagetotable + +function images.serialize(i,...) + return table.serialize(imagetotable(i),...) end -function img.print(i,...) - return table.print(img.totable(i),...) +function images.print(i,...) + return table.print(imagetotable(i),...) end -function img.clone(i,data) +function images.clone(i,data) i.width = data.width or i.width i.height = data.height or i.height -- attr etc return i end -local validsizes = table.tohash(img.boxes()) -local validtypes = table.tohash(img.types()) +local validsizes = table.tohash(images.boxes()) +local validtypes = table.tohash(images.types()) -function img.checksize(size) +function images.checksize(size) if size then size = gsub(size,"box","") return validsizes[size] and size or "crop" @@ -143,7 +147,7 @@ end local indexed = { } -function img.ofindex(n) +function images.ofindex(n) return indexed[n] end @@ -430,7 +434,7 @@ function figures.initialize(request) request.height = h > 0 and h or nil -- request.page = math.max(tonumber(request.page) or 1,1) - request.size = img.checksize(request.size) + request.size = images.checksize(request.size) request.object = request.object == v_yes request["repeat"] = request["repeat"] == v_yes request.preview = request.preview == v_yes @@ -1033,7 +1037,7 @@ function checkers.generic(data) local hash = f_hash_full(name,page,size,color,conversion,resolution,mask) local figure = figures_loaded[hash] if figure == nil then - figure = img.new { + figure = images.new { filename = name, page = page, pagebox = dr.size, @@ -1041,7 +1045,7 @@ function checkers.generic(data) } codeinjections.setfigurecolorspace(data,figure) codeinjections.setfiguremask(data,figure) - figure = figure and img.check(img.scan(figure)) or false + figure = figure and images.check(images.scan(figure)) or false local f, d = codeinjections.setfigurealternative(data,figure) figure, data = f or figure, d or data figures_loaded[hash] = figure @@ -1084,15 +1088,15 @@ function includers.generic(data) if figure == nil then figure = ds.private if figure then - figure = img.copy(figure) - figure = figure and img.clone(figure,data.request) or false + figure = images.copy(figure) + figure = figure and images.clone(figure,data.request) or false end figures_used[hash] = figure end if figure then local nr = figures.boxnumber -- it looks like we have a leak in attributes here .. todo - local box = node.hpack(img.node(figure)) -- img.node(figure) not longer valid + local box = node.hpack(images.node(figure)) -- images.node(figure) not longer valid indexed[figure.index] = figure box.width, box.height, box.depth = figure.width, figure.height, 0 -- new, hm, tricky, we need to do that in tex (yet) texbox[nr] = box diff --git a/tex/context/base/java-ini.lua b/tex/context/base/java-ini.lua index 321e4e24d..673379494 100644 --- a/tex/context/base/java-ini.lua +++ b/tex/context/base/java-ini.lua @@ -6,30 +6,36 @@ if not modules then modules = { } end modules ['java-ini'] = { license = "see context related readme files" } +-- todo: don't flush scripts if no JS key + local format = string.format local concat = table.concat local lpegmatch, P, S, C, Carg, Cc = lpeg.match, lpeg.P, lpeg.S, lpeg.C, lpeg.Carg, lpeg.Cc -local allocate = utilities.storage.allocate -local settings_to_array = utilities.parsers.settings_to_array -local variables = interfaces.variables -local formatters = string.formatters +local allocate = utilities.storage.allocate +local settings_to_array = utilities.parsers.settings_to_array --- todo: don't flush scripts if no JS key +local variables = interfaces.variables +local formatters = string.formatters + +local context = context +local commands = commands -local trace_javascript = false trackers.register("backends.javascript", function(v) trace_javascript = v end) +local trace_javascript = false trackers.register("backends.javascript", function(v) trace_javascript = v end) local report_javascripts = logs.reporter ("interactions","javascripts") local status_javascripts = logs.messenger("interactions","javascripts") -interactions.javascripts = interactions.javascripts or { } -local javascripts = interactions.javascripts +local javascripts = interactions.javascripts or { } +interactions.javascripts = javascripts -javascripts.codes = allocate() -javascripts.preambles = allocate() -javascripts.functions = allocate() +local codes = allocate() +local preambles = allocate() +local functions = allocate() -local codes, preambles, functions = javascripts.codes, javascripts.preambles, javascripts.functions +javascripts.codes = codes +javascripts.preambles = preambles +javascripts.functions = functions local preambled = { } diff --git a/tex/context/base/l-dir.lua b/tex/context/base/l-dir.lua index 3d0576eeb..e47de6031 100644 --- a/tex/context/base/l-dir.lua +++ b/tex/context/base/l-dir.lua @@ -273,9 +273,8 @@ if onwindows then str = str .. "/" .. s end end - local first, middle, last local drive = false - first, middle, last = match(str,"^(//)(//*)(.*)$") + local first, middle, last = match(str,"^(//)(//*)(.*)$") if first then -- empty network path == local path else diff --git a/tex/context/base/l-lpeg.lua b/tex/context/base/l-lpeg.lua index 323c73b69..99b4bbb13 100644 --- a/tex/context/base/l-lpeg.lua +++ b/tex/context/base/l-lpeg.lua @@ -137,7 +137,7 @@ patterns.nonwhitespace = nonwhitespace local stripper = spacer^0 * C((spacer^0 * nonspacer^1)^0) -- from example by roberto ------ collapser = Cs(spacer^0/"" * ((spacer^1 * P(-1) / "") + (spacer^1/" ") + P(1))^0) +----- collapser = Cs(spacer^0/"" * ((spacer^1 * endofstring / "") + (spacer^1/" ") + P(1))^0) local collapser = Cs(spacer^0/"" * nonspacer^0 * ((spacer^0/" " * nonspacer^1)^0)) patterns.stripper = stripper @@ -187,7 +187,7 @@ patterns.singlequoted = squote * patterns.nosquote * squote patterns.doublequoted = dquote * patterns.nodquote * dquote patterns.quoted = patterns.doublequoted + patterns.singlequoted -patterns.propername = R("AZ","az","__") * R("09","AZ","az", "__")^0 * P(-1) +patterns.propername = R("AZ","az","__") * R("09","AZ","az", "__")^0 * endofstring patterns.somecontent = (anything - newline - space)^1 -- (utf8char - newline - space)^1 patterns.beginline = #(1-newline) @@ -421,7 +421,7 @@ function lpeg.replacer(one,two,makefunction,isutf) -- in principle we should sor end end -function lpeg.finder(lst,makefunction) +function lpeg.finder(lst,makefunction) -- beware: slower than find with 'patternless finds' local pattern if type(lst) == "table" then pattern = P(false) @@ -456,8 +456,8 @@ local splitters_f, splitters_s = { }, { } function lpeg.firstofsplit(separator) -- always return value local splitter = splitters_f[separator] if not splitter then - separator = P(separator) - splitter = C((1 - separator)^0) + local pattern = P(separator) + splitter = C((1 - pattern)^0) splitters_f[separator] = splitter end return splitter @@ -466,13 +466,35 @@ end function lpeg.secondofsplit(separator) -- nil if not split local splitter = splitters_s[separator] if not splitter then - separator = P(separator) - splitter = (1 - separator)^0 * separator * C(anything^0) + local pattern = P(separator) + splitter = (1 - pattern)^0 * pattern * C(anything^0) splitters_s[separator] = splitter end return splitter end +local splitters_s, splitters_p = { }, { } + +function lpeg.beforesuffix(separator) -- nil if nothing but empty is ok + local splitter = splitters_s[separator] + if not splitter then + local pattern = P(separator) + splitter = C((1 - pattern)^0) * pattern * endofstring + splitters_s[separator] = splitter + end + return splitter +end + +function lpeg.afterprefix(separator) -- nil if nothing but empty is ok + local splitter = splitters_p[separator] + if not splitter then + local pattern = P(separator) + splitter = pattern * C(anything^0) + splitters_p[separator] = splitter + end + return splitter +end + function lpeg.balancer(left,right) left, right = P(left), P(right) return P { left * ((1 - left - right) + V(1))^0 * right } diff --git a/tex/context/base/l-string.lua b/tex/context/base/l-string.lua index 77c076cc5..9b079b00a 100644 --- a/tex/context/base/l-string.lua +++ b/tex/context/base/l-string.lua @@ -91,6 +91,8 @@ end local pattern = P(" ")^0 * P(-1) +-- patterns.onlyspaces = pattern + function string.is_empty(str) if str == "" then return true diff --git a/tex/context/base/lang-url.lua b/tex/context/base/lang-url.lua index 35381e672..4ed5cdea1 100644 --- a/tex/context/base/lang-url.lua +++ b/tex/context/base/lang-url.lua @@ -8,11 +8,11 @@ if not modules then modules = { } end modules ['lang-url'] = { local utfcharacters, utfvalues, utfbyte, utfchar = utf.characters, utf.values, utf.byte, utf.char -context = context - commands = commands or { } local commands = commands +context = context + --[[ <p>Hyphenating <l n='url'/>'s is somewhat tricky and a matter of taste. I did consider using a dedicated hyphenation pattern or dealing with it by node diff --git a/tex/context/base/lpdf-epa.lua b/tex/context/base/lpdf-epa.lua index 034e6d7e2..61d57b8d3 100644 --- a/tex/context/base/lpdf-epa.lua +++ b/tex/context/base/lpdf-epa.lua @@ -15,11 +15,12 @@ local formatters = string.formatters ----- lpegmatch, lpegpatterns = lpeg.match, lpeg.patterns -local trace_links = false trackers.register("figures.links", function(v) trace_links = v end) +local trace_links = false trackers.register("figures.links", function(v) trace_links = v end) +local report_link = logs.reporter("backend","merging") -local report_link = logs.reporter("backend","merging") - -local backends, lpdf = backends, lpdf +local backends = backends +local lpdf = lpdf +local context = context local variables = interfaces.variables local codeinjections = backends.pdf.codeinjections diff --git a/tex/context/base/lpdf-wid.lua b/tex/context/base/lpdf-wid.lua index 9ea4744f1..5bb809327 100644 --- a/tex/context/base/lpdf-wid.lua +++ b/tex/context/base/lpdf-wid.lua @@ -12,10 +12,13 @@ local texbox, texcount = tex.box, tex.count local settings_to_array = utilities.parsers.settings_to_array local settings_to_hash = utilities.parsers.settings_to_hash -local report_media = logs.reporter("backend","media") -local report_attachment = logs.reporter("backend","attachment") +local report_media = logs.reporter("backend","media") +local report_attachment = logs.reporter("backend","attachment") -local backends, lpdf, nodes = backends, lpdf, nodes +local backends = backends +local lpdf = lpdf +local nodes = nodes +local context = context local nodeinjections = backends.pdf.nodeinjections local codeinjections = backends.pdf.codeinjections diff --git a/tex/context/base/luat-cbk.lua b/tex/context/base/luat-cbk.lua index 5aa12005b..4f044f9ac 100644 --- a/tex/context/base/luat-cbk.lua +++ b/tex/context/base/luat-cbk.lua @@ -306,7 +306,7 @@ function garbagecollector.check(size,criterium) end end --- this will move +-- this will move to a module commands = commands or { } diff --git a/tex/context/base/lxml-ctx.lua b/tex/context/base/lxml-ctx.lua index 968dbda71..1191d6796 100644 --- a/tex/context/base/lxml-ctx.lua +++ b/tex/context/base/lxml-ctx.lua @@ -10,11 +10,13 @@ if not modules then modules = { } end modules ['lxml-ctx'] = { local format, find = string.format, string.find -local xml = xml - +local xml = xml xml.ctx = { } xml.ctx.enhancers = { } +local context = context +local commands = commands + -- hashen function xml.ctx.enhancers.compound(root,lpath,before,tokens,after) -- todo lpeg diff --git a/tex/context/base/lxml-dir.lua b/tex/context/base/lxml-dir.lua index 3c68664ae..48c0ac41e 100644 --- a/tex/context/base/lxml-dir.lua +++ b/tex/context/base/lxml-dir.lua @@ -24,12 +24,13 @@ local formatters = string.formatters -- <directive attribute='cdx' value="*" element="cals:table" setup="cdx:cals:table:*"/> -- </directives> -local lxml, context = lxml, context +local lxml = lxml +local context = context -local getid = lxml.getid +local getid = lxml.getid -lxml.directives = lxml.directives or { } -local directives = lxml.directives +local directives = lxml.directives or { } +lxml.directives = directives local report_lxml = logs.reporter("xml","tex") @@ -106,9 +107,11 @@ directives.handle = handle_setup function directives.setup(root,attribute,element) handle_setup('setup',root,attribute,element) end + function directives.before(root,attribute,element) handle_setup('before',root,attribute,element) end + function directives.after(root,attribute,element) handle_setup('after',root,attribute,element) end diff --git a/tex/context/base/lxml-sor.lua b/tex/context/base/lxml-sor.lua index 951017bcd..aba1c3b8d 100644 --- a/tex/context/base/lxml-sor.lua +++ b/tex/context/base/lxml-sor.lua @@ -9,9 +9,12 @@ if not modules then modules = { } end modules ['lxml-sor'] = { local format, concat, rep = string.format, table.concat, string.rep local lpegmatch = lpeg.match -local xml, lxml = xml, lxml +local xml = xml +local lxml = lxml +local context = context -lxml.sorters = lxml.sorters or { } +local lxmlsorters = lxml.sorters or { } +lxml.sorters = lxmlsorters if not lxml.splitid then local splitter = lpeg.C((1-lpeg.P(":"))^1) * lpeg.P("::") * lpeg.C(lpeg.P(1)^1) @@ -27,7 +30,7 @@ end local lists = { } -function lxml.sorters.reset(name) +function lxmlsorters.reset(name) lists[name] = { sorted = false, entries = { }, @@ -36,7 +39,7 @@ function lxml.sorters.reset(name) } end -function lxml.sorters.add(name,n,key) +function lxmlsorters.add(name,n,key) local list = lists[name] if list.sorted then -- reverse is messed up, we could regenerate it and go on @@ -56,7 +59,7 @@ function lxml.sorters.add(name,n,key) end end -function lxml.sorters.show(name) +function lxmlsorters.show(name) local list = lists[name] local entries = list and list.entries local NC, NR, bold = context.NC, context.NR, context.bold -- somehow bold is not working @@ -92,9 +95,9 @@ function lxml.sorters.show(name) end end -lxml.sorters.compare = sorters.comparers.basic -- (a,b) +lxmlsorters.compare = sorters.comparers.basic -- (a,b) -function lxml.sorters.sort(name) +function lxmlsorters.sort(name) local list = lists[name] local entries = list and list.entries if entries then @@ -117,7 +120,7 @@ function lxml.sorters.sort(name) r.split = splitter(strip(r.key)) end -- sorting - sorters.sort(results,lxml.sorters.compare) + sorters.sort(results,lxmlsorters.compare) -- finalizing list.nofsorted = #results local split = { } @@ -137,7 +140,7 @@ function lxml.sorters.sort(name) end end -function lxml.sorters.flush(name,setup) +function lxmlsorters.flush(name,setup) local list = lists[name] local results = list and list.results local xmlw = context.xmlw diff --git a/tex/context/base/lxml-tex.lua b/tex/context/base/lxml-tex.lua index 112f62751..09ffa2851 100644 --- a/tex/context/base/lxml-tex.lua +++ b/tex/context/base/lxml-tex.lua @@ -27,6 +27,7 @@ local catcodenumbers = catcodes.numbers local ctxcatcodes = catcodenumbers.ctxcatcodes -- todo: use different method local notcatcodes = catcodenumbers.notcatcodes -- todo: use different method +local commands = commands local context = context local contextsprint = context.sprint -- with catcodes (here we use fast variants, but with option for tracing) diff --git a/tex/context/base/m-database.lua b/tex/context/base/m-database.lua index 47854daa0..0aaee7e93 100644 --- a/tex/context/base/m-database.lua +++ b/tex/context/base/m-database.lua @@ -9,17 +9,19 @@ if not modules then modules = { } end modules ['m-database'] = { local sub, gmatch, format = string.sub, string.gmatch, string.format local concat = table.concat local lpegpatterns, lpegmatch, lpegsplitat = lpeg.patterns, lpeg.match, lpeg.splitat -local lpegP, lpegC, lpegS, lpegCt = lpeg.P, lpeg.C, lpeg.S, lpeg.Ct +local lpegP, lpegC, lpegS, lpegCt, lpegCc, lpegCs = lpeg.P, lpeg.C, lpeg.S, lpeg.Ct, lpeg.Cc, lpeg.Cs local stripstring = string.strip +moduledata.database = moduledata.database or { } +moduledata.database.csv = moduledata.database.csv or { } + -- One also needs to enable context.trace, here we only plug in some code (maybe -- some day this tracker will also toggle the main context tracer. -local trace_flush = false trackers.register("module.database.flush", function(v) trace_flush = v end) - +local trace_flush = false trackers.register("module.database.flush", function(v) trace_flush = v end) local report_database = logs.reporter("database") -buffers.database = buffers.database or { } +local context = context local l_tab = lpegpatterns.tab local l_space = lpegpatterns.space @@ -36,7 +38,7 @@ local separators = { -- not interfaced spaces = l_space^1, } -function buffers.database.process(settings) +function moduledata.database.csv.process(settings) local data if settings.type == "file" then local filename = resolvers.finders.byscheme("any",settings.database) @@ -55,7 +57,7 @@ function buffers.database.process(settings) local left, right = settings.left or "", settings.right or "" local setups = settings.setups or "" local strip = settings.strip == v_yes or false - local command = settings.command + local command = settings.command or "" separatorchar = (not separatorchar and ",") or separators[separatorchar] or separatorchar local separator = type(separatorchar) == "string" and lpegS(separatorchar) or separatorchar local whatever = lpegC((1 - separator)^0) @@ -73,12 +75,34 @@ function buffers.database.process(settings) whatever = quotedata + whatever end local checker = commentchar ~= "" and lpegS(commentchar) - local splitter = lpegCt(whatever * (separator * whatever)^0) + if strip then + whatever = whatever / stripstring + end + if left ~= "" then + whatever = lpegCc(left) * whatever + end + if right ~= "" then + whatever = whatever * lpegCc(right) + end + if command ~= "" then + whatever = lpegCc("{") * whatever * lpegCc("}") + end + whatever = whatever * (separator/"" * whatever)^0 + if first ~= "" then + whatever = lpegCc(first) * whatever + end + if last ~= "" then + whatever = whatever * lpegCc(last) + end + if command ~= "" then + whatever = lpegCs(lpegCc(command) * whatever) + else + whatever = lpegCs(whatever) + end local found = false for i=1,#data do local line = data[i] if not lpegmatch(l_empty,line) and (not checker or not lpegmatch(checker,line)) then - local list = lpegmatch(splitter,line) if not found then if setups ~= "" then context.begingroup() @@ -87,39 +111,7 @@ function buffers.database.process(settings) context(before) found = true end - if trace_flush then - local result, r = { }, 0 - r = r + 1 ; result[r] = first - for j=1,#list do - local str = strip and stripstring(list[j]) or list[j] - r = r + 1 ; result[r] = left - if command == "" then - r = r + 1 ; result[r] = str - else - r = r + 1 ; result[r] = command - r = r + 1 ; result[r] = "{" - r = r + 1 ; result[r] = str - r = r + 1 ; result[r] = "}" - end - r = r + 1 ; result[r] = right - end - r = r + 1 ; result[r] = last - context(concat(result)) - else - context(first) - for j=1,#list do - local str = strip and stripstring(list[j]) or list[j] - context(left) - if command == "" then - context(str) - else - context(command) - context(false,str) - end - context(right) - end - context(last) - end + context(lpegmatch(whatever,line)) end end if found then diff --git a/tex/context/base/m-database.mkiv b/tex/context/base/m-database.mkiv index 0285d3bcd..b7357f11a 100644 --- a/tex/context/base/m-database.mkiv +++ b/tex/context/base/m-database.mkiv @@ -52,7 +52,7 @@ \let\currentdatabasename\currentdatabase \let\currentdatabase\empty \fi - \ctxlua{buffers.database.process { + \ctxlua{moduledata.database.csv.process { name = "\currentdatabase", type = "\currentdatabasetype", database = "\currentdatabasename", @@ -120,8 +120,7 @@ first={\endgraf[}, last={]\endgraf}, left={ (}, - right={) }, - command=\ruledhbox] + right={) }] \startbuffer[testbuffer] 1,2,3,4,5 diff --git a/tex/context/base/math-ini.lua b/tex/context/base/math-ini.lua index 7ba1d4514..076a56184 100644 --- a/tex/context/base/math-ini.lua +++ b/tex/context/base/math-ini.lua @@ -21,14 +21,15 @@ local setmathcode, setdelcode = tex.setmathcode, tex.setdelcode local settexattribute = tex.setattribute local floor = math.floor -local context = context +local context = context +local commands = commands -local contextsprint = context.sprint -local contextfprint = context.fprint -- a bit inefficient +local contextsprint = context.sprint +local contextfprint = context.fprint -- a bit inefficient -local trace_defining = false trackers.register("math.defining", function(v) trace_defining = v end) +local trace_defining = false trackers.register("math.defining", function(v) trace_defining = v end) -local report_math = logs.reporter("mathematics","initializing") +local report_math = logs.reporter("mathematics","initializing") mathematics = mathematics or { } local mathematics = mathematics diff --git a/tex/context/base/meta-fig.mkiv b/tex/context/base/meta-fig.mkiv index 7fbc33be9..46dc4cffc 100644 --- a/tex/context/base/meta-fig.mkiv +++ b/tex/context/base/meta-fig.mkiv @@ -54,7 +54,7 @@ \unexpanded\def\MPfigure#1#2% test for dup figure, can be replaced by a textext {\bgroup - \getfiguredimensionsonly[#1]% [\c!object=\v!no] already set + \getfiguredimensions[#1]% [\c!object=\v!no] already set \startMPcode externalfigure "#1" xscaled \the\dimexpr\figurewidth \relax\space % must be points diff --git a/tex/context/base/mult-aux.lua b/tex/context/base/mult-aux.lua index 3c4cbcc0f..bdc626d4c 100644 --- a/tex/context/base/mult-aux.lua +++ b/tex/context/base/mult-aux.lua @@ -12,6 +12,8 @@ interfaces.namespaces = interfaces.namespaces or { } local namespaces = interfaces.namespaces local variables = interfaces.variables +local context = context + local trace_namespaces = false trackers.register("interfaces.namespaces", function(v) trace_namespaces = v end) local report_namespaces = logs.reporter("interface","namespaces") diff --git a/tex/context/base/mult-ini.lua b/tex/context/base/mult-ini.lua index 3b18738de..e3ff904a6 100644 --- a/tex/context/base/mult-ini.lua +++ b/tex/context/base/mult-ini.lua @@ -10,6 +10,9 @@ local format, gmatch, match = string.format, string.gmatch, string.match local lpegmatch = lpeg.match local serialize = table.serialize +local context = context +local commands = commands + local allocate = utilities.storage.allocate local mark = utilities.storage.mark local prtcatcodes = catcodes.numbers.prtcatcodes diff --git a/tex/context/base/node-ser.lua b/tex/context/base/node-ser.lua index b0a6e9952..70743123c 100644 --- a/tex/context/base/node-ser.lua +++ b/tex/context/base/node-ser.lua @@ -14,7 +14,9 @@ local concat, tohash, sortedkeys, printtable = table.concat, table.tohash, table local allocate = utilities.storage.allocate -local nodes, node = nodes, node +local context = context +local nodes = nodes +local node = node local traverse = node.traverse local is_node = node.is_node diff --git a/tex/context/base/node-tsk.lua b/tex/context/base/node-tsk.lua index 596ac765a..b426c6e24 100644 --- a/tex/context/base/node-tsk.lua +++ b/tex/context/base/node-tsk.lua @@ -18,10 +18,11 @@ local report_tasks = logs.reporter("tasks") local allocate = utilities.storage.allocate +local context = context local nodes = nodes -nodes.tasks = nodes.tasks or { } -local tasks = nodes.tasks +local tasks = nodes.tasks or { } +nodes.tasks = tasks local tasksdata = { } -- no longer public diff --git a/tex/context/base/page-inj.lua b/tex/context/base/page-inj.lua index 5b450d60e..f9b56fddb 100644 --- a/tex/context/base/page-inj.lua +++ b/tex/context/base/page-inj.lua @@ -16,6 +16,9 @@ local trace = false trackers.register("pagebuilder.injections",func local variables = interfaces.variables +local context = context +local commands = commands + local v_yes = variables.yes local v_previous = variables.previous local v_next = variables.next diff --git a/tex/context/base/page-pst.lua b/tex/context/base/page-pst.lua index 8586830cf..f714c25d9 100644 --- a/tex/context/base/page-pst.lua +++ b/tex/context/base/page-pst.lua @@ -11,6 +11,9 @@ if not modules then modules = { } end modules ['page-pst'] = { local format, validstring = string.format, string.valid local sortedkeys = table.sortedkeys +local context = context +local commands = commands + local cache = { } local function flush(page) diff --git a/tex/context/base/phys-dim.lua b/tex/context/base/phys-dim.lua index 45a99978d..123593ca8 100644 --- a/tex/context/base/phys-dim.lua +++ b/tex/context/base/phys-dim.lua @@ -53,6 +53,9 @@ local variables = interfaces.variables local v_reverse = variables.reverse local allocate = utilities.storage.allocate +local context = context +local commands = commands + local trace_units = false local report_units = logs.reporter("units") diff --git a/tex/context/base/scrn-but.lua b/tex/context/base/scrn-but.lua index e49372ce9..74f6e0cd9 100644 --- a/tex/context/base/scrn-but.lua +++ b/tex/context/base/scrn-but.lua @@ -6,6 +6,9 @@ if not modules then modules = { } end modules ['scrn-but'] = { license = "see context related readme files" } +local commands = commands +local context = context + local f_two_colon = string.formatters["%s:%s"] function commands.registerbuttons(tag,register,language) diff --git a/tex/context/base/scrn-hlp.lua b/tex/context/base/scrn-hlp.lua index 5f8368c6d..1e9d1f570 100644 --- a/tex/context/base/scrn-hlp.lua +++ b/tex/context/base/scrn-hlp.lua @@ -11,6 +11,9 @@ local format = string.format local help = { } interactions.help = help +local context = context +local commands = commands + local a_help = attributes.private("help") local copy_nodelist = node.copy_list diff --git a/tex/context/base/scrn-wid.lua b/tex/context/base/scrn-wid.lua index 4ad46761e..a2b3e28b9 100644 --- a/tex/context/base/scrn-wid.lua +++ b/tex/context/base/scrn-wid.lua @@ -9,11 +9,15 @@ if not modules then modules = { } end modules ['scrn-wid'] = { interactions = interactions or { } local interactions = interactions -local attachments = { } -local comments = { } -local soundclips = { } -local renderings = { } -local linkedlists = { } +local context = context + +local allocate = utilities.storage.allocate + +local attachments = allocate() +local comments = allocate() +local soundclips = allocate() +local renderings = allocate() +local linkedlists = allocate() interactions.attachments = attachments interactions.soundclips = soundclips diff --git a/tex/context/base/scrp-ini.lua b/tex/context/base/scrp-ini.lua index 18f86475f..56422e622 100644 --- a/tex/context/base/scrp-ini.lua +++ b/tex/context/base/scrp-ini.lua @@ -11,12 +11,16 @@ if not modules then modules = { } end modules ['scrp-ini'] = { local attributes, nodes, node = attributes, nodes, node -local trace_analyzing = false trackers.register("scripts.analyzing", function(v) trace_analyzing = v end) -local trace_injections = false trackers.register("scripts.injections", function(v) trace_injections = v end) +local trace_analyzing = false trackers.register("scripts.analyzing", function(v) trace_analyzing = v end) +local trace_injections = false trackers.register("scripts.injections", function(v) trace_injections = v end) +local trace_splitting = false trackers.register("scripts.splitting", function(v) trace_splitting = v end) +local trace_splitdetail = false trackers.register("scripts.splitring.detail", function(v) trace_splitdetail = v end) local report_preprocessing = logs.reporter("scripts","preprocessing") +local report_splitting = logs.reporter("scripts","splitting") -local utfchar = utf.char +local utfbyte, utfsplit = utf.byte, utf.split +local gmatch = string.gmatch local first_glyph = node.first_glyph or node.first_character local traverse_id = node.traverse_id @@ -29,6 +33,9 @@ local unsetvalue = attributes.unsetvalue local glyph_code = nodecodes.glyph local glue_code = nodecodes.glue +local emwidths = fonts.hashes.emwidths +local exheights = fonts.hashes.exheights + local a_scriptinjection = attributes.private('scriptinjection') local a_scriptsplitting = attributes.private('scriptsplitting') local a_scriptstatus = attributes.private('scriptstatus') @@ -41,6 +48,14 @@ local setmetatableindex = table.setmetatableindex local enableaction = nodes.tasks.enableaction local disableaction = nodes.tasks.disableaction +local insert_node_after = node.insert_after + +local nodepool = nodes.pool +local new_glue = nodepool.glue +local new_rule = nodepool.rule +local new_penalty = nodepool.penalty +----- new_gluespec = nodepool.gluespec + scripts = scripts or { } local scripts = scripts @@ -198,7 +213,7 @@ local function provide(t,k) return v end -setmetatableindex(hash,provide) +setmetatableindex(hash,provide) -- should come from char-def scripts.hash = hash @@ -542,10 +557,289 @@ function scripts.injectors.handler(head) end end -function scripts.splitters.handler(head) - return head, false +-- kind of experimental .. might move to it's own module + +-- function scripts.splitters.handler(head) +-- return head, false +-- end + +local function addwords(tree,data) + if not tree then + tree = { } + end + for word in gmatch(data,"%S+") do + local root = tree + local list = utfsplit(word,true) + for i=1,#list do + local l = utfbyte(list[i]) + local r = root[l] + if not r then + r = { } + root[l] = r + end + if i == #list then + r.final = word -- true -- could be something else, like word in case of tracing + else + root = r + end + end + end + return tree +end + +local loaded = { } + +function splitters.load(handler,files) + local files = handler.files + local tree = handler.tree or { } + handler.tree = tree + if not files then + return + elseif type(files) == "string" then + files = { files } + handler.files = files + end + if trace_splitting then + report_splitting("loading splitter data for language/script %a",handler.name) + end + loaded[handler.name or "unknown"] = (loaded[handler.name or "unknown"] or 0) + 1 + statistics.starttiming(loaded) + for i=1,#files do + local filename = files[i] + local fullname = resolvers.findfile(filename) + if fullname == "" then + fullname = resolvers.findfile(filename .. ".gz") + end + if fullname ~= "" then + if trace_splitting then + report_splitting("loading file %a",fullname) + end + local suffix, gzipped = gzip.suffix(fullname) + if suffix == "lua" then + local specification = table.load(fullname,gzipped and gzip.load) + if specification then + local lists = specification.lists + if lists then + for i=1,#lists do + local entry = lists[i] + local data = entry.data + if data then + if entry.compression == "zlib" then + data = zlib.decompress(data) + if entry.length and entry.length ~= #data then + report_splitting("compression error in file %a",fullname) + end + end + if data then + addwords(tree,data) + end + end + end + end + end + else + local data = gzipped and io.loadgzip(fullname) or io.loaddata(fullname) + if data then + addwords(tree,data) + end + end + else + report_splitting("unknown file %a",filename) + end + end + statistics.stoptiming(loaded) + return tree +end + +statistics.register("loaded split lists", function() + if next(loaded) then + return string.format("%s, load time: %s",table.sequenced(loaded),statistics.elapsedtime(loaded)) + end +end) + +-- function splitters.addlist(name,filename) +-- local handler = scripts.handlers[name] +-- if handler and filename then +-- local files = handler.files +-- if not files then +-- files = { } +-- elseif type(files) == "string" then +-- files = { files } +-- end +-- handler.files = files +-- if type(filename) == "string" then +-- filename = utilities.parsers.settings_to_array(filename) +-- end +-- if type(filename) == "table" then +-- for i=1,#filename do +-- files[#files+1] = filenames[i] +-- end +-- end +-- end +-- end +-- +-- commands.setscriptsplitterlist = splitters.addlist + +local categories = characters.categories or { } + +local function hit(root,head) + local current = head.next + local lastrun = false + local lastfinal = false + while current and current.id == glyph_code do + local char = current.char + local newroot = root[char] + if newroot then + local final = newroot.final + if final then + lastrun = current + lastfinal = final + end + root = newroot + elseif categories[char] == "mn" then + -- continue + else + return lastrun, lastfinal + end + current = current.next + end + if lastrun then + return lastrun, lastfinal + end end +local tree, attr, proc + +function splitters.handler(head) + local current = head + local done = false + while current do + if current.id == glyph_code then + local a = current[a_scriptsplitting] + if a then + if a ~= attr then + local handler = numbertohandler[a] + tree = handler.tree or { } + attr = a + proc = handler.splitter + end + if proc then + local root = tree[current.char] + if root then + -- we don't check for attributes in the hitter (yet) + local last, final = hit(root,current) + if last then + local next = last.next + if next and next.id == glyph_code then + local nextchar = next.char + if tree[nextchar] then + if trace_splitdetail then + if type(final) == "string" then + report_splitting("advance %s processing between <%s> and <%c>","with",final,nextchar) + else + report_splitting("advance %s processing between <%c> and <%c>","with",char,nextchar) + end + end + head, current = proc(handler,head,current,last,1) + done = true + else + if trace_splitdetail then + -- could be punctuation + if type(final) == "string" then + report_splitting("advance %s processing between <%s> and <%c>","without",final,nextchar) + else + report_splitting("advance %s processing between <%c> and <%c>","without",char,nextchar) + end + end + head, current = proc(handler,head,current,last,2) + done = true + end + end + end + end + end + end + end + current = current.next + end + return head, done +end + +local function marker(head,current,font,color) -- could become: nodes.tracers.marker + local ex = exheights[font] + local em = emwidths [font] + head, current = insert_node_after(head,current,new_penalty(10000)) + head, current = insert_node_after(head,current,new_glue(-0.05*em)) + head, current = insert_node_after(head,current,new_rule(0.05*em,1.5*ex,0.5*ex)) + setnodecolor(current,color) + return head, current +end + +-- local function process(handler,head,first,last) +-- dataset = numbertodataset[first[a_scriptsplitting]] +-- stretch = emwidths[first.font]*dataset.inter_word_stretch_factor +-- return insert_node_after(head,last,new_glue(0,stretch)) +-- end +-- +-- local cache = { } table.setmetatableindex(cache,function(t,k) +-- local v = new_gluespec(0,k) +-- nodepool.register(v) +-- t[k] = v +-- return v +-- end) +-- return insert_node_after(head,last,new_glue(cache[last_s])) + +local last_a, last_f, last_s, last_q + +function splitters.insertafter(handler,head,first,last,detail) + local a = first[a_scriptsplitting] + local f = first.font + if a ~= last_a or f ~= last_f then + last_s = emwidths[f] * numbertodataset[a].inter_word_stretch_factor + last_a = a + last_f = f + end + if trace_splitting then + head, last = marker(head,last,f,detail == 2 and "trace:r" or "trace:g") + end + if ignore then + return head, last + else + return insert_node_after(head,last,new_glue(0,last_s)) + end +end + +-- word-xx.lua: +-- +-- return { +-- comment = "test", +-- copyright = "not relevant", +-- language = "en", +-- timestamp = "2013-05-20 14:15:21", +-- version = "1.00", +-- lists = { +-- { +-- -- data = "we thrive information in thick worlds because of our marvelous and everyday capacity to select edit single out structure highlight group pair merge harmonize synthesize focus organize condense reduce boil down choose categorize catalog classify list abstract scan look into idealize isolate discriminate distinguish screen pigeonhole pick over sort integrate blend inspect filter lump skip smooth chunk average approximate cluster aggregate outline summarize itemize review dip into flip through browse glance into leaf through skim refine enumerate glean synopsize winnow the wheat from the chaff and separate the sheep from the goats", +-- data = "abstract aggregate and approximate average because blend boil browse capacity catalog categorize chaff choose chunk classify cluster condense dip discriminate distinguish down edit enumerate everyday filter flip focus from glance glean goats group harmonize highlight idealize in information inspect integrate into isolate itemize leaf list look lump marvelous merge of organize our out outline over pair pick pigeonhole reduce refine review scan screen select separate sheep single skim skip smooth sort structure summarize synopsize synthesize the thick thrive through to we wheat winnow worlds", +-- }, +-- }, +-- } + +scripts.installmethod { + name = "test", + splitter = splitters.insertafter, + initializer = splitters.load, + files = { + -- "scrp-imp-word-test.lua", + "word-xx.lua", + }, + datasets = { + default = { + inter_word_stretch_factor = 0.25, -- of quad + }, + }, +} + -- new plugin: local registercontext = fonts.specifiers.registercontext diff --git a/tex/context/base/scrp-ini.mkiv b/tex/context/base/scrp-ini.mkiv deleted file mode 100644 index fe62295bb..000000000 --- a/tex/context/base/scrp-ini.mkiv +++ /dev/null @@ -1,94 +0,0 @@ -%D \module -%D [ file=scrp-ini, -%D version=2009.02.06, -%D title=\CONTEXT\ Script Macros, -%D subtitle=Initialization, -%D author=Hans Hagen, -%D date=\currentdate, -%D copyright={PRAGMA ADE \& \CONTEXT\ Development Team}] -%C -%C This module is part of the \CONTEXT\ macro||package and is -%C therefore copyrighted by \PRAGMA. See mreadme.pdf for -%C details. - -% here we collect code from other places (was organized differently) - -\registerctxluafile{scrp-ini}{1.001} -\registerctxluafile{scrp-cjk}{1.001} -\registerctxluafile{scrp-eth}{1.001} - -\definesystemattribute[scriptinjection][public] -\definesystemattribute[scriptsplitting][public] -\definesystemattribute[scriptstatus] [public] - -%D Since scripts need specific \LUA\ code we use hard coded attribute -%D values, but we might have more tricks at some time, so we use a -%D proper define macro too. - -\unprotect - -\installcorenamespace{script} - -\installcommandhandler \??script {script} \??script - -\let\setupscripts\setupscript % be nice - -% presets are global and are currently defined in lua - -\appendtoks - \setuevalue\currentscript{\setscript[\currentscript]}% -\to \everydefinescript - -\unexpanded\def\scripts_basics_set - {\ctxlua{scripts.set("\currentscript","\scriptparameter\c!method","\scriptparameter\c!preset")}} - -\unexpanded\def\setscript[#1]% - {\edef\currentscript{#1}% - \scripts_basics_set} - -\unexpanded\def\resetscript - {\ctxlua{scripts.reset()}} - -\unexpanded\def\startscript[#1]% - {\begingroup - \edef\currentscript{#1}% - \scripts_basics_set} - -\unexpanded\def\stopscript - {\endgroup} - -% % todo: -% -% \unexpanded\def\setscriptsplitterlist -% {\dodoubleargument\scripts_basics_set_splitter_list} -% -% \def\scripts_basics_set_splitter_list[#1][#2]% -% {\ctxcommand{setscriptsplitterlist("#1","#2")} - -% \setscript[hangul] \hangul \startscript[hangul] - -\definescript [hangul] [\c!method=hangul] -\definescript [hanzi] [\c!method=hanzi] -\definescript [nihongo] [\c!method=nihongo] -\definescript [ethiopic] [\c!method=ethiopic] -\definescript [thai] [\c!method=thai] - -\definescript [latin] [\c!method=] % resets the attribute (also currentscript) - -\definescript [test] [\c!method=test] - -% a new trick (at some point we will predefine more scripts and consider a link with the above) - -\appendtoks - \ifx\currentscript\empty - \doifelse{\scriptparameter\s!features}\v!auto\enableautofontscript\disableautofontscript - \fi -\to \everysetupscript - -\unexpanded\def\enableautofontscript {\ctxcommand{enableautofontscript ()}} -\unexpanded\def\disableautofontscript{\ctxcommand{disableautofontscript()}} - -\definefontfeature[latn][script=latn] -\definefontfeature[grek][script=grek] - -\protect \endinput diff --git a/tex/context/base/scrp-tha.lua b/tex/context/base/scrp-tha.lua new file mode 100644 index 000000000..ec5df07c0 --- /dev/null +++ b/tex/context/base/scrp-tha.lua @@ -0,0 +1,57 @@ +if not modules then modules = { } end modules ['scrp-tha'] = { + version = 1.001, + comment = "companion to scrp-ini.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This module needs dictionary files that looks as follows. At some point +-- we will add these files to the distribution. +-- +-- word-th.lua: +-- +-- return { +-- comment = "The data is taken from http://thailinux.gits.net.th/websvn/wsvn/software.swath by Phaisarn Charoenpornsawat and Theppitak Karoonboonyanan.", +-- copyright = "gnu general public license", +-- language = "th", +-- compiling = "mtxrun --script patterns --words --update --compress word-th.lua", +-- timestamp = "0000-00-00 00:00:00", +-- version = "1.00", +-- lists = { +-- { filename = "tdict-city.txt" }, +-- { filename = "tdict-collection.txt" }, +-- { filename = "tdict-common.txt" }, +-- { filename = "tdict-country.txt" }, +-- { filename = "tdict-district.txt" }, +-- { filename = "tdict-geo.txt" }, +-- { filename = "tdict-history.txt" }, +-- { filename = "tdict-ict.txt" }, +-- { filename = "tdict-lang-ethnic.txt" }, +-- { filename = "tdict-proper.txt" }, +-- { filename = "tdict-science.txt" }, +-- { filename = "tdict-spell.txt" }, +-- { filename = "tdict-std-compound.txt" }, +-- { filename = "tdict-std.txt" }, +-- }, +-- } + +-- Currently there is nothing additional special here, first we need a +-- ConTeXt user who uses it. It's a starting point. + +local splitters = scripts.splitters + +scripts.installmethod { + name = "thai", + splitter = splitters.insertafter, + initializer = splitters.load, + files = { + -- "scrp-imp-word-thai.lua", + "word-th.lua", + }, + datasets = { + default = { + inter_word_stretch_factor = 0.25, -- of quad + }, + }, +} diff --git a/tex/context/base/spac-hor.lua b/tex/context/base/spac-hor.lua index 09920bd46..c9d6e2b15 100644 --- a/tex/context/base/spac-hor.lua +++ b/tex/context/base/spac-hor.lua @@ -8,6 +8,10 @@ if not modules then modules = { } end modules ['spac-hor'] = { local match = string.match local utfbyte = utf.byte + +local context = context +local commands = commands + local chardata = characters.data local can_have_space = table.tohash { diff --git a/tex/context/base/spac-ver.lua b/tex/context/base/spac-ver.lua index 7d030ab1a..36fa357d0 100644 --- a/tex/context/base/spac-ver.lua +++ b/tex/context/base/spac-ver.lua @@ -33,7 +33,7 @@ local formatters = string.formatters local P, C, R, S, Cc = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.Cc -local nodes, node, trackers, attributes, context = nodes, node, trackers, attributes, context +local nodes, node, trackers, attributes, context, commands = nodes, node, trackers, attributes, context, commands local variables = interfaces.variables diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf Binary files differindex 996b7ae13..124f648e8 100644 --- a/tex/context/base/status-files.pdf +++ b/tex/context/base/status-files.pdf diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf Binary files differindex 6a18ef05b..fd994d224 100644 --- a/tex/context/base/status-lua.pdf +++ b/tex/context/base/status-lua.pdf diff --git a/tex/context/base/strc-blk.lua b/tex/context/base/strc-blk.lua index 791f8f99b..935b6c061 100644 --- a/tex/context/base/strc-blk.lua +++ b/tex/context/base/strc-blk.lua @@ -13,7 +13,10 @@ local find, format, validstring = string.find, string.format, string.valid local settings_to_set, settings_to_array = utilities.parsers.settings_to_set, utilities.parsers.settings_to_array local allocate = utilities.storage.allocate -local structures, context = structures, context +local context = context +local commands = commands + +local structures = structures structures.blocks = structures.blocks or { } diff --git a/tex/context/base/strc-doc.lua b/tex/context/base/strc-doc.lua index 50a9e67a0..a3aeb31e7 100644 --- a/tex/context/base/strc-doc.lua +++ b/tex/context/base/strc-doc.lua @@ -41,9 +41,10 @@ local trace_detail = false trackers.register("structures.detail", fu local report_structure = logs.reporter("structure","sectioning") -local structures = structures local context = context +local commands = commands +local structures = structures local helpers = structures.helpers local documents = structures.documents local sections = structures.sections diff --git a/tex/context/base/strc-ini.lua b/tex/context/base/strc-ini.lua index fd7c10f79..1e60e253f 100644 --- a/tex/context/base/strc-ini.lua +++ b/tex/context/base/strc-ini.lua @@ -34,7 +34,8 @@ local xmlcatcodes = catcodenumbers.xmlcatcodes local notcatcodes = catcodenumbers.notcatcodes local txtcatcodes = catcodenumbers.txtcatcodes -local context, commands = context, commands +local context = context +local commands = commands local pushcatcodes = context.pushcatcodes local popcatcodes = context.popcatcodes diff --git a/tex/context/base/strc-lev.lua b/tex/context/base/strc-lev.lua index 50a63c938..947889e1e 100644 --- a/tex/context/base/strc-lev.lua +++ b/tex/context/base/strc-lev.lua @@ -8,6 +8,9 @@ if not modules then modules = { } end modules ['strc-lev'] = { local insert, remove = table.insert, table.remove +local context = context +local commands = commands + local sections = structures.sections local default = interfaces.variables.default diff --git a/tex/context/base/strc-lst.lua b/tex/context/base/strc-lst.lua index 305b6a6fa..d5597109f 100644 --- a/tex/context/base/strc-lst.lua +++ b/tex/context/base/strc-lst.lua @@ -27,6 +27,9 @@ local trace_lists = false trackers.register("structures.lists", function( local report_lists = logs.reporter("structure","lists") +local context = context +local commands = commands + local structures = structures local lists = structures.lists local sections = structures.sections @@ -122,6 +125,8 @@ function lists.groupindex(name,group) return groupindex and groupindex[group] or 0 end +-- we could use t (as hash key) in order to check for dup entries + function lists.addto(t) local m = t.metadata local u = t.userdata @@ -130,12 +135,13 @@ function lists.addto(t) end local numberdata = t.numberdata local group = numberdata and numberdata.group + local name = m.name if not group then -- forget about it elseif group == "" then group, numberdata.group = nil, nil else - local groupindex = groupindices[m.name][group] + local groupindex = groupindices[name][group] if groupindex then numberdata.numbers = cached[groupindex].numberdata.numbers end @@ -154,7 +160,10 @@ function lists.addto(t) setcomponent(t) -- might move to the tex end end if group then - groupindices[m.name][group] = p + groupindices[name][group] = p + end + if trace_lists then + report_lists("added %a, internal %a",name,p) end return p end @@ -181,10 +190,17 @@ end -- this is the main pagenumber enhancer +local enhanced = { } + function lists.enhance(n) - -- todo: symbolic names for counters local l = cached[n] - if l then + if not l then + report_lists("enhancing %a, unknown internal",n) + elseif enhanced[n] then + if trace_lists then + report_lists("enhancing %a, name %a, duplicate ignored",n,name) + end + else local metadata = l.metadata local references = l.references -- @@ -196,19 +212,23 @@ function lists.enhance(n) -- tags local kind = metadata.kind local name = metadata.name + if trace_lists then + report_lists("enhancing %a, name %a",n,name) + end if references then -- is this used ? local tag = tags.getid(kind,name) if tag and tag ~= "?" then references.tag = tag end - --~ references.listindex = n end -- specific enhancer (kind of obsolete) local enhancer = kind and lists.enhancers[kind] if enhancer then enhancer(l) end + -- + enhanced[n] = true return l end end diff --git a/tex/context/base/strc-mar.lua b/tex/context/base/strc-mar.lua index 7b3ac11e1..26603fe72 100644 --- a/tex/context/base/strc-mar.lua +++ b/tex/context/base/strc-mar.lua @@ -12,7 +12,9 @@ if not modules then modules = { } end modules ['strc-mar'] = { local insert, concat = table.insert, table.concat local tostring, next, rawget = tostring, next, rawget local lpegmatch = lpeg.match -local match = string.match + +local context = context +local commands = commands local allocate = utilities.storage.allocate local setmetatableindex = table.setmetatableindex @@ -121,7 +123,7 @@ local function sweep(head,first,last) end local list = n.list if list then - first, last = sweep(list, first, last) + first, last = sweep(list,first,last) end end end @@ -659,8 +661,10 @@ function marks.fetchallmarks(name,range) fetchallmarks(name,range ) -- here we have a few helpers .. will become commands.* +local pattern = lpeg.afterprefix("li::") + function marks.title(tag,n) - local listindex = match(n,"^li::(.-)$") + local listindex = lpegmatch(pattern,n) if listindex then commands.savedlisttitle(tag,listindex,"marking") else @@ -669,7 +673,7 @@ function marks.title(tag,n) end function marks.number(tag,n) -- no spec - local listindex = match(n,"^li::(.-)$") + local listindex = lpegmatch(pattern,n) if listindex then commands.savedlistnumber(tag,listindex) else diff --git a/tex/context/base/strc-not.lua b/tex/context/base/strc-not.lua index 882e00a44..8aae8878b 100644 --- a/tex/context/base/strc-not.lua +++ b/tex/context/base/strc-not.lua @@ -91,10 +91,10 @@ end local function getn(tag) local nd = notedata[tag] - return (nd and #nd) or 0 + return nd and #nd or 0 end -notes.get = get +notes.get = get notes.getn = getn -- we could make a special enhancer diff --git a/tex/context/base/strc-not.mkvi b/tex/context/base/strc-not.mkvi index 76816d035..19f84e906 100644 --- a/tex/context/base/strc-not.mkvi +++ b/tex/context/base/strc-not.mkvi @@ -622,8 +622,15 @@ \unexpanded\def\strc_notes_inject_symbol_nop {\strc_notes_inject_symbol_indeed\conditionalfalse} -\unexpanded\def\strc_notes_inject_symbol_snc - {\currentconstructionsynchronize} % this flushes the data to the list +% % this flushes the data to the list +% +% \unexpanded\def\strc_notes_inject_symbol_snc +% {\currentconstructionsynchronize} +% +% but instead we need to do this with the content + +\unexpanded\def\strc_notes_inject_symbol_snc % so this will go away probably + {} \unexpanded\def\strc_notes_inject_symbol_indeed#synchronize% {\removeunwantedspaces diff --git a/tex/context/base/strc-pag.lua b/tex/context/base/strc-pag.lua index f70d37d63..a71c6bca6 100644 --- a/tex/context/base/strc-pag.lua +++ b/tex/context/base/strc-pag.lua @@ -26,6 +26,7 @@ local counterdata = counters.data local variables = interfaces.variables local context = context +local commands = commands local processors = typesetters.processors local applyprocessor = processors.apply diff --git a/tex/context/base/strc-ref.lua b/tex/context/base/strc-ref.lua index 284418c48..03ae94823 100644 --- a/tex/context/base/strc-ref.lua +++ b/tex/context/base/strc-ref.lua @@ -16,6 +16,7 @@ if not modules then modules = { } end modules ['strc-ref'] = { local format, find, gmatch, match, concat = string.format, string.find, string.gmatch, string.match, table.concat local texcount, texsetcount = tex.count, tex.setcount +local floor = math.floor local rawget, tonumber = rawget, tonumber local lpegmatch = lpeg.match local copytable = table.copy @@ -46,6 +47,7 @@ local report_empty = logs.reporter("references","empty") local variables = interfaces.variables local constants = interfaces.constants local context = context +local commands = commands local v_default = variables.default local v_url = variables.url @@ -140,16 +142,10 @@ job.register('structures.references.collected', tobesaved, initializer, finalize local maxreferred = 1 local nofreferred = 0 --- local function initializer() -- can we use a tobesaved as metatable for collected? --- tobereferred = references.tobereferred --- referred = references.referred --- nofreferred = #referred --- end - local function initializer() -- can we use a tobesaved as metatable for collected? tobereferred = references.tobereferred referred = references.referred - setmetatableindex(referred,get) -- hm, what is get ? + nofreferred = #referred end -- We make the array sparse (maybe a finalizer should optionally return a table) because @@ -219,6 +215,8 @@ local function referredpage(n) return texcount.realpageno end +-- setmetatableindex(referred,function(t,k) return referredpage(k) end ) + references.referredpage = referredpage function references.registerpage(n) -- called in the backend code @@ -1931,7 +1929,8 @@ local specials = references.testspecials -- real page to determine if we need contrastlocation as that is more lightweight. local function checkedpagestate(n,page) - local r, p = referredpage(n), tonumber(page) + local r = referredpage(n) + local p = tonumber(page) if not p then return 0 elseif p > r then @@ -1944,7 +1943,9 @@ local function checkedpagestate(n,page) end local function setreferencerealpage(actions) - actions = actions or references.currentset + if not actions then + actions = references.currentset + end if not actions then return 0 else @@ -1976,7 +1977,9 @@ end -- normally such an analysis happens in the backend code function references.analyze(actions) - actions = actions or references.currentset + if not actions then + actions = references.currentset + end if not actions then actions = { realpage = 0, pagestate = 0 } elseif actions.pagestate then @@ -1995,12 +1998,15 @@ function references.analyze(actions) end function commands.referencepagestate(actions) - actions = actions or references.currentset + if not actions then + actions = references.currentset + end if not actions then context(0) else if not actions.pagestate then references.analyze(actions) -- delayed unless explicitly asked for +-- print("NO STATE",actions.reference,actions.pagestate) end context(actions.pagestate) end diff --git a/tex/context/base/strc-reg.lua b/tex/context/base/strc-reg.lua index 40cd3455b..f4682c90d 100644 --- a/tex/context/base/strc-reg.lua +++ b/tex/context/base/strc-reg.lua @@ -35,6 +35,7 @@ local splitprocessor = processors.split local variables = interfaces.variables local context = context +local commands = commands local matchingtilldepth, numberatdepth = sections.matchingtilldepth, sections.numberatdepth diff --git a/tex/context/base/symb-ini.lua b/tex/context/base/symb-ini.lua index deeef667a..9586338be 100644 --- a/tex/context/base/symb-ini.lua +++ b/tex/context/base/symb-ini.lua @@ -6,6 +6,7 @@ if not modules then modules = { } end modules ['symb-ini'] = { license = "see context related readme files" } +local context, commands = context, commands local variables = interfaces.variables diff --git a/tex/context/base/syst-lua.lua b/tex/context/base/syst-lua.lua index ef524c339..e47041444 100644 --- a/tex/context/base/syst-lua.lua +++ b/tex/context/base/syst-lua.lua @@ -10,9 +10,10 @@ local format, find, match, rep = string.format, string.find, string.match, strin local tonumber = tonumber local S, lpegmatch, lpegtsplitat = lpeg.S, lpeg.match, lpeg.tsplitat -local context = context +commands = commands or { } +local commands = commands -commands = commands or { } +local context = context function commands.writestatus(...) logs.status(...) end -- overloaded later diff --git a/tex/context/base/toks-ini.lua b/tex/context/base/toks-ini.lua index ef4b5406b..0f0c016f8 100644 --- a/tex/context/base/toks-ini.lua +++ b/tex/context/base/toks-ini.lua @@ -5,6 +5,7 @@ if not modules then modules = { } end modules ['toks-ini'] = { license = "see context related readme files" } +local context, commands = context, commands local utfbyte, utfchar, utfvalues = utf.byte, utf.char, utf.values local format, gsub = string.format, string.gsub diff --git a/tex/context/base/typo-prc.lua b/tex/context/base/typo-prc.lua index 5b74abd0b..a6c27ede6 100644 --- a/tex/context/base/typo-prc.lua +++ b/tex/context/base/typo-prc.lua @@ -8,7 +8,7 @@ if not modules then modules = { } end modules ['typo-prc'] = { -- moved from strc-ini.lua - +local context, commands = context, commands local formatters = string.formatters local lpegmatch, patterns, P, C, Cs = lpeg.match, lpeg.patterns, lpeg.P, lpeg.C, lpeg.Cs diff --git a/tex/context/base/util-lua.lua b/tex/context/base/util-lua.lua index f3be9dcd2..61d1190b2 100644 --- a/tex/context/base/util-lua.lua +++ b/tex/context/base/util-lua.lua @@ -41,291 +41,92 @@ luautilities.suffixes = { -- environment.loadpreprocessedfile can be set to a preprocessor -if jit or status.luatex_version >= 74 then - - local function register(name) - if tracestripping then - report_lua("stripped bytecode from %a",name or "unknown") - end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - if code and code ~= "" then - code = load(code) - if code then - code = dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) - if code and code ~= "" then - register(name) - io.savedata(lucfile,code) - return true, 0 - end - else - report_lua("fatal error %a in file %a",1,luafile) - end - else - report_lua("fatal error %a in file %a",2,luafile) - end - return false, 0 +local function register(name) + if tracestripping then + report_lua("stripped bytecode from %a",name or "unknown") end + strippedchunks[#strippedchunks+1] = name + luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 +end - -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - name = name or fullname - local code = environment.loadpreprocessedfile and environment.loadpreprocessedfile(fullname) or loadfile(fullname) +local function stupidcompile(luafile,lucfile,strip) + local code = io.loaddata(luafile) + if code and code ~= "" then + code = load(code) if code then - code() - end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip or luautilities.alwaysstripcode then + code = dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) + if code and code ~= "" then register(name) - return load(dump(code,true)), 0 - else - return code, 0 + io.savedata(lucfile,code) + return true, 0 end - elseif luautilities.alwaysstripcode then - register(name) - return load(dump(code,true)), 0 else - return code, 0 - end - end - - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then - code = load(code) - if not code then - report_lua("fatal error %a in file %a",3,name) - end - register(name) - code = dump(code,true) - end - return load(code), 0 - end - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - report_lua("compiling %a into %a",luafile,lucfile) - os.remove(lucfile) - local done = stupidcompile(luafile,lucfile,strip ~= false) - if done then - report_lua("dumping %a into %a stripped",luafile,lucfile) - if cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then - report_lua("removing %a",luafile) - os.remove(luafile) - end - end - return done - end - - function luautilities.loadstripped(...) - local l = load(...) - if l then - return load(dump(l,true)) - end - end - -else - - -- The next function was posted by Peter Cawley on the lua list and strips line - -- number information etc. from the bytecode data blob. We only apply this trick - -- when we store data tables. Stripping makes the compressed format file about - -- 1MB smaller (and uncompressed we save at least 6MB). - -- - -- You can consider this feature an experiment, so it might disappear. There is - -- no noticeable gain in runtime although the memory footprint should be somewhat - -- smaller (and the file system has a bit less to deal with). - -- - -- Begin of borrowed code ... works for Lua 5.1 which LuaTeX currently uses ... - - local function register(name,before,after) - local delta = before - after - if tracestripping then - report_lua("bytecodes stripped from %a, # before %s, # after %s, delta %s",name,before,after,delta) - end - strippedchunks[#strippedchunks+1] = name - luautilities.nofstrippedchunks = luautilities.nofstrippedchunks + 1 - luautilities.nofstrippedbytes = luautilities.nofstrippedbytes + delta - return delta - end - - local strip_code_pc - - if _MAJORVERSION == 5 and _MINORVERSION == 1 then - - strip_code_pc = function(dump,name) - local before = #dump - local version, format, endian, int, size, ins, num = byte(dump,5,11) - local subint - if endian == 1 then - subint = function(dump, i, l) - local val = 0 - for n = l, 1, -1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - else - subint = function(dump, i, l) - local val = 0 - for n = 1, l, 1 do - val = val * 256 + byte(dump,i + n - 1) - end - return val, i + l - end - end - local strip_function - strip_function = function(dump) - local count, offset = subint(dump, 1, size) - local stripped, dirty = rep("\0", size), offset + count - offset = offset + count + int * 2 + 4 - offset = offset + int + subint(dump, offset, int) * ins - count, offset = subint(dump, offset, int) - for n = 1, count do - local t - t, offset = subint(dump, offset, 1) - if t == 1 then - offset = offset + 1 - elseif t == 4 then - offset = offset + size + subint(dump, offset, size) - elseif t == 3 then - offset = offset + num - end - end - count, offset = subint(dump, offset, int) - stripped = stripped .. sub(dump,dirty, offset - 1) - for n = 1, count do - local proto, off = strip_function(sub(dump,offset, -1)) - stripped, offset = stripped .. proto, offset + off - 1 - end - offset = offset + subint(dump, offset, int) * int + int - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size + int * 2 - end - count, offset = subint(dump, offset, int) - for n = 1, count do - offset = offset + subint(dump, offset, size) + size - end - stripped = stripped .. rep("\0", int * 3) - return stripped, offset - end - dump = sub(dump,1,12) .. strip_function(sub(dump,13,-1)) - local after = #dump - local delta = register(name,before,after) - return dump, delta + report_lua("fatal error %a in file %a",1,luafile) end - else - - strip_code_pc = function(dump,name) - return dump, 0 - end - + report_lua("fatal error %a in file %a",2,luafile) end + return false, 0 +end - -- ... end of borrowed code. +-- quite subtle ... doing this wrong incidentally can give more bytes +function luautilities.loadedluacode(fullname,forcestrip,name) -- quite subtle ... doing this wrong incidentally can give more bytes - - function luautilities.loadedluacode(fullname,forcestrip,name) - -- quite subtle ... doing this wrong incidentally can give more bytes - local code = environment.loadpreprocessedfile and environment.preprocessedloadfile(fullname) or loadfile(fullname) - if code then - code() + name = name or fullname + local code = environment.loadpreprocessedfile and environment.loadpreprocessedfile(fullname) or loadfile(fullname) + if code then + code() + end + if forcestrip and luautilities.stripcode then + if type(forcestrip) == "function" then + forcestrip = forcestrip(fullname) end - if forcestrip and luautilities.stripcode then - if type(forcestrip) == "function" then - forcestrip = forcestrip(fullname) - end - if forcestrip then - local code, n = strip_code_pc(dump(code),name) - return load(code), n - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) - else - return code, 0 - end - elseif luautilities.alwaysstripcode then - return load(strip_code_pc(dump(code),name)) + if forcestrip or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)), 0 else return code, 0 end + elseif luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)), 0 + else + return code, 0 end +end - function luautilities.strippedloadstring(code,forcestrip,name) -- not executed - local n = 0 - if (forcestrip and luautilities.stripcode) or luautilities.alwaysstripcode then - code = load(code) - if not code then - report_lua("fatal error in file %a",name) - end - code, n = strip_code_pc(dump(code),name) - end - return load(code), n - end - - local function stupidcompile(luafile,lucfile,strip) - local code = io.loaddata(luafile) - local n = 0 - if code and code ~= "" then - code = load(code) - if not code then - report_lua("fatal error in file %a",luafile) - end - code = dump(code) - if strip then - code, n = strip_code_pc(code,luautilities.stripcode or luautilities.alwaysstripcode,luafile) -- last one is reported - end - if code and code ~= "" then - io.savedata(lucfile,code) - end +function luautilities.strippedloadstring(code,forcestrip,name) -- not executed + if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then + code = load(code) + if not code then + report_lua("fatal error %a in file %a",3,name) end - return n + register(name) + code = dump(code,true) end + return load(code), 0 +end - local luac_normal = "texluac -o %q %q" - local luac_strip = "texluac -s -o %q %q" - - function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true - report_lua("compiling %a into %a",luafile,lucfile) - os.remove(lucfile) - local done = false - if strip ~= false then - strip = true - end - if forcestupidcompile then - fallback = true - elseif strip then - done = os.spawn(format(luac_strip, lucfile,luafile)) == 0 - else - done = os.spawn(format(luac_normal,lucfile,luafile)) == 0 - end - if not done and fallback then - local n = stupidcompile(luafile,lucfile,strip) - if n > 0 then - report_lua("%a dumped into %a (%i bytes stripped)",luafile,lucfile,n) - else - report_lua("%a dumped into %a (unstripped)",luafile,lucfile) - end - cleanup = false -- better see how bad it is - done = true -- hm - end - if done and cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then +function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) -- defaults: cleanup=false strip=true + report_lua("compiling %a into %a",luafile,lucfile) + os.remove(lucfile) + local done = stupidcompile(luafile,lucfile,strip ~= false) + if done then + report_lua("dumping %a into %a stripped",luafile,lucfile) + if cleanup == true and lfs.isfile(lucfile) and lfs.isfile(luafile) then report_lua("removing %a",luafile) os.remove(luafile) end - return done end + return done +end - luautilities.loadstripped = loadstring - +function luautilities.loadstripped(...) + local l = load(...) + if l then + return load(dump(l,true)) + end end -- local getmetatable, type = getmetatable, type diff --git a/tex/context/base/util-str.lua b/tex/context/base/util-str.lua index 4890a11d6..638995772 100644 --- a/tex/context/base/util-str.lua +++ b/tex/context/base/util-str.lua @@ -618,7 +618,7 @@ local builder = Cs { "start", ["a"] = (prefix_any * P("a")) / format_a, -- %a => '...' (forces tostring) ["A"] = (prefix_any * P("A")) / format_A, -- %A => "..." (forces tostring) -- - ["*"] = Cs(((1-P("%"))^1 + P("%%")/"%%%%")^1) / format_rest, -- rest (including %%) + ["*"] = Cs(((1-P("%"))^1 + P("%%")/"%%")^1) / format_rest, -- rest (including %%) -- ["!"] = Carg(2) * prefix_any * P("!") * C((1-P("!"))^1) * P("!") / format_extension, } diff --git a/tex/context/patterns/word-xx.lua b/tex/context/patterns/word-xx.lua new file mode 100644 index 000000000..f8b38fe75 --- /dev/null +++ b/tex/context/patterns/word-xx.lua @@ -0,0 +1,14 @@ +return { + ["comment"]="test", + ["copyright"]="not relevant", + ["language"]="xx", + ["lists"]={ + { +-- ["data"]="we thrive information in thick worlds because of our marvelous and everyday capacity to select edit single out structure highlight group pair merge harmonize synthesize focus organize condense reduce boil down choose categorize catalog classify list abstract scan look into idealize isolate discriminate distinguish screen pigeonhole pick over sort integrate blend inspect filter lump skip smooth chunk average approximate cluster aggregate outline summarize itemize review dip into flip through browse glance into leaf through skim refine enumerate glean synopsize winnow the wheat from the chaff and separate the sheep from the goats", + ["data"]="abstract aggregate and approximate average because blend boil browse capacity catalog categorize chaff choose chunk classify cluster condense dip discriminate distinguish down edit enumerate everyday filter flip focus from glance glean goats group harmonize highlight idealize in information inspect integrate into isolate itemize leaf list look lump marvelous merge of organize our out outline over pair pick pigeonhole reduce refine review scan screen select separate sheep single skim skip smooth sort structure summarize synopsize synthesize the thick thrive through to we wheat winnow worlds", + }, + }, + ["timestamp"]="2013-05-20 14:15:21", + ["version"]="1.00", +} + diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index aa3b3958b..929f38c47 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 05/31/13 21:57:24 +-- merge date : 06/04/13 19:42:30 do -- begin closure to overcome local limits and interference @@ -197,7 +197,7 @@ patterns.unspacer=((patterns.spacer^1)/"")^0 patterns.singlequoted=squote*patterns.nosquote*squote patterns.doublequoted=dquote*patterns.nodquote*dquote patterns.quoted=patterns.doublequoted+patterns.singlequoted -patterns.propername=R("AZ","az","__")*R("09","AZ","az","__")^0*P(-1) +patterns.propername=R("AZ","az","__")*R("09","AZ","az","__")^0*endofstring patterns.somecontent=(anything-newline-space)^1 patterns.beginline=#(1-newline) patterns.longtostring=Cs(whitespace^0/""*nonwhitespace^0*((whitespace^0/" "*(patterns.quoted+nonwhitespace)^1)^0)) @@ -372,7 +372,7 @@ function lpeg.replacer(one,two,makefunction,isutf) return pattern end end -function lpeg.finder(lst,makefunction) +function lpeg.finder(lst,makefunction) local pattern if type(lst)=="table" then pattern=P(false) @@ -401,8 +401,8 @@ local splitters_f,splitters_s={},{} function lpeg.firstofsplit(separator) local splitter=splitters_f[separator] if not splitter then - separator=P(separator) - splitter=C((1-separator)^0) + local pattern=P(separator) + splitter=C((1-pattern)^0) splitters_f[separator]=splitter end return splitter @@ -410,12 +410,31 @@ end function lpeg.secondofsplit(separator) local splitter=splitters_s[separator] if not splitter then - separator=P(separator) - splitter=(1-separator)^0*separator*C(anything^0) + local pattern=P(separator) + splitter=(1-pattern)^0*pattern*C(anything^0) + splitters_s[separator]=splitter + end + return splitter +end +local splitters_s,splitters_p={},{} +function lpeg.beforesuffix(separator) + local splitter=splitters_s[separator] + if not splitter then + local pattern=P(separator) + splitter=C((1-pattern)^0)*pattern*endofstring splitters_s[separator]=splitter end return splitter end +function lpeg.afterprefix(separator) + local splitter=splitters_p[separator] + if not splitter then + local pattern=P(separator) + splitter=pattern*C(anything^0) + splitters_p[separator]=splitter + end + return splitter +end function lpeg.balancer(left,right) left,right=P(left),P(right) return P { left*((1-left-right)+V(1))^0*right } @@ -2851,7 +2870,7 @@ local builder=Cs { "start", ["W"]=(prefix_any*P("W"))/format_W, ["a"]=(prefix_any*P("a"))/format_a, ["A"]=(prefix_any*P("A"))/format_A, - ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%%%")^1)/format_rest, + ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, } local direct=Cs ( |