diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/context/lua/luatools.lua | 79 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-babel.lua | 368 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-chars.lua | 81 | ||||
| -rw-r--r-- | scripts/context/lua/mtxrun.lua | 2348 | ||||
| -rw-r--r-- | scripts/context/ruby/base/pdf.rb | 2 | ||||
| -rw-r--r-- | scripts/context/ruby/base/tex.rb | 55 | ||||
| -rw-r--r-- | scripts/context/ruby/base/texutil.rb | 84 | ||||
| -rw-r--r-- | scripts/context/ruby/www/dir.rb | 6 |
8 files changed, 1770 insertions, 1253 deletions
diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua index 84899275c..d53180cfa 100644 --- a/scripts/context/lua/luatools.lua +++ b/scripts/context/lua/luatools.lua @@ -5056,6 +5056,7 @@ messages.help = [[ --show-path show path expansion of ... --var-value report value of variable --find-file report file location +--find-path report path of file --make or --ini make luatex format --run or --fmt= run luatex format --luafile=str lua inifile (default is <progname>.lua) @@ -5203,22 +5204,26 @@ input.report(banner,"\n") local ok = true -if environment.arguments["selfmerge"] then - utils.merger.selfmerge(own.name,own.libs,own.list) -elseif environment.arguments["selfclean"] then - utils.merger.selfclean(own.name) -elseif environment.arguments["selfupdate"] then - input.my_prepare_b(instance) - input.verbose = true - input.update_script(instance,own.name,"luatools") -elseif environment.arguments["generate"] then - instance.renewcache = true - input.verbose = true +if environment.arguments["find-file"] then input.my_prepare_b(instance) -elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then + instance.format = environment.arguments["format"] or instance.format + if instance.pattern then + instance.allresults = true + input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) + else + input.for_files(instance, input.find_files, environment.files, instance.my_format) + end +elseif environment.arguments["find-path"] then input.my_prepare_b(instance) - input.verbose = true - input.my_make_format(instance,environment.files[1] or "") + local path = input.find_file(instance, environment.files[1], instance.my_format) + if input.verbose then + input.report(file.dirname(path)) + else + print(file.dirname(path)) + end +--~ elseif environment.arguments["first-writable-path"] then +--~ input.my_prepare_b(instance) +--~ input.report(input.first_writable_path(instance,environment.files[1] or ".")) elseif environment.arguments["run"] then input.my_prepare_a(instance) -- ! no need for loading databases input.verbose = true @@ -5227,15 +5232,6 @@ elseif environment.arguments["fmt"] then input.my_prepare_a(instance) -- ! no need for loading databases input.verbose = true input.my_run_format(instance,environment.arguments["fmt"], environment.files[1] or "") -elseif environment.arguments["variables"] or environment.arguments["show-variables"] then - input.my_prepare_a(instance) - input.list_variables(instance) -elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then - input.my_prepare_a(instance) - input.list_expansions(instance) -elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then - input.my_prepare_a(instance) - input.list_configurations(instance) elseif environment.arguments["expand-braces"] then input.my_prepare_a(instance) input.for_files(instance, input.expand_braces, environment.files) @@ -5251,18 +5247,6 @@ elseif environment.arguments["show-path"] or environment.arguments["path-value"] elseif environment.arguments["var-value"] or environment.arguments["show-value"] then input.my_prepare_a(instance) input.for_files(instance, input.var_value, environment.files) -elseif environment.arguments["find-file"] then - input.my_prepare_b(instance) - instance.format = environment.arguments["format"] or instance.format - if instance.pattern then - instance.allresults = true - input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) - else - input.for_files(instance, input.find_files, environment.files, instance.my_format) - end ---~ elseif environment.arguments["first-writable-path"] then ---~ input.my_prepare_b(instance) ---~ input.report(input.first_writable_path(instance,environment.files[1] or ".")) elseif environment.arguments["format-path"] then input.my_prepare_b(instance) input.report(caches.setpath(instance,"format")) @@ -5271,6 +5255,31 @@ elseif instance.pattern then -- brrr instance.format = environment.arguments["format"] or instance.format instance.allresults = true input.for_files(instance, input.find_files, { instance.pattern }, instance.my_format) +elseif environment.arguments["generate"] then + instance.renewcache = true + input.verbose = true + input.my_prepare_b(instance) +elseif environment.arguments["make"] or environment.arguments["ini"] or environment.arguments["compile"] then + input.my_prepare_b(instance) + input.verbose = true + input.my_make_format(instance,environment.files[1] or "") +elseif environment.arguments["selfmerge"] then + utils.merger.selfmerge(own.name,own.libs,own.list) +elseif environment.arguments["selfclean"] then + utils.merger.selfclean(own.name) +elseif environment.arguments["selfupdate"] then + input.my_prepare_b(instance) + input.verbose = true + input.update_script(instance,own.name,"luatools") +elseif environment.arguments["variables"] or environment.arguments["show-variables"] then + input.my_prepare_a(instance) + input.list_variables(instance) +elseif environment.arguments["expansions"] or environment.arguments["show-expansions"] then + input.my_prepare_a(instance) + input.list_expansions(instance) +elseif environment.arguments["configurations"] or environment.arguments["show-configurations"] then + input.my_prepare_a(instance) + input.list_configurations(instance) elseif environment.arguments["help"] or (environment.files[1]=='help') or (#environment.files==0) then if not input.verbose then input.verbose = true diff --git a/scripts/context/lua/mtx-babel.lua b/scripts/context/lua/mtx-babel.lua new file mode 100644 index 000000000..5ef9ae934 --- /dev/null +++ b/scripts/context/lua/mtx-babel.lua @@ -0,0 +1,368 @@ +-- data tables by Thomas A. Schmitz + +dofile(input.find_file(instance,"luat-log.lua")) + +texmf.instance = instance -- we need to get rid of this / maybe current instance in global table + +scripts = scripts or { } +scripts.babel = scripts.babel or { } + +do + + local replace_01 = { -- <' * | + a = "ᾅ", + h = "ᾕ", + w = "ᾥ", + } + + local replace_02 = { -- >' * | + a = "ᾄ", + h = "ᾔ", + w = "ᾤ", + } + + local replace_03 = { -- <` * | + a = "ᾃ", + h = "ᾓ", + w = "ᾣ", + } + + local replace_04 = { -- >` * | + a = "ᾂ", + h = "ᾒ", + w = "ᾢ", + } + + local replace_05 = { -- <~ * | + a = "ᾇ", + h = "ᾗ", + w = "ᾧ", + } + + local replace_06 = { -- >~ * | + a = "ᾆ", + h = "ᾖ", + w = "ᾦ" + } + + local replace_07 = { -- "' * + i = "ΐ", + u = "ΰ", + } + + local replace_08 = { -- "` * + i = "ῒ", + u = "ῢ", + } + + local replace_09 = { -- "~ * + i = "ῗ", + u = "ῧ", + } + + local replace_10 = { -- <' * + a = "ἅ", + e = "ἕ", + h = "ἥ", + i = "ἵ", + o = "ὅ", + u = "ὕ", + w = "ὥ", + A = "Ἅ", + E = "Ἕ", + H = "Ἥ", + I = "Ἵ", + O = "Ὅ", + U = "Ὕ", + W = "Ὥ", + } + + local replace_11 = { -- >' * + a = "ἄ", + e = "ἔ", + h = "ἤ", + i = "ἴ", + o = "ὄ", + u = "ὔ", + w = "ὤ", + A = "Ἄ", + E = "Ἔ", + H = "Ἤ", + I = "Ἴ", + O = "Ὄ", + U = "῎Υ", + W = "Ὤ", + } + + local replace_12 = { -- <` * + a = "ἃ", + e = "ἓ", + h = "ἣ", + i = "ἳ", + o = "ὃ", + u = "ὓ", + w = "ὣ", + A = "Ἃ", + E = "Ἒ", + H = "Ἣ", + I = "Ἳ", + O = "Ὃ", + U = "Ὓ", + W = "Ὣ", + } + + local replace_13 = { -- >` * + a = "ἂ", + e = "ἒ", + h = "ἢ", + i = "ἲ", + o = "ὂ", + u = "ὒ", + w = "ὢ", + A = "Ἂ", + E = "Ἒ", + H = "Ἢ", + I = "Ἲ", + O = "Ὂ", + U = "῍Υ", + W = "Ὢ", + } + + local replace_14 = { -- <~ * + a = "ἇ", + h = "ἧ", + i = "ἷ", + u = "ὗ", + w = "ὧ", + A = "Ἇ", + H = "Ἧ", + I = "Ἷ", + U = "Ὗ", + W = "Ὧ", + } + + local replace_15 = { -- >~ * + a = "ἆ", + h = "ἦ", + i = "ἶ", + u = "ὖ", + w = "ὦ", + A = "Ἆ", + H = "Ἦ", + I = "Ἶ", + U = "῏Υ", + W = "Ὦ", + } + + local replace_16 = { -- ' * | + a = "ᾴ", + h = "ῄ", + w = "ῴ", + } + + local replace_17 = { -- ` * | + a = "ᾲ", + h = "ῂ", + w = "ῲ", + } + + local replace_18 = { -- ~ * | + a = "ᾷ", + h = "ῇ", + w = "ῷ" + } + + local replace_19 = { -- ' * + a = "ά", + e = "έ", + h = "ή", + i = "ί", + o = "ό", + u = "ύ", + w = "ώ", + } + + local replace_20 = { -- ` * + a = "ὰ", + e = "ὲ", + h = "ὴ", + i = "ὶ", + o = "ὸ", + u = "ὺ", + w = "ὼ", + } + + local replace_21 = { -- ~ * + a = "ᾶ", + h = "ῆ", + i = "ῖ", + u = "ῦ", + w = "ῶ", + } + + local replace_22 = { -- < * + a = "ἁ", + e = "ἑ", + h = "ἡ", + i = "ἱ", + o = "ὁ", + u = "ὑ", + w = "ὡ", + r = "ῥ", + A = "Ἁ", + E = "Ἑ", + H = "Ἡ", + I = "Ἱ", + O = "Ὁ", + U = "Ὑ", + W = "Ὡ", + } + + local replace_23 = { -- > * + a = "ἀ", + e = "ἐ", + h = "ἠ", + i = "ἰ", + o = "ὀ", + u = "ὐ", + w = "ὠ", + A = "Ἀ", + E = "Ἐ", + H = "Ἠ", + I = "Ἰ", + O = "Ὀ", + U = "᾿Υ", + W = "Ὠ", + } + + local replace_24 = { -- * | + a = "ᾳ", + h = "ῃ", + w = "ῳ", + } + + local replace_25 = { -- " * + i = "ϊ", + u = "ϋ", + } + + local replace_26 = { -- * + a = "α", + b = "β", + g = "γ", + d = "δ", + e = "ε", + z = "ζ", + h = "η", + j = "θ", + i = "ι", + k = "κ", + l = "λ", + m = "μ", + n = "ν", + x = "ξ", + o = "ο", + p = "π", + r = "ρ", + s = "σ", + c = "ς", + t = "τ", + u = "υ", + f = "φ", + q = "χ", + y = "ψ", + w = "ω", + A = "Α", + B = "Β", + G = "Γ", + D = "Δ", + E = "Ε", + Z = "Ζ", + H = "Η", + J = "Θ", + I = "Ι", + K = "Κ", + L = "Λ", + M = "Μ", + N = "Ν", + X = "Ξ", + O = "Ο", + P = "Π", + R = "Ρ", + S = "Σ", + T = "Τ", + U = "Υ", + F = "Φ", + Q = "Χ", + Y = "Ψ", + W = "Ω" + } + + local skips_01 = lpeg.P("\\") * lpeg.R("az", "AZ")^1 + local skips_02 = lpeg.P("[") * (1- lpeg.S("[]"))^1 * lpeg.P("]") + + local stage_01 = (lpeg.P("<'") * lpeg.Cs(1) * lpeg.P('|')) / replace_01 + local stage_02 = (lpeg.P(">'") * lpeg.Cs(1) * lpeg.P('|')) / replace_02 + local stage_03 = (lpeg.P("<`") * lpeg.Cs(1) * lpeg.P('|')) / replace_03 + local stage_04 = (lpeg.P(">`") * lpeg.Cs(1) * lpeg.P('|')) / replace_04 + local stage_05 = (lpeg.P("<~") * lpeg.Cs(1) * lpeg.P('|')) / replace_05 + local stage_06 = (lpeg.P(">~") * lpeg.Cs(1) * lpeg.P('|')) / replace_06 + local stage_07 = (lpeg.P('"\'') * lpeg.Cs(1) ) / replace_07 + local stage_08 = (lpeg.P('"`') * lpeg.Cs(1) ) / replace_08 + local stage_09 = (lpeg.P('"~') * lpeg.Cs(1) ) / replace_09 + local stage_10 = (lpeg.P("<'") * lpeg.Cs(1) ) / replace_10 + local stage_11 = (lpeg.P(">'") * lpeg.Cs(1) ) / replace_11 + local stage_12 = (lpeg.P("<`") * lpeg.Cs(1) ) / replace_12 + local stage_13 = (lpeg.P(">`") * lpeg.Cs(1) ) / replace_13 + local stage_14 = (lpeg.P(">~") * lpeg.Cs(1) ) / replace_14 + local stage_15 = (lpeg.P(">~") * lpeg.Cs(1) ) / replace_15 + local stage_16 = (lpeg.P("'") * lpeg.Cs(1) * lpeg.P('|')) / replace_16 + local stage_17 = (lpeg.P("`") * lpeg.Cs(1) * lpeg.P('|')) / replace_17 + local stage_18 = (lpeg.P("~") * lpeg.Cs(1) * lpeg.P('|')) / replace_18 + local stage_19 = (lpeg.P("'") * lpeg.Cs(1) ) / replace_19 + local stage_20 = (lpeg.P("`") * lpeg.Cs(1) ) / replace_20 + local stage_21 = (lpeg.P("~") * lpeg.Cs(1) ) / replace_21 + local stage_22 = (lpeg.P("<") * lpeg.Cs(1) ) / replace_22 + local stage_23 = (lpeg.P(">") * lpeg.Cs(1) ) / replace_23 + local stage_24 = (lpeg.Cs(1) * lpeg.P('|') ) / replace_24 + local stage_25 = (lpeg.P('"') * lpeg.Cs(1) ) / replace_25 + local stage_26 = (lpeg.Cs(1) ) / replace_26 + + local stages = + skips_01 + skips_02 + + stage_01 + stage_02 + stage_03 + stage_04 + stage_05 + + stage_06 + stage_07 + stage_08 + stage_09 + stage_10 + + stage_11 + stage_12 + stage_13 + stage_14 + stage_15 + + stage_16 + stage_17 + stage_18 + stage_19 + stage_20 + + stage_21 + stage_22 + stage_23 + stage_24 + stage_25 + + stage_26 + + local parser = lpeg.Cs((stages + 1)^0) + + -- lpeg.print(parser): 254 lines + + function scripts.babel.convert(filename) + if filename and filename ~= empty then + local data = io.loaddata(filename) + if data then + data = parser:match(data) + io.savedata(filename .. ".utf", data) + end + end + end + +end + +banner = banner .. " | conversion tools " + +messages.help = [[ +--convert convert babel codes into utf +]] + +input.verbose = true + +if environment.argument("convert") then + scripts.babel.convert(environment.files[1] or "") +else + input.help(banner,messages.help) +end diff --git a/scripts/context/lua/mtx-chars.lua b/scripts/context/lua/mtx-chars.lua index 28d7b4a40..470846419 100644 --- a/scripts/context/lua/mtx-chars.lua +++ b/scripts/context/lua/mtx-chars.lua @@ -75,16 +75,97 @@ if not characters then characters = { } end end end +scripts.chars.banner_utf_1 = [[ +% filename : enco-utf.tex +% comment : generated by mtxrun --script chars --utf +% author : Hans Hagen, PRAGMA-ADE, Hasselt NL +% copyright: PRAGMA ADE / ConTeXt Development Team +% license : see context related readme files + +\ifx\setcclcucx\undefined + + \def\setcclcucx #1 #2 #3 % + {\global\catcode"#1=11 + \global\lccode "#1="#2 + \global\uccode "#1="#3 } + +\fi +]] + +scripts.chars.banner_utf_2 = [[ + +% lc/uc/catcode mappings + +]] + +scripts.chars.banner_utf_3 = [[ + +% named characters mapped onto utf + +]] + +scripts.chars.banner_utf_4 = [[ + +\endinput +]] + +function scripts.chars.makeencoutf() + local chartable = input.find_file(instance,"char-def.lua") or "" + if chartable ~= "" then + dofile(chartable) + if characters and characters.data then + local f = io.open("enco-utf.tex", 'w') + if f then + local char, format = unicode.utf8.char, string.format + f:write(scripts.chars.banner_utf_1) + f:write(scripts.chars.banner_utf_2) + local list = table.sortedkeys(characters.data) + local length = 0 + for i=1,#list do + local code = list[i] + if code <= 0xFFFF then + local chr = characters.data[code] + local cc = chr.category + if cc == 'll' or cc == 'lu' or cc == 'lt' then + if not chr.lccode then chr.lccode = code end + if not chr.uccode then chr.uccode = code end + f:write(format("\\setcclcucx %04X %04X %04X %% %s\n",code,chr.lccode,chr.uccode,chr.description)) + end + if #(chr.contextname or "") > length then + length = #chr.contextname + end + end + end + f:write(scripts.chars.banner_utf_3) + for i=1,#list do + local code = list[i] + if code > 0x7F and code <= 0xFFFF then + local chr = characters.data[code] + if chr.contextname then + f:write(format("\\def\\%s{%s} %% %s\n", chr.contextname:rpadd(length," "), char(code),chr.description)) + end + end + end + f:write(scripts.chars.banner_utf_4) + f:close() + end + end + end +end + banner = banner .. " | character tools " messages.help = [[ --stix convert stix table to math table +--utf generate enco-utf.tex (used by xetex) ]] if environment.argument("stix") then local inname = environment.files[1] or "" local outname = environment.files[2] or "" scripts.chars.stixtomkiv(inname,outname) +elseif environment.argument("utf") then + scripts.chars.makeencoutf() else input.help(banner,messages.help) end diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 2a3a496a3..baad28e84 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -168,7 +168,7 @@ end --~ split = lpeg.Ct(c*(p*c)^0) --~ splitters[separator] = split --~ end ---~ return lpeg.match(split,self) +--~ return lpeg.match(split,self) -- split:match(self) --~ else --~ return { } --~ end @@ -325,7 +325,7 @@ end --~ return self .. self.rep(chr or " ",n-#self) --~ end -function string:padd(n,chr) +function string:rpadd(n,chr) local m = n-#self if m > 0 then return self .. self.rep(chr or " ",m) @@ -334,6 +334,17 @@ function string:padd(n,chr) end end +function string:lpadd(n,chr) + local m = n-#self + if m > 0 then + return self.rep(chr or " ",m) .. self + else + return self + end +end + +string.padd = string.rpadd + function is_number(str) return str:find("^[%-%+]?[%d]-%.?[%d+]$") == 1 end @@ -530,6 +541,8 @@ end do + -- one of my first exercises in lua ... + -- 34.055.092 32.403.326 arabtype.tma -- 1.620.614 1.513.863 lmroman10-italic.tma -- 1.325.585 1.233.044 lmroman10-regular.tma @@ -889,6 +902,25 @@ function table.tohash(t) return h end +function table.contains(t, v) + if t then + for i=1, #t do + if t[i] == v then + return true + end + end + end + return false +end + +function table.count(t) + local n, e = 0, next(t) + while e do + n, e = n + 1, next(t,e) + end + return n +end + --~ function table.are_equal(a,b) --~ return table.serialize(a) == table.serialize(b) --~ end @@ -1387,12 +1419,20 @@ function boolean.tonumber(b) if b then return 1 else return 0 end end -function toboolean(str) - if type(str) == "string" then - return str == "true" or str == "yes" or str == "on" or str == "1" - elseif type(str) == "number" then - return tonumber(str) ~= 0 - elseif type(str) == "nil" then +function toboolean(str,tolerant) + if tolerant then + if type(str) == "string" then + return str == "true" or str == "yes" or str == "on" or str == "1" + elseif type(str) == "number" then + return tonumber(str) ~= 0 + elseif type(str) == "nil" then + return false + else + return str + end + elseif str == "true" then + return true + elseif str == "false" then return false else return str @@ -1427,13 +1467,14 @@ if not modules then modules = { } end modules ['l-xml'] = { license = "see context related readme files" } --- todo: ns, tg = s:match("^(.-):?([^:]+)$") +-- RJ: key=value ... lpeg.Ca(lpeg.Cc({}) * (pattern-producing-key-and-value / rawset)^0) --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides parent access; a first version used different tricky but was less optimized to we -went this route.</p> +went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. +The find based parser can be found in l-xml-edu.lua along with other older code.</p> <p>Expecially the lpath code is experimental, we will support some of xpath, but only things that make sense for us; as compensation it is possible to hook in your @@ -1442,7 +1483,7 @@ this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> files.</p> <typing> -a/b/c /*/c (todo: a/b/(pattern)/d) +a/b/c /*/c a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) </typing> @@ -1457,48 +1498,86 @@ tex = tex or { } xml.trace_lpath = false xml.trace_print = false +xml.trace_remap = false --[[ldx-- -<p>First a hack to enable namespace resolving.</p> +<p>First a hack to enable namespace resolving. A namespace is characterized by +a <l n='url'/>. The following function associates a namespace prefix with a +pattern. We use <l n='lpeg'/>, which in this case is more than twice as fast as a +find based solution where we loop over an array of patterns. Less code and +much cleaner.</p> --ldx]]-- +xml.xmlns = { } + do - xml.xmlns = { } + local parser = lpeg.P(false) -- printing shows that this has no side effects + + --[[ldx-- + <p>The next function associates a namespace prefix with an <l n='url'/>. This + normally happens independent of parsing.</p> - local data = { } + <typing> + xml.registerns("mml","mathml") + </typing> + --ldx]]-- - function xml.registerns(namespace,pattern) - data[#data+1] = { namespace:lower(), pattern:lower() } + function xml.registerns(namespace, pattern) -- pattern can be an lpeg + parser = parser + lpeg.C(lpeg.P(pattern:lower())) / namespace end + --[[ldx-- + <p>The next function also registers a namespace, but this time we map a + given namespace prefix onto a registered one, using the given + <l n='url'/>. This used for attributes like <t>xmlns:m</t>.</p> + + <typing> + xml.checkns("m","http://www.w3.org/mathml") + </typing> + --ldx]]-- + function xml.checkns(namespace,url) - url = url:lower() - for i=1,#data do - local d = data[i] - if url:find(d[2]) then - if namespace ~= d[1] then - xml.xmlns[namespace] = d[1] - end - end + local ns = parser:match(url:lower()) + if ns and namespace ~= ns then + xml.xmlns[namespace] = ns end end + --[[ldx-- + <p>Next we provide a way to turn an <l n='url'/> into a registered + namespace. This used for the <t>xmlns</t> attribute.</p> + + <typing> + resolvedns = xml.resolvens("http://www.w3.org/mathml") + </typing> + + This returns <t>mml</t>. + --ldx]]-- + function xml.resolvens(url) - url = url:lower() - for i=1,#data do - local d = data[i] - if url:find(d[2]) then - return d[1] - end - end - return "" + return parser:match(url:lower()) or "" end + --[[ldx-- + <p>A namespace in an element can be remapped onto the registered + one efficiently by using the <t>xml.xmlns</t> table.</p> + --ldx]]-- + end --[[ldx-- -<p>Next comes the loader. The dreadful doctype comes in many disguises:</p> +<p>This version uses <l n='lpeg'/>. We follow the same approach as before, stack and top and +such. This version is about twice as fast which is mostly due to the fact that +we don't have to prepare the stream for cdata, doctype etc etc. This variant is +is dedicated to Luigi Scarso, who challenged me with 40 megabyte <l n='xml'/> files that +took 12.5 seconds to load (1.5 for file io and the rest for tree building). With +the <l n='lpeg'/> implementation we got that down to less 7.3 seconds. Loading the 14 +<l n='context'/> interface definition files (2.6 meg) went down from 1.05 seconds to 0.55.</p> + +<p>Next comes the parser. The rather messy doctype definition comes in many +disguises so it is no surprice that later on have to dedicate quite some +<l n='lpeg'/> code to it.</p> <typing> <!DOCTYPE Something PUBLIC "... ..." "..." [ ... ] > @@ -1508,320 +1587,466 @@ end <!DOCTYPE Something [ ... ] > <!DOCTYPE Something > </typing> + +<p>The code may look a bit complex but this is mostly due to the fact that we +resolve namespaces and attach metatables. There is only one public function:</p> + +<typing> +local x = xml.convert(somestring) +</typing> + +<p>An optional second boolean argument tells this function not to create a root +element.</p> --ldx]]-- do - -- Loading 12 cont-*.xml and keys-*.xml files totaling to 2.62 MBytes takes 1.1 sec - -- on a windows vista laptop with dual core 7600 (2.3 Ghz), which is not that bad. - -- Of this half time is spent on doctype etc parsing. - - local doctype_patterns = { - "<!DOCTYPE%s+(.-%s+PUBLIC%s+%b\"\"%s+%b\"\"%s+%b[])%s*>", - "<!DOCTYPE%s+(.-%s+PUBLIC%s+%b\"\"%s+%b\"\")%s*>", - "<!DOCTYPE%s+(.-%s+SYSTEM%s+%b\"\"%s+%b[])%s*>", - "<!DOCTYPE%s+(.-%s+SYSTEM%s+%b\"\")%s*>", - "<!DOCTYPE%s+(.-%s%b[])%s*>", - "<!DOCTYPE%s+(.-)%s*>" - } + local remove, nsremap = table.remove, xml.xmlns - -- We assume no "<" which is the lunatic part of the xml spec - -- especially since ">" is permitted; otherwise we need a char - -- by char parser ... more something for later ... normally - -- entities will be used anyway. + local stack, top, dt, at, xmlns, errorstr = {}, {}, {}, {}, {}, nil - -- data = data:gsub(nothing done) is still a copy so we find first + local mt = { __tostring = xml.text } - local function prepare(data,text) - -- pack (for backward compatibility) - if type(data) == "table" then - data = table.concat(data,"") - end - -- CDATA - if data:find("<%!%[CDATA%[") then - data = data:gsub("<%!%[CDATA%[(.-)%]%]>", function(txt) - text[#text+1] = txt or "" - return string.format("<@cd@>%s</@cd@>",#text) - end) - end - -- DOCTYPE - if data:find("<!DOCTYPE ") then - data = data:gsub("^(.-)(<[^%!%?])", function(a,b) - if a:find("<!DOCTYPE ") then - for _,v in ipairs(doctype_patterns) do - a = a:gsub(v, function(d) - text[#text+1] = d or "" - return string.format("<@dd@>%s</@dd@>",#text) - end) - end - end - return a .. b - end,1) + local function add_attribute(namespace,tag,value) + if tag == "xmlns" then + xmlns[#xmlns+1] = xml.resolvens(value) + at[tag] = value + elseif ns == "xmlns" then + xml.checkns(tag,value) + at["xmlns:" .. tag] = value + else + at[tag] = value end - -- comment / does not catch doctype - data = data:gsub("<%!%-%-(.-)%-%->", function(txt) - text[#text+1] = txt or "" - return string.format("<@cm@>%s</@cm@>",#text) - end) - -- processing instructions / altijd 1 - data = data:gsub("<%?(.-)%?>", function(txt) - text[#text+1] = txt or "" - return string.format("<@pi@>%s</@pi@>",#text) - end) - return data, text end + local function add_begin(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = { ns=namespace or "", nr=resolved, tg=tag, at=at, dt={}, __p__ = stack[#stack] } + setmetatable(top, mt) + dt = top.dt + stack[#stack+1] = top + at = { } + end + local function add_end(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local toclose = remove(stack) + top = stack[#stack] + if #stack < 1 then + errorstr = string.format("nothing to close with %s", tag) + elseif toclose.tg ~= tag then -- no namespace check + errorstr = string.format("unable to close %s with %s", toclose.tg, tag) + end + dt = top.dt + dt[#dt+1] = toclose + if at.xmlns then + remove(xmlns) + end + end + local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + setmetatable(top, mt) + dt = top.dt + dt[#dt+1] = { ns=namespace or "", nr=resolved, tg=tag, at=at, dt={}, __p__ = top } + at = { } + if at.xmlns then + remove(xmlns) + end + end + local function add_text(text) + dt[#dt+1] = text + end + local function add_special(what, spacing, text) + if #spacing > 0 then + dt[#dt+1] = spacing + end + top = stack[#stack] + setmetatable(top, mt) + dt[#dt+1] = { special=true, ns="", tg=what, dt={text} } + end + local function set_message(txt) + errorstr = "garbage at the end of the file: " .. txt:gsub("([ \n\r\t]*)","") + end + + local space = lpeg.S(' \r\n\t') + local open = lpeg.P('<') + local close = lpeg.P('>') + local squote = lpeg.S("'") + local dquote = lpeg.S('"') + local equal = lpeg.P('=') + local slash = lpeg.P('/') + local colon = lpeg.P(':') + local valid = lpeg.R('az', 'AZ', '09') + lpeg.S('_-.') + local name_yes = lpeg.C(valid^1) * colon * lpeg.C(valid^1) + local name_nop = lpeg.C(lpeg.P(true)) * lpeg.C(valid^1) + local name = name_yes + name_nop + + local utfbom = lpeg.P('\000\000\254\255') + lpeg.P('\255\254\000\000') + + lpeg.P('\255\254') + lpeg.P('\254\255') + lpeg.P('\239\187\191') -- no capture + + local spacing = lpeg.C(space^0) + local justtext = lpeg.C((1-open)^1) + local somespace = space^1 + local optionalspace = space^0 + + local value = (squote * lpeg.C((1 - squote)^0) * squote) + (dquote * lpeg.C((1 - dquote)^0) * dquote) + local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute + local attributes = attribute^0 + + local text = justtext / add_text + local balanced = lpeg.P { "[" * ((1 - lpeg.S"[]") + lpeg.V(1))^0 * "]" } -- taken from lpeg manual, () example + + local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty + local beginelement = (spacing * open * name * attributes * optionalspace * close) / add_begin + local endelement = (spacing * open * slash * name * optionalspace * close) / add_end + + local begincomment = open * lpeg.P("!--") + local endcomment = lpeg.P("--") * close + local begininstruction = open * lpeg.P("?") + local endinstruction = lpeg.P("?") * close + local begincdata = open * lpeg.P("![CDATA[") + local endcdata = lpeg.P("]]") * close + + local someinstruction = lpeg.C((1 - endinstruction)^0) + local somecomment = lpeg.C((1 - endcomment )^0) + local somecdata = lpeg.C((1 - endcdata )^0) + + local begindoctype = open * lpeg.P("!DOCTYPE") + local enddoctype = close + local publicdoctype = lpeg.P("PUBLIC") * somespace * value * somespace * value * somespace * balanced^0 + local systemdoctype = lpeg.P("SYSTEM") * somespace * value * somespace * balanced^0 + local simpledoctype = (1-close)^1 * balanced^0 + local somedoctype = lpeg.C((somespace * lpeg.P(publicdoctype + systemdoctype + simpledoctype) * optionalspace)^0) + + local instruction = (spacing * begininstruction * someinstruction * endinstruction) / function(...) add_special("@pi@",...) end + local comment = (spacing * begincomment * somecomment * endcomment ) / function(...) add_special("@cm@",...) end + local cdata = (spacing * begincdata * somecdata * endcdata ) / function(...) add_special("@cd@",...) end + local doctype = (spacing * begindoctype * somedoctype * enddoctype ) / function(...) add_special("@dd@",...) end + + -- nicer but slower: + -- + -- local instruction = (lpeg.Cc("@pi@") * spacing * begininstruction * someinstruction * endinstruction) / add_special + -- local comment = (lpeg.Cc("@cm@") * spacing * begincomment * somecomment * endcomment ) / add_special + -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special + -- local doctype = (lpeg.Cc("@dd@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special + + local trailer = space^0 * (justtext/set_message)^0 - -- maybe we will move the @tg@ stuff to a dedicated key, say 'st'; this will speed up - -- serializing and testing + -- comment + emptyelement + text + cdata + instruction + lpeg.V("parent"), -- 6.5 seconds on 40 MB database file + -- text + comment + emptyelement + cdata + instruction + lpeg.V("parent"), -- 5.8 + -- text + lpeg.V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 - function xml.convert(data,no_root,collapse) - local crap = { } - data, crap = prepare(data, crap) - local nsremap = xml.xmlns - local remove = table.remove - local stack, top = {}, {} - local i, j, errorstr = 1, 1, nil + + local grammar = lpeg.P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * lpeg.V("parent") * trailer, + parent = beginelement * lpeg.V("children")^0 * endelement, + children = text + lpeg.V("parent") + emptyelement + comment + cdata + instruction, + } + + function xml.convert(data, no_root) -- no collapse any more + stack, top, at, xmlns, errorstr, result = {}, {}, {}, {}, nil, nil stack[#stack+1] = top top.dt = { } - local dt = top.dt - local id = 0 - local namespaces = { } - local mt = { __tostring = xml.text } - while true do - local ni, first, attributes, last, fulltag - ni, j, first, fulltag, attributes, last = data:find("<(/-)([^%s%>/]+)%s*([^>]-)%s*(/-)>", j) - if not ni then break end - local namespace, tag = fulltag:match("^(.-):(.+)$") - if attributes ~= "" then - local t = {} - for ns, tag, _, value in attributes:gmatch("(%w-):?(%w+)=([\"\'])(.-)%3") do - if tag == "xmlns" then -- not ok yet - namespaces[#stack] = xml.resolvens(value) - elseif ns == "" then - t[tag] = value - elseif ns == "xmlns" then - xml.checkns(tag,value) - else - t[tag] = value - end - end - attributes = t - else - attributes = { } - end - if namespace then -- realtime remapping - namespace = nsremap[namespace] or namespace - else - namespace, tag = namespaces[#stack] or "", fulltag - end - local text = data:sub(i, ni-1) - if text == "" or (collapse and text:find("^%s*$")) then - -- no need for empty text nodes, beware, also packs <a>x y z</a> - -- so is not that useful unless used with empty elements - else - dt[#dt+1] = text - end - if first == "/" then - -- end tag - local toclose = remove(stack) -- remove top - top = stack[#stack] - namespaces[#stack] = nil - if #stack < 1 then - errorstr = string.format("nothing to close with %s", tag) - break - elseif toclose.tg ~= tag then -- no namespace check - errorstr = string.format("unable to close %s with %s", toclose.tg, tag) - break - end - if tag:find("^@..@$") then - dt[1] = crap[tonumber(dt[1])] or "" - end - dt = top.dt - dt[#dt+1] = toclose - elseif last == "/" then - -- empty element tag - dt[#dt+1] = { ns = namespace, tg = tag, dt = { }, at = attributes, __p__ = top } - -- setmetatable(top, { __tostring = xml.text }) - setmetatable(top, mt) - else - -- begin tag - top = { ns = namespace, tg = tag, dt = { }, at = attributes, __p__ = stack[#stack] } - -- setmetatable(top, { __tostring = xml.text }) - setmetatable(top, mt) - dt = top.dt - stack[#stack+1] = top - end - i = j + 1 - end - if not errorstr then - local text = data:sub(i) - if dt and not text:find("^%s*$") then - dt[#dt+1] = text - end - if #stack > 1 then - errorstr = string.format("unclosed %s", stack[#stack].tg) - end + dt = top.dt + if not data or data == "" then + errorstr = "empty xml file" + elseif not grammar:match(data) then + errorstr = "invalid xml file" end if errorstr then - stack = { { tg = "error", dt = { errorstr } } } - -- setmetatable(stack, { __tostring = xml.text }) + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={} } } } setmetatable(stack, mt) - end - if no_root then - return stack[1] + if xml.error_handler then xml.error_handler("load",errorstr) end else - local t = { ns = "", tg = '@rt@', dt = stack[1].dt } - -- setmetatable(t, { __tostring = xml.text }) - setmetatable(t, mt) - for k,v in ipairs(t.dt) do - if type(v) == "table" and v.tg ~= "@pi@" and v.tg ~= "@dd@" and v.tg ~= "@cm@" then - t.ri = k -- rootindex + result = stack[1] + end + if not no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={} } + setmetatable(result, mt) + for k,v in ipairs(result.dt) do + if type(v) == "table" and not v.special then -- always table -) + result.ri = k -- rootindex break end end - return t end + return result end - function xml.copy(old,tables,parent) -- fast one - tables = tables or { } - if old then - local new = { } - if not table[old] then - table[old] = new - end - for i,v in pairs(old) do - -- new[i] = (type(v) == "table" and (table[v] or xml.copy(v, tables, table))) or v - if type(v) == "table" then - new[i] = table[v] or xml.copy(v, tables, table) - else - new[i] = v - end - end - local mt = getmetatable(old) - if mt then - setmetatable(new,mt) - end - return new - else - return { } - end + --[[ldx-- + <p>Packaging data in an xml like table is done with the following + function. Maybe it will go away (when not used).</p> + --ldx]]-- + + function xml.package(tag,attributes,data) + local ns, tg = tag:match("^(.-):?([^:]+)$") + local t = { ns = ns, tg = tg, dt = data or "", at = attributes or {} } + setmetatable(t, mt) + return t end + xml.error_handler = (logs and logs.report) or print + end -function xml.load(filename,collapse) +--[[ldx-- +<p>We cannot load an <l n='lpeg'/> from a filehandle so we need to load +the whole file first. The function accepts a string representing +a filename or a file handle.</p> +--ldx]]-- + +function xml.load(filename) if type(filename) == "string" then - local root, f = { }, io.open(filename,'r') -- no longer 'rb' + local root, f = { }, io.open(filename,'r') if f then - root = xml.convert(f:read("*all"),false,collapse) + root = xml.convert(f:read("*all")) f:close() + else + -- if we want an error: root = xml.convert("") end - return root + return root -- no nil but an empty table if it fails else - return xml.convert(filename:read("*all"),false,collapse) + return xml.convert(filename:read("*all")) end end -function xml.root(root) - return (root.ri and root.dt[root.ri]) or root +--[[ldx-- +<p>When we inject new elements, we need to convert strings to +valid trees, which is what the next function does.</p> +--ldx]]-- + +function xml.toxml(data) + if type(data) == "string" then + local root = { xml.convert(data,true) } + return (#root > 1 and root) or root[1] + else + return data + end end -function xml.toxml(data,collapse) - local t = { xml.convert(data,true,collapse) } - if #t > 1 then - return t +--[[ldx-- +<p>For copying a tree we use a dedicated function instead of the +generic table copier. Since we know what we're dealing with we +can speed up things a bit. The second argument is not to be used!</p> +--ldx]]-- + +function xml.copy(old,tables) + if old then + tables = tables or { } + local new = { } + if not tables[old] then + tables[old] = new + end + for k,v in pairs(old) do + new[k] = (type(v) == "table" and (tables[v] or xml.copy(v, tables))) or v + end + local mt = getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new else - return t[1] + return { } end end -function xml.serialize(e, handle, textconverter, attributeconverter) - handle = handle or (tex and tex.sprint) or io.write - if not e then - -- quit - elseif e.command and xml.command then -- test for command == "" ? - xml.command(e) - elseif e.tg then - local format, serialize = string.format, xml.serialize - local ens, etg, eat, edt = e.ns, e.tg, e.at, e.dt - -- no spaces, so no flush needed (check) - if etg == "@pi@" then - handle(format("<?%s?>",edt[1])) - elseif etg == "@cm@" then - handle(format("<!--%s-->",edt[1])) - elseif etg == "@cd@" then - handle(format("<![CDATA[%s]]>",edt[1])) - elseif etg == "@dd@" then - handle(format("<!DOCTYPE %s>",edt[1])) - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter) +--[[ldx-- +<p>In <l n='context'/> serializing the tree or parts of the tree is a major +actitivity which is why the following function is pretty optimized resulting +in a few more lines of code than needed. The variant that uses the formatting +function for all components is about 15% slower than the concatinating +alternative.</p> +--ldx]]-- + +do + + -- todo: add <?xml version='1.0' standalone='yes'?> when not present + + local fallbackhandle = (tex and tex.sprint) or io.write + + function xml.serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) + if not e then + -- quit + elseif not nocommands and e.command and xml.command then + xml.command(e) else - local ats = eat and next(eat) and { } - if ats then - if attributeconverter then - for k,v in pairs(eat) do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) + handle = handle or fallbackhandle + local etg = e.tg + if etg then + -- local format = string.format + if e.special then + local edt = e.dt + local spc = specialconverter and specialconverter[etg] + if spc then + local result = spc(edt[1]) + if result then + handle(result) + else + -- no need to handle any further + end + elseif etg == "@pi@" then + -- handle(format("<?%s?>",edt[1])) + handle("<?" .. edt[1] .. "?>") -- maybe table.join(edt) + elseif etg == "@cm@" then + -- handle(format("<!--%s-->",edt[1])) + handle("<!--" .. edt[1] .. "-->") + elseif etg == "@cd@" then + -- handle(format("<![CDATA[%s]]>",edt[1])) + handle("<![CDATA[" .. edt[1] .. "]]>") + elseif etg == "@dd@" then + -- handle(format("<!DOCTYPE %s>",edt[1])) + handle("<!DOCTYPE " .. edt[1] .. ">") + elseif etg == "@rt@" then + xml.serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) end else - for k,v in pairs(eat) do - ats[#ats+1] = format('%s=%q',k,v) - end - end - end - if ens ~= "" then - if edt and #edt > 0 then + local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } if ats then - handle(format("<%s:%s %s>",ens,etg,table.concat(ats," "))) - else - handle(format("<%s:%s>",ens,etg)) + local format = string.format + if attributeconverter then + for k,v in pairs(eat) do + ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) + end + else + for k,v in pairs(eat) do + ats[#ats+1] = format('%s=%q',k,v) + end + end end - for i=1,#edt do - serialize(edt[i],handle,textconverter,attributeconverter) + if ern and xml.trace_remap then + if ats then + ats[#ats+1] = string.format("xmlns:remapped='%s'",ern) + else + ats = { string.format("xmlns:remapped='%s'",ern) } + end end - handle(format("</%s:%s>",ens,etg)) - else - if ats then - handle(format("<%s:%s %s/>",ens,etg,table.concat(ats," "))) + if ens ~= "" then + if edt and #edt > 0 then + if ats then + -- handle(format("<%s:%s %s>",ens,etg,table.concat(ats," "))) + handle("<" .. ens .. ":" .. etg .. " " .. table.concat(ats," ") .. ">") + else + -- handle(format("<%s:%s>",ens,etg)) + handle("<" .. ens .. ":" .. etg .. ">") + end + local serialize = xml.serialize + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + if textconverter then + handle(textconverter(e)) + else + handle(e) + end + else + serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) + end + end + -- handle(format("</%s:%s>",ens,etg)) + handle("</" .. ens .. ":" .. etg .. ">") + else + if ats then + -- handle(format("<%s:%s %s/>",ens,etg,table.concat(ats," "))) + handle("<%" .. ens .. ":" .. etg .. table.concat(ats," ") .. "/>") + else + -- handle(format("<%s:%s/>",ens,etg)) + handle("<%" .. ens .. ":" .. "/>") + end + end else - handle(format("<%s:%s/>",ens,etg)) + if edt and #edt > 0 then + if ats then + -- handle(format("<%s %s>",etg,table.concat(ats," "))) + handle("<" .. etg .. " " .. table.concat(ats," ") .. ">") + else + -- handle(format("<%s>",etg)) + handle("<" .. etg .. ">") + end + local serialize = xml.serialize + for i=1,#edt do + serialize(edt[i],handle,textconverter,attributeconverter,specialconverter,nocommands) + end + -- handle(format("</%s>",etg)) + handle("</" .. etg .. ">") + else + if ats then + -- handle(format("<%s %s/>",etg,table.concat(ats," "))) + handle("<" .. etg .. table.concat(ats," ") .. "/>") + else + -- handle(format("<%s/>",etg)) + handle("<" .. etg .. "/>") + end + end end end - else - if edt and #edt > 0 then - if ats then - handle(format("<%s %s>",etg,table.concat(ats," "))) - else - handle(format("<%s>",etg)) - end - for i=1,#edt do - serialize(edt[i],handle,textconverter,attributeconverter) - end - handle(format("</%s>",etg)) + elseif type(e) == "string" then + if textconverter then + handle(textconverter(e)) else - if ats then - handle(format("<%s %s/>",etg,table.concat(ats," "))) - else - handle(format("<%s/>",etg)) - end + handle(e) + end + else + local serialize = xml.serialize + for i=1,#e do + serialize(e[i],handle,textconverter,attributeconverter,specialconverter,nocommands) end end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - for i=1,#e do - xml.serialize(e[i],handle,textconverter,attributeconverter) + end + + function xml.checkbom(root) + if root.ri then + local dt, found = root.dt, false + for k,v in ipairs(dt) do + if type(v) == "table" and v.special and v.tg == "@pi" and v.dt:find("xml.*version=") then + found = true + break + end + end + if not found then + table.insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + table.insert(dt, 2, "\n" ) + end end end + end -function xml.string(e,handle) -- weird one that may become obsolete - if e.tg then +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +function xml.tostring(root) -- 25% overhead due to collecting + if root then + if type(root) == 'string' then + return root + elseif next(root) then + local result = { } + xml.serialize(root,function(s) result[#result+1] = s end) + return table.concat(result,"") + end +end + return "" +end + +--[[ldx-- +<p>The next function operated on the content only and needs a handle function +that accepts a string.</p> +--ldx]]-- + +function xml.string(e,handle) + if not handle or (e.special and e.tg ~= "@rt@") then + -- nothing + elseif e.tg then local edt = e.dt if edt then for i=1,#edt do @@ -1833,6 +2058,21 @@ function xml.string(e,handle) -- weird one that may become obsolete end end +--[[ldx-- +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>The save function is given below.</p> +--ldx]]-- + function xml.save(root,name) local f = io.open(name,"w") if f then @@ -1841,535 +2081,67 @@ function xml.save(root,name) end end -function xml.stringify(root) - if root then - if type(root) == 'string' then - return root - elseif next(root) then - local result = { } - xml.serialize(root,function(s) result[#result+1] = s end) - return table.concat(result,"") - end - end - return "" -end - -xml.tostring = xml.stringify - -do - - -- print - - local newline = lpeg.P("\n") - local space = lpeg.P(" ") - local content = lpeg.C((1-newline)^1) - - if tex then - - -- taco: we need a kind of raw print into tex, i.e. embedded \n's become lineendings - -- for tex and an empty line a par; could be a c-wrapper around existing stuff; i - -- played a lot with tex.print but that does not work ok (should be obeylines save) - - local buffer = {} - - local function cprint(s) - buffer[#buffer+1] = s - end - local function nprint( ) - if #buffer > 0 then - if xml.trace_print then - texio.write_nl(string.format("tex.print : [[[%s]]]", table.join(buffer))) - end - tex.print(table.join(buffer)) - buffer = {} - else - if xml.trace_print then - texio.write_nl(string.format("tex.print : [[[%s]]]", "")) - end - tex.print("") - end - end - local function fprint() - if #buffer > 0 then - if xml.trace_print then - texio.write_nl(string.format("tex.sprint: [[[%s]]]", table.join(buffer))) - end - tex.sprint(table.join(buffer)) - buffer = { } - end - end - - local line_n = newline / nprint - local line_c = content / cprint - local capture = (line_n + line_c)^0 - - local function sprint(root) - if not root then - -- quit - elseif type(root) == 'string' then - lpeg.match(capture,root) - elseif next(root) then - xml.serialize(root, sprint, nil, nil, true) - end - end - - function xml.sprint(root) - buffer = {} - sprint(root) - if #buffer > 0 then - nprint() - end - end - - xml.sflush = fprint - - else - - function xml.sprint(root) - if not root then - -- quit - elseif type(root) == 'string' then - print(root) - elseif next(root) then - xml.serialize(root, xml.sprint, nil, nil, true) - end - end - - end - - function xml.tprint(root) - if type(root) == "table" then - for i=1,#root do - xml.sprint(root[i]) - end - elseif type(root) == "string" then - xml.sprint(root) - end - end - - -- lines (looks hackery, but we cannot pass variables in capture functions) - - local buffer, flush = {}, nil - - local function cprint(s) - buffer[#buffer+1] = s - end - local function nprint() - flush() - end - - local line_n = newline / nprint - local line_c = content / cprint - local capture = (line_n + line_c)^0 - - function lines(root) - if not root then - -- quit - elseif type(root) == 'string' then - lpeg.match(capture,root) - elseif next(root) then - xml.serialize(root, lines) - end - end - - function xml.lines(root) - local result = { } - flush = function() - result[#result+1] = table.join(buffer) - buffer = { } - end - buffer = {} - lines(root) - if #buffer > 0 then - result[#result+1] = table.join(buffer) - end - return result - end +--[[ldx-- +<p>A few helpers:</p> +--ldx]]-- +function xml.body(root) + return (root.ri and root.dt[root.ri]) or root end function xml.text(root) - return (root and xml.stringify(root)) or "" + return (root and xml.tostring(root)) or "" end function xml.content(root) return (root and root.dt and xml.tostring(root.dt)) or "" end -function xml.body(t) -- removes initial pi - if t and t.dt and t.tg == "@rt@" then - for k,v in ipairs(t.dt) do - if type(v) == "table" and v.tg ~= "@pi@" then - return v - end - end - end - return t -end +--[[ldx-- +<p>The next helper erases an element but keeps the table as it is, +and since empty strings are not serialized (effectively) it does +not harm. Copying the table would take more time. Usage:</p> --- call: e[k] = xml.empty() or xml.empty(e,k) +<typing> +dt[k] = xml.empty() or xml.empty(dt,k) +</typing> +--ldx]]-- -function xml.empty(e,k) -- erases an element but keeps the table intact - if e and k then - e[k] = "" - return e[k] +function xml.empty(dt,k) + if dt and k then + dt[k] = "" + return dt[k] else return "" end end --- call: e[k] = xml.assign(t) or xml.assign(e,k,t) +--[[ldx-- +<p>The next helper assigns a tree (or string). Usage:</p> + +<typing> +dt[k] = xml.assign(root) or xml.assign(dt,k,root) +</typing> +--ldx]]-- -function xml.assign(e,k,t) -- assigns xml tree / more testing will be done - if e and k then - if type(t) == "table" then - e[k] = xml.body(t) - else - e[k] = t -- no parsing - end - return e[k] +function xml.assign(dt,k,root) + if dt and k then + dt[k] = (type(root) == "table" and xml.body(root)) or root + return dt[k] else - return xml.body(t) + return xml.body(root) end end --- 0=nomatch 1=match 2=wildcard 3=ancestor - --- "tag" --- "tag1/tag2/tag3" --- "*/tag1/tag2/tag3" --- "/tag1/tag2/tag3" --- "/tag1/tag2|tag3" --- "tag[@att='value'] --- "tag1|tag2[@att='value'] - -function xml.tag(e) - return e.tg or "" -end - -function xml.att(e,a) - return (e.at and e.at[a]) or "" -end - -xml.attribute = xml.att - ---~ local cache = { } - ---~ local function f_fault ( ) return 0 end ---~ local function f_wildcard( ) return 2 end ---~ local function f_result (b) if b then return 1 else return 0 end end - ---~ function xml.lpath(str) --maybe @rt@ special ---~ if not str or str == "" then ---~ str = "*" ---~ end ---~ local m = cache[str] ---~ if not m then ---~ -- todo: text() ---~ if type(str) == "table" then ---~ if xml.trace_lpath then print("lpath", "table" , "inherit") end ---~ m = str ---~ elseif str == "/" then ---~ if xml.trace_lpath then print("lpath", "/", "root") end ---~ m = false ---~ elseif str == "*" then ---~ if xml.trace_lpath then print("lpath", "no string or *", "wildcard") end ---~ m = true ---~ else ---~ str = str:gsub("^//","") -- any ---~ if str == "" then ---~ if xml.trace_lpath then print("lpath", "//", "wildcard") end ---~ m = true ---~ else ---~ m = { } ---~ if not str:find("^/") then ---~ m[1] = 2 ---~ end ---~ for v in str:gmatch("([^/]+)") do ---~ if v == "" or v == "*" then ---~ if #m > 0 then -- when not, then we get problems with root being second (after <?xml ...?> (we could start at dt[2]) ---~ if xml.trace_lpath then print("lpath", "empty or *", "wildcard") end ---~ m[#m+1] = 2 ---~ end ---~ elseif v == ".." then ---~ if xml.trace_lpath then print("lpath", "..", "ancestor") end ---~ m[#m+1] = 3 ---~ else ---~ local a, b = v:match("^(.+)::(.-)$") ---~ if a and b then ---~ if a == "ancestor" then ---~ if xml.trace_lpath then print("lpath", a, "ancestor") end ---~ m[#m+1] = 3 ---~ -- todo: b ---~ elseif a == "pi" then ---~ if xml.trace_lpath then print("lpath", a, "processing instruction") end ---~ local expr = "^" .. b .. " " ---~ m[#m+1] = function(e) ---~ if e.tg == '@pi@' and e.dt[1]:find(expr) then ---~ return 6 ---~ else ---~ return 0 ---~ end ---~ end ---~ end ---~ else ---~ local n, a, t = v:match("^(.-)%[@(.-)=(.-)%]$") ---~ if n and a and t then ---~ -- todo: namespace, negate ---~ -- t = t:gsub("^\'(.*)\'$", "%1") ---~ -- t = t:gsub("^\"(.*)\"$", "%1") ---~ -- t = t:sub(2,-2) -- "" or '' mandate ---~ t = t:gsub("^([\'\"])(.-)%1$", "%2") ---~ if n:find("|") then ---~ local tt = n:split("|") ---~ if xml.trace_lpath then print("lpath", "match", t, n) end ---~ m[#m+1] = function(e,i) ---~ for i=1,#tt do ---~ if e.at and e.tg == tt[i] and e.at[a] == t then return 1 end ---~ end ---~ return 0 ---~ end ---~ else ---~ if xml.trace_lpath then print("lpath", "match", t, n) end ---~ m[#m+1] = function(e) ---~ if e.at and e.ns == s and e.tg == n and e.at[a] == t then ---~ return 1 ---~ else ---~ return 0 ---~ end ---~ end ---~ end ---~ else -- todo, better tracing (string.format, ook negate etc) ---~ local negate = v:sub(1,1) == '^' ---~ if negate then v = v:sub(2) end ---~ if v:find("|") then ---~ local t = { } ---~ for s in v:gmatch("([^|]+)") do ---~ local ns, tg = s:match("^(.-):(.+)$") ---~ if tg == "*" then ---~ if xml.trace_lpath then print("lpath", "or wildcard", ns, tg) end ---~ t[#t+1] = function(e) return e.ns == ns end ---~ elseif tg then ---~ if xml.trace_lpath then print("lpath", "or match", ns, tg) end ---~ t[#t+1] = function(e) return e.ns == ns and e.tg == tg end ---~ else ---~ if xml.trace_lpath then print("lpath", "or match", s) end ---~ t[#t+1] = function(e) return e.ns == "" and e.tg == s end ---~ end ---~ end ---~ if negate then ---~ m[#m+1] = function(e) ---~ for i=1,#t do if t[i](e) then return 0 end end return 1 ---~ end ---~ else ---~ m[#m+1] = function(e) ---~ for i=1,#t do if t[i](e) then return 1 end end return 0 ---~ end ---~ end ---~ else ---~ if xml.trace_lpath then print("lpath", "match", v) end ---~ local ns, tg = v:match("^(.-):(.+)$") ---~ if not tg then ns, tg = "", v end ---~ if tg == "*" then ---~ if ns ~= "" then ---~ m[#m+1] = function(e) ---~ if ns == e.ns then return 1 else return 0 end ---~ end ---~ end ---~ elseif negate then ---~ m[#m+1] = function(e) ---~ if ns == e.ns and tg == e.tg then return 0 else return 1 end ---~ end ---~ else ---~ m[#m+1] = function(e) ---~ if ns == e.ns and tg == e.tg then return 1 else return 0 end ---~ end ---~ end ---~ end ---~ end ---~ end ---~ end ---~ end ---~ end ---~ end ---~ if xml.trace_lpath then ---~ print("# lpath criteria:", (type(m) == "table" and #m) or "none") ---~ end ---~ cache[str] = m ---~ end ---~ return m ---~ end - ---~ -- if handle returns true, then quit - ---~ function xml.traverse(root,pattern,handle,reverse,index,wildcard) ---~ if not root then -- error ---~ return false ---~ elseif pattern == false then -- root ---~ handle(root,root.dt,root.ri) ---~ return false ---~ elseif pattern == true then -- wildcard ---~ local traverse = xml.traverse ---~ local rootdt = root.dt ---~ if rootdt then ---~ local start, stop, step = 1, #rootdt, 1 ---~ if reverse then ---~ start, stop, step = stop, start, -1 ---~ end ---~ for k=start,stop,step do ---~ if handle(root,rootdt,root.ri or k) then return false end ---~ if not traverse(rootdt[k],true,handle,reverse) then return false end ---~ end ---~ end ---~ return false ---~ elseif root and root.dt then ---~ index = index or 1 ---~ local match = pattern[index] or f_wildcard ---~ local traverse = xml.traverse ---~ local rootdt = root.dt ---~ local start, stop, step = 1, #rootdt, 1 ---~ if reverse and index == #pattern then -- maybe no index test here / error? ---~ start, stop, step = stop, start, -1 ---~ end ---~ for k=start,stop,step do ---~ local e = rootdt[k] ---~ if e.tg then ---~ local m = (type(match) == "function" and match(e,root)) or match ---~ if m == 1 then -- match ---~ if index < #pattern then ---~ if not traverse(e,pattern,handle,reverse,index+1) then return false end ---~ else ---~ if handle(root,rootdt,root.ri or k) then ---~ return false ---~ end ---~ -- tricky, where do we pick up, is this ok now ---~ if pattern[1] == 2 then -- start again with new root (we need a way to inhibit this) ---~ if not traverse(e,pattern,handle,reverse,1) then return false end ---~ end ---~ end ---~ elseif m == 2 then -- wildcard ---~ if index < #pattern then ---~ -- <parent><a><b></b><c></c></a></parent> : "a" (true) "/a" (true) "b" (true) "/b" (false) ---~ -- not good yet, we need to pick up any prev level which is 2 ---~ local p = pattern[2] ---~ if index == 1 and p then ---~ local mm = (type(p) == "function" and p(e,root)) or p -- pattern[2](e,root) ---~ if mm == 1 then ---~ if #pattern == 2 then ---~ if handle(root,rootdt,k) then ---~ return false ---~ end ---~ -- hack ---~ if pattern[1] == 2 then -- start again with new root (we need a way to inhibit this) ---~ if not traverse(e,pattern,handle,reverse,1) then return false end ---~ end ---~ else ---~ if not traverse(e,pattern,handle,reverse,3) then return false end ---~ end ---~ else ---~ if not traverse(e,pattern,handle,reverse,index+1,true) then return false end ---~ end ---~ else ---~ if not traverse(e,pattern,handle,reverse,index+1,true) then return false end ---~ end ---~ elseif handle(root,rootdt,k) then ---~ return false ---~ end ---~ elseif m == 3 then -- ancestor ---~ local ep = e.__p__ ---~ if index < #pattern then ---~ if not traverse(ep,pattern,handle,reverse,index+1) then return false end ---~ elseif handle(root,rootdt,k) then ---~ return false ---~ end ---~ elseif m == 4 then -- just root ---~ if handle(root,rootdt,k) then ---~ return false ---~ end ---~ elseif m == 6 then -- pi ---~ if handle(root,rootdt,k) then ---~ return false ---~ end ---~ elseif wildcard then -- maybe two kind of wildcards: * ** // ---~ if not traverse(e,pattern,handle,reverse,index,wildcard) then return false end ---~ end ---~ end ---~ end ---~ end ---~ return true ---~ end - ---~ Y a/b ---~ Y /a/b ---~ Y a/*/b ---~ Y a//b ---~ Y child:: ---~ Y .// ---~ Y .. ---~ N id("tag") ---~ Y parent:: ---~ Y child:: ---~ N preceding-sibling:: (same name) ---~ N following-sibling:: (same name) ---~ N preceding-sibling-of-self:: (same name) ---~ N following-sibling-or-self:: (same name) ---~ Y ancestor:: ---~ N descendent:: ---~ N preceding:: ---~ N following:: ---~ N self::node() ---~ N node() == alles ---~ N a[position()=5] ---~ Y a[5] ---~ Y a[-5] ---~ N a[first()] ---~ N a[last()] ---~ Y a/(b|c|d)/e/f ---~ N (c/d|e) ---~ Y a/b[@bla] ---~ Y a/b[@bla='oeps'] ---~ Y a/b[@bla=='oeps'] ---~ Y a/b[@bla<>'oeps'] ---~ Y a/b[@bla!='oeps'] ---~ Y a/b/@bla - ---~ Y ^/a/c (root) ---~ Y ^^/a/c (docroot) ---~ Y root::a/c (docroot) - ---~ no wild card functions (yet) - ---~ s = "/a//b/*/(c|d|e)/(f|g)/h[4]/h/child::i/j/(a/b)/p[-1]/q[4]/ancestor::q/r/../s/./t[@bla='true']/k" - --- // == /**/ --- / = ^ (root) +--[[ldx-- +<p>We've now arrived at an intersting part: accessing the tree using a subset +of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We +will explain more about its usage in other documents.</p> +--ldx]]-- do - function analyze(str) - if not str then - return "" - else - local tmp, result, map, key = { }, { }, { }, str - str = str:gsub("(%b[])", function(s) tmp[#tmp+1] = s return '[['..#tmp..']]' end) - str = str:gsub("(%b())", function(s) tmp[#tmp+1] = s return '[['..#tmp..']]' end) - str = str:gsub("(%^+)([^/])", "%1/%2") - str = str:gsub("//+", "/**/") - str = str:gsub(".*root::", "^/") - str = str:gsub("child::", "") - str = str:gsub("ancestor::", "../") - str = str:gsub("self::", "./") - str = str:gsub("^/", "^/") - for s in str:gmatch("([^/]+)") do - s = s:gsub("%[%[(%d+)%]%]",function(n) return tmp[tonumber(n)] end) - result[#result+1] = s - end - cache[key] = result - return result - end - end - - actions = { + local actions = { [10] = "stay", [11] = "parent", [12] = "subtree root", @@ -2381,112 +2153,168 @@ do [21] = "match one of", [22] = "match and attribute eq", [23] = "match and attribute ne", - [23] = "match and attribute present", + [24] = "match one of and attribute eq", + [25] = "match one of and attribute ne", + [27] = "has attribute", + [28] = "has value", + [29] = "fast match", [30] = "select", [40] = "processing instruction", } - function compose(result) - if not result or #result == 0 then + local map = { } + + local space = lpeg.S(' \r\n\t') + local squote = lpeg.S("'") + local dquote = lpeg.S('"') + local lparent = lpeg.P('(') + local rparent = lpeg.P(')') + local atsign = lpeg.P('@') + local lbracket = lpeg.P('[') + local rbracket = lpeg.P(']') + local exclam = lpeg.P('!') + local period = lpeg.P('.') + local eq = lpeg.P('==') + lpeg.P('=') + local ne = lpeg.P('<>') + lpeg.P('!=') + local star = lpeg.P('*') + local slash = lpeg.P('/') + local colon = lpeg.P(':') + local bar = lpeg.P('|') + local hat = lpeg.P('^') + local valid = lpeg.R('az', 'AZ', '09') + lpeg.S('_-') + local name_yes = lpeg.C(valid^1) * colon * lpeg.C(valid^1) + local name_nop = lpeg.C(lpeg.P(true)) * lpeg.C(valid^1) + local name = name_yes + name_nop + local number = lpeg.C((lpeg.S('+-')^0 * lpeg.R('09')^1)) / tonumber + local names = (bar^0 * name)^1 + local morenames = name * (bar^0 * name)^1 + local instructiontag = lpeg.P('pi::') + local spacing = lpeg.C(space^0) + local somespace = space^1 + local optionalspace = space^0 + local text = lpeg.C(valid^0) + local value = (squote * lpeg.C((1 - squote)^0) * squote) + (dquote * lpeg.C((1 - dquote)^0) * dquote) + local empty = 1-slash + + local is_eq = lbracket * atsign * name * eq * value * rbracket + local is_ne = lbracket * atsign * name * ne * value * rbracket + local is_attribute = lbracket * atsign * name * rbracket + local is_value = lbracket * value * rbracket + local is_number = lbracket * number * rbracket + + local is_one = name + local is_none = exclam * name + local is_one_of = ((lparent * names * rparent) + morenames) + local is_none_of = exclam * ((lparent * names * rparent) + morenames) + + local stay = (period ) + local parent = (period * period ) / function( ) map[#map+1] = { 11 } end + local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end + local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end + local any = (star ) / function( ) map[#map+1] = { 14 } end + local many = (star * star ) / function( ) map[#map+1] = { 15 } end + local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end + + local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end + local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end + local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end + local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end + + local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end + local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end + local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end + local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end + + local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end + local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end + local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end + local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end + + local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end + local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end + local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end + local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end + local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end + local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end + + local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end + local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? + local crap = (1-slash)^1 + + -- a few ugly goodies: + + local docroottag = lpeg.P('^^') / function( ) map[#map+1] = { 12 } end + local subroottag = lpeg.P('^') / function( ) map[#map+1] = { 13 } end + local roottag = lpeg.P('root::') / function( ) map[#map+1] = { 12 } end + local parenttag = lpeg.P('parent::') / function( ) map[#map+1] = { 11 } end + local childtag = lpeg.P('child::') + local selftag = lpeg.P('self::') + + -- there will be more and order will be optimized + + local selector = ( + instruction + + many + any + + parent + stay + + dont_position + position + + dont_match_one_of_and_eq + dont_match_one_of_and_ne + + match_one_of_and_eq + match_one_of_and_ne + + dont_match_and_eq + dont_match_and_ne + + match_and_eq + match_and_ne + + has_attribute + has_value + + dont_match_one_of + match_one_of + + dont_match + match + + crap + empty + ) + + local grammar = lpeg.P { "startup", + startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * lpeg.V("followup"), + followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, + } + + function compose(str) + if not str or str == "" then -- wildcard return true - elseif #result == 1 then - local r = result[1][1] - if r == "14" or r == "15" then - -- wildcard + elseif str == '/' then + -- root + return false + else + map = { } + grammar:match(str) + if #map == 0 then return true - elseif r == "12" then - -- root - return false - end - end - local map = { } - for r=1,#result do - local ri = result[r] - if ri == "." then - -- skip - elseif ri == ".." then - map[#map+1] = { 11 } - elseif ri == "^" then - map[#map+1] = { 12 } - elseif ri == "^^" then - map[#map+1] = { 13 } - elseif ri == "*" then - map[#map+1] = { 14 } - elseif ri == "**" then - map[#map+1] = { 15 } else - local m = ri:match("^%((.*)%)$") -- (a|b|c) - if m or ri:find('|') then - m = m or ri - if m:find("[%[%]%(%)%/]") then -- []()/ - -- error - else - local t = { 21 } - for s in m:gmatch("([^|])") do - local ns, tg = s:match("^(.-):?([^:]+)$") - t[#t+1] = ns - t[#t+1] = tg - end - map[#map+1] = t - end - else - local s, f = ri:match("^(.-)%[%s*(.+)%s*%]$") --aaa[bbb] - if s and f then - local ns, tg = s:match("^(.-):?([^:]+)$") - local at, op, vl = f:match("^@(.-)([!=<>]?)([^!=<>]+)$") -- [@a=='b'] - if op and op ~= "" then - if op == '=' or op == '==' then - map[#map+1] = { 22, ns, tg, at, (vl:gsub("^([\'\"])(.*)%1$", "%2")) } - elseif op == '<>' or op == '!=' then - map[#map+1] = { 23, ns, tg, at, (vl:gsub("^([\'\"])(.*)%1$", "%2")) } - else - -- error - end - elseif f:find("^([%-%+%d]+)$")then - map[#map+1] = { 30, ns, tg, tonumber(f) } - elseif vl ~= "" then - map[#map+1] = { 24, ns, tg, vl } - end - else - local pi = ri:match("^pi::(.-)$") - if pi then - map[#map+1] = { 40, pi } - else - map[#map+1] = { 20, ri:match("^(.-):?([^:]+)$") } - end + local m = map[1][1] + if #map == 1 then + if m == 14 or m == 15 then + -- wildcard + return true + elseif m == 12 then + -- root + return false end + elseif #map == 2 and m == 12 and map[2][1] == 20 then + return { { 29, map[2][2], map[2][3] } } end + if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then + table.insert(map, 1, { 16 }) + end + return map end end - -- if we have a symbol, we can prepend that to the string, which is faster - local mm = map[1] or { } - local r = mm[1] or 0 - if #map == 1 then - if r == 14 or r == 15 then - -- wildcard - return true - elseif r == 12 then - -- root - return false - end - end - if r ~= 11 and r ~= 12 and r ~= 13 and r ~= 14 and r ~= 15 then - table.insert(map, 1, { 16 }) - end - return map end - cache = { } + local cache = { } - function xml.lpath(pattern) + function xml.lpath(pattern,trace) if type(pattern) == "string" then local result = cache[pattern] if not result then - result = compose(analyze(pattern)) + result = compose(pattern) cache[pattern] = result end - if xml.trace_lpath then + if trace or xml.trace_lpath then xml.lshow(result) end return result @@ -2495,23 +2323,58 @@ do end end - function xml.lshow(pattern) + local fallbackreport = (texio and texio.write) or io.write + + function xml.lshow(pattern,report) + report = report or fallbackreport local lp = xml.lpath(pattern) if lp == false then - print("root") + report(" -: root\n") elseif lp == true then - print("wildcard") + report(" -: wildcard\n") else - if type(pattern) ~= "table" then - print("pattern: " .. tostring(pattern)) + if type(pattern) == "string" then + report(string.format("pattern: %s\n",pattern)) end for k,v in ipairs(lp) do - print(k,actions[v[1]],table.join(v," ",2)) + if #v > 1 then + local t = { } + for i=2,#v do + local vv = v[i] + if type(vv) == "string" then + t[#t+1] = (vv ~= "" and vv) or "#" + elseif type(vv) == "boolean" then + t[#t+1] = (vv and "==") or "<>" + end + end + report(string.format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],table.join(t," "))) + else + report(string.format("%2i: %s %s\n", k,v[1],actions[v[1]])) + end end end end - function xml.traverse(root,pattern,handle,reverse,index,wildcard) +end + +--[[ldx-- +<p>An <l n='lpath'/> is converted to a table with instructions for traversing the +tree. Hoever, simple cases are signaled by booleans. Because we don't know in +advance what we want to do with the found element the handle gets three arguments:</p> + +<lines> +<t>r</t> : the root element of the data table +<t>d</t> : the data table of the result +<t>t</t> : the index in the data table of the result +</lines> + +<p> Access to the root and data table makes it possible to construct insert and delete +functions.</p> +--ldx]]-- + +do + + function xml.traverse(root,pattern,handle,reverse,index,parent,wildcard) if not root then -- error return false elseif pattern == false then -- root @@ -2531,103 +2394,172 @@ do end end return false - elseif root and root.dt then + elseif root.dt then index = index or 1 local action = pattern[index] local command = action[1] - if (command == 16 or command == 12) and index == 1 then -- initial - wildcard = true - index = index + 1 - action = pattern[index] - command = action[1] - end - local traverse = xml.traverse - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[4] < 0 then - start, stop, step = stop, start, -1 - dn = -1 + if command == 29 then -- fast case /oeps + local rootdt = root.dt + for k=1,#rootdt do + local e = rootdt[k] + local ns, tg = e.rn or e.ns, e.tg + if ns == action[2] and tg == action[3] then + if handle(root,rootdt,k) then return false end + end end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - local e = rootdt[k] - local ns, tg = e.ns, e.tg - if tg then + elseif command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not xml.traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + if (command == 16 or command == 12) and index == 1 then -- initial + wildcard = true + index = index + 1 + action = pattern[index] + command = action and action[1] or 0 -- something is wrong + end + if command == 11 then -- parent + local ep = root.__p__ or parent + if index < #pattern then + if not xml.traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + else + local traverse = xml.traverse + local rootdt = root.dt + local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 if command == 30 then - if ns == action[2] and tg == action[3] then - n = n + dn - if n == action[4] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,true) then return false end + if action[5] < 0 then + start, stop, step = stop, start, -1 + dn = -1 end - else - local matched = false - if command == 20 then -- match - matched = ns == action[2] and tg == action[3] - elseif command == 21 then -- match one of - for i=2,#action,2 do - if ns == action[i] and tg == action[i+1] then - matched = true - break + elseif reverse and index == #pattern then + start, stop, step = stop, start, -1 + end + for k=start,stop,step do + local e = rootdt[k] + local ns, tg = e.rn or e.ns, e.tg + if tg then + if command == 30 then + local matched = ns == action[3] and tg == action[4] + if action[2] then matched = not matched end + if matched then + n = n + dn + if n == action[5] then + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + break + end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end - end - elseif command == 22 then -- eq - matched = ns == action[2] and tg == action[3] and e.at[action[4]] == action[5] - elseif command == 23 then -- ne - matched = ns == action[2] and tg == action[3] and e.at[action[4]] ~= action[5] - elseif command == 24 then -- present - matched = ns == action[2] and tg == action[3] and e.at[action[4]] - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end else - if not traverse(e,pattern,handle,reverse,index+1,true) then return false end - end - elseif command == 11 then -- parent - local ep = e.__p__ - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - break - elseif command == 40 and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then + local matched, multiple = false, false + if command == 20 then -- match + matched = ns == action[2] and tg == action[3] + if action[2] then matched = not matched end + elseif command == 21 then -- match one of + multiple = true + for i=2,#action,2 do + if ns == action[i] and tg == action[i+1] then matched = true break end + end + if action[2] then matched = not matched end + elseif command == 22 then -- eq + matched = ns == action[3] and tg == action[4] + if action[2] then matched = not matched end + matched = matched and e.at[action[6]] == action[7] + elseif command == 23 then -- ne + matched = ns == action[3] and tg == action[4] + if action[2] then matched = not matched end + matched = mached and e.at[action[6]] ~= action[7] + elseif command == 24 then -- one of eq + multiple = true + for i=3,#action-2,2 do + if ns == action[i] and tg == action[i+1] then matched = true break end + end + if action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] == action[#action] + elseif command == 25 then -- one of ne + multiple = true + for i=3,#action-2,2 do + if ns == action[i] and tg == action[i+1] then matched = true break end + end + if action[2] then matched = not matched end + matched = matched and e.at[action[#action-1]] ~= action[#action] + elseif command == 27 then -- has attribute + local ans = action[3] + matched = ns == action[3] and tg == action[4] + if action[2] then matched = not matched end + matched = matched and e.at[action[5]] + elseif command == 28 then -- has value + local edt = e.dt + matched = ns == action[3] and tg == action[4] + if action[2] then matched = not matched end + matched = matched and edt and edt[1] == action[5] + end + if matched then -- combine tg test and at test + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + if wildcard and multiple then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end + end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 14 then -- any + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root) then return false end + end + elseif command == 15 then -- many + if index == #pattern then + if handle(root,rootdt,root.ri or k) then return false end + else + if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end + end + -- not here : 11 + elseif command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end + elseif handle(root,rootdt,k) then + return false + end + elseif command == 40 and e.special and tg == "@pi@" then -- pi + local pi = action[2] + if pi ~= "" then + local pt = e.dt[1] + if pt and pt:find(pi) then + if handle(root,rootdt,k) then + return false + end + end + elseif handle(root,rootdt,k) then return false end + elseif wildcard then + if not traverse(e,pattern,handle,reverse,index,root,true) then return false end end - elseif handle(root,rootdt,k) then - return false end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,true) then return false end + else + -- not here : 11 + if command == 11 then -- parent + local ep = e.__p__ or parent + if index < #pattern then + if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end + elseif handle(root,rootdt,k) then + return false + end + break -- else loop + end end end end @@ -2636,15 +2568,71 @@ do return true end +end + +--[[ldx-- +<p>Next come all kind of locators and manipulators. The most generic function here +is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace +can be path of a search path, as in:</p> + +<typing> +local r, d, k = xml.filter(root,"/a/b/c/position(4)" +</typing> +--ldx]]-- + +do + local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert xml.filters = { } + --[[ldx-- + <p>For splitting the filter function from the path specification, we can + use string matching or lpeg matching. Here the difference in speed is + neglectable but the lpeg variant is more robust.</p> + --ldx]]-- + + -- function xml.filter(root,pattern) + -- local pat, fun, arg = pattern:match("^(.+)/(.-)%((.*)%)$") + -- if fun then + -- return (xml.filters[fun] or xml.filters.default)(root,pat,arg) + -- else + -- pat, arg = pattern:match("^(.+)/@(.-)$") + -- if arg then + -- return xml.filters.attributes(root,pat,arg) + -- else + -- return xml.filters.default(root,pattern) + -- end + -- end + -- end + + -- not faster but hipper ... although ... i can't get rid of the trailing / in the path + + local name = (lpeg.R("az","AZ")+lpeg.R("_-"))^1 + local path = lpeg.C(((1-lpeg.P('/'))^0 * lpeg.P('/'))^1) + local argument = lpeg.P { "(" * lpeg.C(((1 - lpeg.S("()")) + lpeg.V(1))^0) * ")" } + local action = lpeg.Cc(1) * path * lpeg.C(name) * argument + local attribute = lpeg.Cc(2) * path * lpeg.P('@') * lpeg.C(name) + + local parser = action + attribute + + function xml.filter(root,pattern) + local kind, a, b, c = parser:match(pattern) + if kind == 1 then + return (xml.filters[b] or xml.filters.default)(root,a,c) + elseif kind == 2 then + return xml.filters.attributes(root,a,b) + else + return xml.filters.default(root,pattern) + end + end + function xml.filters.default(root,pattern) local rt, dt, dk traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) return dt and dt[dk], rt, dt, dk end + function xml.filters.reverse(root,pattern) local rt, dt, dk traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') @@ -2698,17 +2686,14 @@ do traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) if i == 0 then return dt and dt[dk], rt, dt, dk - else - return nil, nil, nil, nil end - else - return nil, nil, nil, nil end + return nil, nil, nil, nil end function xml.filters.attributes(root,pattern,arguments) local rt, dt, dk traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = dt and dt[dk] and dt[dk].at + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) if ekat then if arguments then return ekat[arguments] or "", rt, dt, dk @@ -2722,69 +2707,33 @@ do function xml.filters.attribute(root,pattern,arguments) local rt, dt, dk traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = dt and dt[dk] and dt[dk].at + local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) return (ekat and ekat[arguments]) or "" end function xml.filters.text(root,pattern,arguments) - local ek, dt, dk, rt = xml.filters.index(root,pattern,arguments) - return (ek and ek.dt) or "", rt, dt, dk - end - - function xml.filter(root,pattern) - local pat, fun, arg = pattern:match("^(.+)/(.-)%((.*)%)$") - if fun then - return (xml.filters[fun] or xml.filters.default)(root,pat,arg) - else - pat, arg = pattern:match("^(.+)/@(.-)$") - if arg then - return xml.filters.attributes(root,pat,arg) + local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) + if dtk then + local dtkdt = dtk.dt + if #dtkdt == 1 and type(dtkdt[1]) == "string" then + return dtkdt[1], rt, dt, dk else - return xml.filters.default(root,pattern) + return xml.tostring(dtkdt), rt, dt, dk end + else + return "", rt, dt, dk end end - xml.filters.position = xml.filters.index - - -- these may go away - - xml.index_element = xml.filters.index - xml.count_elements = xml.filters.count - xml.first_element = xml.filters.first - xml.last_element = xml.filters.last - xml.index_text = xml.filters.text - xml.first_text = function (root,pattern) return xml.filters.text(root,pattern, 1) end - xml.last_text = function (root,pattern) return xml.filters.text(root,pattern,-1) end - - -- so far - - function xml.get_text(root,pattern,reverse) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end, reverse) - local ek = dt and dt[dk] - return (ek and ek.dt) or "", rt, dt, dk - end - - function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok - end - - function xml.get_element(root,pattern,reverse) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end, reverse) - return dt and dt[dk], rt, dt, dk - end + --[[ldx-- + <p>The following functions collect elements and texts.</p> + --ldx]]-- - -- these may change - - function xml.all_elements(root, pattern, ignorespaces) -- ok? + function xml.collect_elements(root, pattern, ignorespaces) local rr, dd = { }, { } traverse(root, lpath(pattern), function(r,d,k) local dk = d and d[k] if dk then - if ignorespaces and type(dk) == "string" and dk:find("^[\s\n]*$") then + if ignorespaces and type(dk) == "string" and dk:find("^%s*$") then -- ignore else local n = #rr+1 @@ -2795,8 +2744,8 @@ do return dd, rr end - function xml.all_texts(root, pattern, flatten) -- crap - local t, r = { }, { } + function xml.collect_texts(root, pattern, flatten) + local t = { } -- no r collector traverse(root, lpath(pattern), function(r,d,k) if d then local ek = d[k] @@ -2813,10 +2762,76 @@ do else t[#t+1] = "" end - r[#r+1] = r end) - return t, r + return t + end + + --[[ldx-- + <p>Often using an iterators looks nicer in the code than passing handler + functions. The <l n='lua'/> book describes how to use coroutines for that + purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits + code like:</p> + + <typing> + for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) end + </typing> + + <p>Which will print all the titles in the document. The iterator variant takes + 1.5 times the runtime of the function variant which si due to the overhead in + creating the wrapper. So, instead of:</p> + + <typing> + function xml.filters.first(root,pattern) + for rt,dt,dk in xml.elements(root,pattern) + return dt and dt[dk], rt, dt, dk + end + return nil, nil, nil, nil + end + </typing> + + <p>We use the function variants in the filters.</p> + --ldx]]-- + + function xml.elements(root,pattern,reverse) + return coroutine.wrap(function() traverse(root, lpath(pattern), coroutine.yield, reverse) end) + end + + function xml.each_element(root, pattern, handle, reverse) + local ok + traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) + return ok + end + + function xml.process_elements(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local dkdt = d[k].dt + if dkdt then + for i=1,#dkdt do + local v = dkdt[i] + if v.tg then handle(v) end + end + end + end) + end + + function xml.process_attributes(root, pattern, handle) + traverse(root, lpath(pattern), function(r,d,k) + local ek = d[k] + local a = ek.at or { } + handle(a) + if next(a) then + ek.at = a + else + ek.at = nil + end + end) + end + + --[[ldx-- + <p>We've now arrives at the functions that manipulate the tree.</p> + --ldx]]-- function xml.inject_element(root, pattern, element, prepend) if root and element then @@ -2868,7 +2883,7 @@ do function xml.insert_element(root, pattern, element, before) -- todo: element als functie if root and element then if pattern == "/" then - xml.inject_element(root, pattern, element, before) -- todo: element als functie + xml.inject_element(root, pattern, element, before) else local matches, collect = { }, nil if type(element) == "string" then @@ -2898,8 +2913,6 @@ do end end - -- first, last, each - xml.insert_element_after = xml.insert_element xml.insert_element_before = function(r,p,e) xml.insert_element(r,p,e,true) end xml.inject_element_after = xml.inject_element @@ -2930,24 +2943,47 @@ do end end - function xml.process(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - if d[k].dt then - for k,v in ipairs(d[k].dt) do - if v.tg then handle(v) end + function xml.include(xmldata,element,attribute,pathlist,collapse) + element = element or 'ctx:include' + attribute = attribute or 'name' + pathlist = pathlist or { '.' } + -- todo, check op ri + local function include(r,d,k) + local ek = d[k] + local name = (ek.at and ek.at[attribute]) or "" + if name ~= "" then + -- maybe file lookup in tree + local fullname + for _, path in ipairs(pathlist) do + if path == '.' then + fullname = name + else + fullname = file.join(path,name) + end + local f = io.open(fullname) + if f then + xml.assign(d,k,xml.load(f,collapse)) + f:close() + break + else + xml.empty(d,k) + end end + else + xml.empty(d,k) end - end) + end + while xml.each_element(xmldata, element, include) do end end - function xml.strip(root, pattern) + function xml.strip_whitespace(root, pattern) traverse(root, lpath(pattern), function(r,d,k) local dkdt = d[k].dt - if dkdt then + if dkdt then -- can be optimized local t = { } for i=1,#dkdt do local str = dkdt[i] - if type(str) == "string" and str:find("^[\032\010\012\013]*$") then + if type(str) == "string" and str:find("^[ \n\r\t]*$") then -- stripped else t[#t+1] = str @@ -2958,8 +2994,6 @@ do end) end - -- - function xml.rename_space(root, oldspace, newspace) -- fast variant local ndt = #root.dt local rename = xml.rename_space @@ -2968,6 +3002,9 @@ do if type(e) == "table" then if e.ns == oldspace then e.ns = newspace + if e.rn then + e.rn = newspace + end end local edt = e.dt if edt then @@ -2987,83 +3024,30 @@ do d[k].ns = newns end) end - - -- function xml.process_attributes(root, pattern, handle) - -- traverse(root, lpath(pattern), function(e,k) handle(e[k].at) end) - -- end - - function xml.process_attributes(root, pattern, handle) + function xml.check_namespace(root, pattern, newns) traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then - ek.at = a - else - ek.at = nil + local dk = d[k] + if (not dk.rn or dk.rn == "") and dk.ns == "" then + dk.rn = newns end end) end - - function xml.package(tag,attributes,data) - local n, t = tag:match("^(.-):(.+)$") - if attributes then - return { ns = n or "", tg = t or tag, dt = data or "", at = attributes } - else - return { ns = n or "", tg = t or tag, dt = data or "" } - end - end - - -- some special functions, handy for the manual: - - function xml.gsub(t,old,new) - if t.dt then - for k,v in ipairs(t.dt) do - if type(v) == "string" then - t.dt[k] = v:gsub(old,new) - else - xml.gsub(v,old,new) - end - end - end - end - - function xml.strip_leading_spaces(ek, e, k) -- cosmetic, for manual - if e and k and e[k-1] and type(e[k-1]) == "string" then - local s = e[k-1]:match("\n(%s+)") - xml.gsub(ek,"\n"..string.rep(" ",#s),"\n") - end - end - - function xml.serialize_path(root,lpath,handle) - local ek, e, k = xml.first_element(root,lpath) - ek = xml.copy(ek) - xml.strip_leading_spaces(ek,e,k) - xml.serialize(ek,handle) - end - - -- http://www.lua.org/pil/9.3.html (or of course the book) - -- - -- it's nice to have an iterator but it comes with some extra overhead - -- - -- for r, d, k in xml.elements(xml.load('text.xml'),"title") do print(d[k]) end - - function xml.elements(root,pattern,reverse) - return coroutine.wrap(function() traverse(root, lpath(pattern), coroutine.yield, reverse) end) + function xml.remap_name(root, pattern, newtg, newns, newrn) + traverse(root, lpath(pattern), function(r,d,k) + local dk = d[k] + dk.tg = newtg + dk.ns = newns + dk.rn = newrn + end) end - -- the iterator variant needs 1.5 times the runtime of the function variant - -- - -- function xml.filters.first(root,pattern) - -- for rt,dt,dk in xml.elements(root,pattern) - -- return dt and dt[dk], rt, dt, dk - -- end - -- return nil, nil, nil, nil - -- end +end - -- todo xml.gmatch for text +--[[ldx-- +<p>Here are a few synonyms.</p> +--ldx]]-- -end +xml.filters.position = xml.filters.index xml.count = xml.filters.count xml.index = xml.filters.index @@ -3072,7 +3056,10 @@ xml.first = xml.filters.first xml.last = xml.filters.last xml.each = xml.each_element -xml.all = xml.all_elements +xml.process = xml.process_element +xml.strip = xml.strip_whitespace +xml.collect = xml.collect_elements +xml.all = xml.collect_elements xml.insert = xml.insert_element_after xml.inject = xml.inject_element_after @@ -3081,39 +3068,38 @@ xml.before = xml.insert_element_before xml.delete = xml.delete_element xml.replace = xml.replace_element --- a few helpers, the may move to lxml modules +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- -function xml.include(xmldata,element,attribute,pathlist,collapse) - element = element or 'ctx:include' - attribute = attribute or 'name' - pathlist = pathlist or { '.' } - -- todo, check op ri - local function include(r,d,k) - local ek = d[k] - local name = (ek.at and ek.at[attribute]) or "" - if name ~= "" then - -- maybe file lookup in tree - local fullname - for _, path in ipairs(pathlist) do - if path == '.' then - fullname = name - else - fullname = file.join(path,name) - end - local f = io.open(fullname) - if f then - xml.assign(d,k,xml.load(f,collapse)) - f:close() - break - else - xml.empty(d,k) - end +function xml.gsub(t,old,new) + if t.dt then + for k,v in ipairs(t.dt) do + if type(v) == "string" then + t.dt[k] = v:gsub(old,new) + else + xml.gsub(v,old,new) end - else - xml.empty(d,k) end end - while xml.each(xmldata, element, include) do end +end + +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k and d[k-1] and type(d[k-1]) == "string" then + local s = d[k-1]:match("\n(%s+)") + xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") + end +end + +function xml.serialize_path(root,lpath,handle) + local dk, r, d, k = xml.first(root,lpath) + dk = xml.copy(dk) + xml.strip_leading_spaces(dk,d,k) + xml.serialize(dk,handle) end xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } @@ -3124,22 +3110,37 @@ function xml.unescaped(str) return str:gsub("(&.-;)", xml.unescapes) end function xml.cleansed (str) return str:gsub("<.->" , '' ) end -- "%b<>" function xml.join(t,separator,lastseparator) - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return table.join(result,separator,1,#result-1) .. lastseparator .. result[#result] + if #t > 0 then + local result = { } + for k,v in pairs(t) do + result[k] = xml.tostring(v) + end + if lastseparator then + return table.join(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] + else + return table.join(result,separator) + end else - return table.join(result,separator) + return "" end end -do if utf then +--[[ldx-- +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> +--ldx]]-- + +do if unicode and unicode.utf8 then + + xml.entities = xml.entities or { } -- xml.entities.handler == function + + local char = unicode.utf8.char local function toutf(s) - return utf.char(tonumber(s,16)) + return char(tonumber(s,16)) end function xml.utfize(root) @@ -3147,25 +3148,50 @@ do if utf then for k=1,#d do local dk = d[k] if type(dk) == "string" then - d[k] = dk:gsub("&#x(.-);",toutf) + -- test prevents copying if no match + if dk:find("&#x.-;") then + d[k] = dk:gsub("&#x(.-);",toutf) + end else xml.utfize(dk) end end end -else - function xml.utfize() - print("entity to utf conversion is not available") + + local entities = xml.entities + + local function resolve(e) + local e = entities[e] + if e then + return e + elseif e:find("#x") then + return char(tonumber(s:sub(3),16)) + else + local h = entities.handler + return (h and h(e)) or "&" .. e .. ";" + end end -end end + function xml.resolve_entities(root) + local d = root.dt + for k=1,#d do + local dk = d[k] + if type(dk) == "string" then + if dk:find("&.-;") then + d[k] = dk:gsub("&(.-);",resolve) + end + else + xml.utfize(dk) + end + end + end ---- examples +end end ---~ for _, e in ipairs(xml.filters.elements(ctxrunner.xmldata,"ctx:message")) do ---~ print(">>>",xml.tostring(e.dt)) ---~ end +--~ xml.lshow("/../../../a/(b|c)[@d='e']/f") +--~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") +--~ xml.lshow("/../../../a/!b[@d!='e']/f") -- filename : l-utils.lua diff --git a/scripts/context/ruby/base/pdf.rb b/scripts/context/ruby/base/pdf.rb index 9f4e9a6c3..5aec06fc5 100644 --- a/scripts/context/ruby/base/pdf.rb +++ b/scripts/context/ruby/base/pdf.rb @@ -7,7 +7,7 @@ module PDFview @method = 'default' # 'xpdf' - @opencalls['default'] = "pdfopen --file" + @opencalls['default'] = "pdfopen --file" # "pdfopen --back --file" @opencalls['xpdf'] = "xpdfopen" @closecalls['default'] = "pdfclose --file" diff --git a/scripts/context/ruby/base/tex.rb b/scripts/context/ruby/base/tex.rb index ceb9473c0..73b382af9 100644 --- a/scripts/context/ruby/base/tex.rb +++ b/scripts/context/ruby/base/tex.rb @@ -72,6 +72,7 @@ class TEX @@backends = Hash.new @@mappaths = Hash.new @@runoptions = Hash.new + @@tcxflag = Hash.new @@draftoptions = Hash.new @@texformats = Hash.new @@mpsformats = Hash.new @@ -169,13 +170,19 @@ class TEX ['cont-en','cont-nl','cont-de','cont-it', 'cont-fr','cont-cz','cont-ro','cont-uk'] .each do |f| @@texprocstr[f] = "\\emergencyend" end - # @@runoptions['xetex'] = ['--8bit','-output-driver="xdvipdfmx -E -d 4 -V 5 -q"'] - @@runoptions['xetex'] = ['--8bit','-output-driver="xdvipdfmx -E -d 4 -V 5"'] - @@runoptions['pdfetex'] = ['--8bit'] # obsolete - @@runoptions['pdftex'] = ['--8bit'] # pdftex is now pdfetex - @@runoptions['luatex'] = ['--file-line-error'] @@runoptions['aleph'] = ['--8bit'] + @@runoptions['luatex'] = ['--file-line-error'] @@runoptions['mpost'] = ['--8bit'] + @@runoptions['pdfetex'] = ['--8bit'] # obsolete + @@runoptions['pdftex'] = ['--8bit'] # pdftex is now pdfetex + @@runoptions['xetex'] = ['--8bit','-output-driver="xdvipdfmx -E -d 4 -V 5"'] + + @@tcxflag['aleph'] = true + @@tcxflag['luatex'] = false + @@tcxflag['mpost'] = true + @@tcxflag['pdfetex'] = true + @@tcxflag['pdftex'] = true + @@tcxflag['xetex'] = false @@draftoptions['pdftex'] = ['--draftmode'] @@ -540,11 +547,16 @@ class TEX "--ini" end end - def tcxflag(file="natural.tcx") - if Kpse.miktex? then - "-tcx=#{file}" + def tcxflag(engine) + if @@tcxflag[engine] then + file = "natural.tcx" + if Kpse.miktex? then + "-tcx=#{file}" + else + "-translate-file=#{file}" + end else - "-translate-file=#{file}" + "" end end @@ -661,7 +673,7 @@ class TEX texformats.each do |texformat| report("generating tex format #{texformat}") progname = validprogname([getvariable('progname'),texformat,texengine]) - runcommand([quoted(texengine),prognameflag(progname),iniflag,tcxflag,prefixed(texformat,texengine),texmakeextras(texformat)]) + runcommand([quoted(texengine),prognameflag(progname),iniflag,tcxflag(texengine),prefixed(texformat,texengine),texmakeextras(texformat)]) end end else @@ -682,7 +694,7 @@ class TEX mpsformats.each do |mpsformat| report("generating mps format #{mpsformat}") progname = validprogname([getvariable('progname'),mpsformat,mpsengine]) - if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,tcxflag,runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then + if not runcommand([quoted(mpsengine),prognameflag(progname),iniflag,tcxflag(mpsengine),runoptions(mpsengine),mpsformat,mpsmakeextras(mpsformat)]) then setvariable('error','no format made') end end @@ -1559,7 +1571,7 @@ end run_luatools("--fmt=#{texformat} #{filename}") else progname = validprogname([getvariable('progname'),texformat,texengine]) - runcommand([quoted(texengine),prognameflag(progname),formatflag(texengine,texformat),tcxflag,runoptions(texengine),filename,texprocextras(texformat)]) + runcommand([quoted(texengine),prognameflag(progname),formatflag(texengine,texformat),tcxflag(texengine),runoptions(texengine),filename,texprocextras(texformat)]) end # true else @@ -1574,8 +1586,7 @@ end if mpsengine && mpsformat then ENV["MPXCOMMAND"] = "0" unless mpx progname = validprogname([getvariable('progname'),mpsformat,mpsengine]) - runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),tcxflag,runoptions(mpsengine),mpname,mpsprocextras(mpsformat)]) - # runcommand([quoted(mpsengine),formatflag(mpsengine,mpsformat),tcxflag,runoptions(mpsengine),mpname,mpsprocextras(mpsformat)]) + runcommand([quoted(mpsengine),prognameflag(progname),formatflag(mpsengine,mpsformat),tcxflag(mpsengine),runoptions(mpsengine),mpname,mpsprocextras(mpsformat)]) true else false @@ -1790,10 +1801,10 @@ end forcexml = getvariable('forcexml') -if dummyfile || forcexml then # after ctx? - jobsuffix = makestubfile(rawname,rawbase,forcexml) - checkxmlfile(rawname) -end + if dummyfile || forcexml then # after ctx? + jobsuffix = makestubfile(rawname,rawbase,forcexml) + checkxmlfile(rawname) + end # preprocess files @@ -1929,7 +1940,15 @@ end end end # goto . + ok = runtex(File.suffixed(if dummyfile || forcexml then rawbase else rawname end,jobsuffix)) + +if getvariable('texengine') == "xetex" then + ok = true +end + +############################ + # goto tmp/jobname when present if ok && (nofruns > 1) then unless getvariable('nompmode') then diff --git a/scripts/context/ruby/base/texutil.rb b/scripts/context/ruby/base/texutil.rb index 726e31381..9c43f00e9 100644 --- a/scripts/context/ruby/base/texutil.rb +++ b/scripts/context/ruby/base/texutil.rb @@ -400,7 +400,7 @@ class TeXUtil def MyCommands::writer(logger,handle) handle << logger.banner("commands: #{@@commands.size}") @@commands.each do |c| - handle << "#{c}\n" + handle << "#{c}%\n" end end @@ -494,7 +494,7 @@ class TeXUtil end end list.each do |entry| - handle << "\\synonymentry{#{entry.type}}{#{entry.command}}{#{entry.key}}{#{entry.data}}\n" + handle << "\\synonymentry{#{entry.type}}{#{entry.command}}{#{entry.key}}{#{entry.data}}%\n" end end @@ -602,7 +602,7 @@ class TeXUtil end else # @entry, @key = cleanupsplit(@entry), cleanupsplit(@key) -@entry, @key = cleanupsplit(@entry), xcleanupsplit(@key) + @entry, @key = cleanupsplit(@entry), xcleanupsplit(@key) end @sortkey = sorter.simplify(@key) # special = @sortkey =~ /^([^a-zA-Z\\])/o @@ -632,23 +632,23 @@ class TeXUtil end end -def xcleanupsplit(target) # +a+b+c &a&b&c a+b+c a&b&c - t = Array.new - case target[0,1] - when '&' then - t = target.sub(/^./o,'').split(/([^\\])\&/o) - when '+' then - t = target.sub(/^./o,'').split(/([^\\])\+/o) - else - # t = target.split(/([^\\])[\&\+]/o) - # t = target.split(/[\&\+]/o) - t = target.split(/(?!\\)[\&\+]/o) # lookahead - end - if not t[1] then t[1] = " " end # we need some entry else we get subentries first - if not t[2] then t[2] = " " end # we need some entry else we get subentries first - return t.join(@@split) -end - + def xcleanupsplit(target) # +a+b+c &a&b&c a+b+c a&b&c + t = Array.new + case target[0,1] + when '&' then + t = target.sub(/^./o,'').split(/([^\\])\&/o) + when '+' then + t = target.sub(/^./o,'').split(/([^\\])\+/o) + else + # t = target.split(/([^\\])[\&\+]/o) + # t = target.split(/[\&\+]/o) + t = target.split(/(?!\\)[\&\+]/o) # lookahead + end + if not t[1] then t[1] = " " end # we need some entry else we get subentries first + if not t[2] then t[2] = " " end # we need some entry else we get subentries first + if not t[3] then t[3] = " " end # we need some entry else we get subentries first + return t.join(@@split) + end def <=> (other) @sortkey <=> other.sortkey end @@ -661,10 +661,10 @@ end def Register.flushsavedline(handle) if @@collapse && ! @@savedfrom.empty? then if ! @@savedto.empty? then - handle << "\\registerfrom#{@@savedfrom}" - handle << "\\registerto#{@@savedto}" + handle << "\\registerfrom#{@@savedfrom}%" + handle << "\\registerto#{@@savedto}%" else - handle << "\\registerpage#{@@savedfrom}" + handle << "\\registerpage#{@@savedfrom}%" end end @@savedhowto, @@savedfrom, @@savedto, @@savedentry = '', '', '', '' @@ -706,15 +706,15 @@ end elsif alpha == @@specialsymbol then character = @@specialbanner elsif alpha.length > 1 then - # character = "\\getvalue\{#{alpha}\}" - character = "\\#{alpha}" + # character = "\\getvalue\{#{alpha}\}%" + character = "\\#{alpha}%" else character = "\\unknown" end - handle << "\\registerentry{#{entry.type}}{#{character}}\n" + handle << "\\registerentry{#{entry.type}}{#{character}}%\n" end end - current = [entry.entry.split(@@split),'','',''].flatten + current = [entry.entry.split(@@split),'','','',''].flatten howto = current.collect do |e| e + '::' + entry.texthowto end @@ -724,38 +724,51 @@ end previous[0] = howto[0].dup previous[1] = '' previous[2] = '' + previous[3] = '' end if howto[1] == previous[1] then current[1] = '' else previous[1] = howto[1].dup previous[2] = '' + previous[3] = '' end if howto[2] == previous[2] then current[2] = '' else previous[2] = howto[2].dup + previous[3] = '' + end + if howto[3] == previous[3] then + current[3] = '' + else + previous[3] = howto[3].dup end copied = false unless current[0].empty? then Register.flushsavedline(handle) - handle << "\\registerentrya{#{entry.type}}{#{current[0]}}\n" + handle << "\\registerentrya{#{entry.type}}{#{current[0]}}%\n" copied = true end unless current[1].empty? then Register.flushsavedline(handle) - handle << "\\registerentryb{#{entry.type}}{#{current[1]}}\n" + handle << "\\registerentryb{#{entry.type}}{#{current[1]}}%\n" copied = true end unless current[2].empty? then Register.flushsavedline(handle) - handle << "\\registerentryc{#{entry.type}}{#{current[2]}}\n" + handle << "\\registerentryc{#{entry.type}}{#{current[2]}}%\n" + copied = true + end + unless current[3].empty? then + Register.flushsavedline(handle) + handle << "\\registerentryd{#{entry.type}}{#{current[3]}}%\n" copied = true end @nofentries += 1 if copied if entry.realpage.to_i == 0 then Register.flushsavedline(handle) - handle << "\\registersee{#{entry.type}}{#{entry.pagehowto},#{entry.texthowto}}{#{entry.seetoo}}{#{entry.page}}\n" ; + handle << "\\registersee{#{entry.type}}{#{entry.pagehowto},#{entry.texthowto}}{#{entry.seetoo}}{#{entry.page}}%\n" ; lastpage, lastrealpage = entry.page, entry.realpage copied = false # no page ! elsif @@savedhowto != entry.pagehowto and ! entry.pagehowto.empty? then @@ -763,14 +776,14 @@ end end # beware, we keep multiple page entries per realpage because of possible prefix usage if copied || ! ((lastpage == entry.page) && (lastrealpage == entry.realpage)) then - nextentry = "{#{entry.type}}{#{previous[0]}}{#{previous[1]}}{#{previous[2]}}{#{entry.pagehowto},#{entry.texthowto}}" + nextentry = "{#{entry.type}}{#{previous[0]}}{#{previous[1]}}{#{previous[2]}}{#{previous[3]}}{#{entry.pagehowto},#{entry.texthowto}}" savedline = "{#{entry.type}}{#{@@savedhowto},#{entry.texthowto}}{#{entry.location}}{#{entry.page}}{#{entry.realpage}}" if entry.state == 1 then # from Register.flushsavedline(handle) - handle << "\\registerfrom#{savedline}\n" + handle << "\\registerfrom#{savedline}%\n" elsif entry.state == 3 then # to Register.flushsavedline(handle) - handle << "\\registerto#{savedline}\n" + handle << "\\registerto#{savedline}%\n" @@savedhowto = '' # test elsif @@collapse then if savedentry != nextentry then @@ -779,7 +792,7 @@ end savedTo, savedentry = savedline, nextentry end else - handle << "\\registerpage#{savedline}\n" + handle << "\\registerpage#{savedline}%\n" @@savedhowto = '' # test end @nofpages += 1 @@ -1027,6 +1040,7 @@ end begin if f = File.open(File.suffixed(filename,'tuo'),'w') then @plugins.writers(f) + f << "\\endinput\n" f.close end rescue diff --git a/scripts/context/ruby/www/dir.rb b/scripts/context/ruby/www/dir.rb index 09e088d77..115fd8911 100644 --- a/scripts/context/ruby/www/dir.rb +++ b/scripts/context/ruby/www/dir.rb @@ -62,9 +62,9 @@ class WWW end u = dir_uri(@variables.get('path') || '.') str << "<div class='dir-view'>\n<pre>\n" - str << "<a href=\"#{u}&n=#{d1}\">name</A>".ljust(49+u.length) - str << "<a href=\"#{u}&m=#{d1}\">last modified</A>".ljust(41+u.length) - str << "<a href=\"#{u}&s=#{d1}\">size</A>".rjust(31+u.length) << "\n" << "\n" + str << "<a href=\"#{u}&n=#{d1}\">name</a>".ljust(49+u.length) + str << "<a href=\"#{u}&m=#{d1}\">last modified</a>".ljust(41+u.length) + str << "<a href=\"#{u}&s=#{d1}\">size</a>".rjust(31+u.length) << "\n" << "\n" # parent path if showdirs && ! hidden.include?('..') then dname = "parent directory" |